鸟语天空

百度AI开放平台接入

作者：追风剑情 发布于：2021-8-9 17:39 分类：Unity3d

using System;
using System.Collections;
using System.Collections.Generic;
using System.Net.Http;
using UnityEngine;
using UnityEngine.Networking;
/// <summary>
/// 百度AI开放平台
/// 人像分割 https://ai.baidu.com/tech/body/seg
/// 在线文档 https://ai.baidu.com/ai-doc/BODY/Fk3cpyxua
/// 错误码查询 https://ai.baidu.com/ai-doc/IMAGERECOGNITION/Rk3bcxhio
/// 
/// 人像动漫化 https://ai.baidu.com/tech/imageprocess/selfie_anime
/// 在线文档 https://ai.baidu.com/ai-doc/IMAGEPROCESS/Mk4i6olx5
/// </summary>
public class BaiduAISDK : MonoBehaviour
{
    //人像分割请求地址
    public const string BODY_SEG_URL = "https://aip.baidubce.com/rest/2.0/image-classify/v1/body_seg";
    public const string BODY_SEG_APP_ID = "";
    public const string BODY_SEG_APP_KEY = "";
    public const string BODY_SEG_APP_SECRET = "";
    public AccessTokenResult bodySegAccessTokenResult;
    //人像分割完成
    public Action<Texture2D, Texture2D, Texture2D> BodySegCompleted;
    //未检测到人像
    public Action OnNotFoundPersionError;
    //人像分割失败
    public Action BodySegError;
    //是否从人像分割图中裁剪出人物图片
    public bool IsClipPersion = false;
    //是否将人像上下居中显示
    public bool IsVeticallyMiddle = false;
 
    //人像动漫化请求地址
    public const string SELFIE_ANIME_URL = "https://aip.baidubce.com/rest/2.0/image-process/v1/selfie_anime";
    public const string SELFIE_ANIME_APP_ID = "";
    public const string SELFIE_ANIME_APP_KEY = "";
    public const string SELFIE_ANIME_APP_SECRET = "";
    public AccessTokenResult selfieAnimeAccessTokenResult;
    //人像动漫化完成
    public Action<Texture2D> SelfieAnimeCompleted;
    //人像动漫化失败
    public Action SelfieAnimeError;
 
    //证书验证
    public class AcceptCertificateHandler : CertificateHandler
    {
        //负责拒绝或接受在 https 请求上接收到的证书
        protected override bool ValidateCertificate(byte[] certificateData)
        {
            return true;
        }
    }
 
    private void Start()
    {
        Debug.Log("*** request body_seg access token...");
        bodySegAccessTokenResult = GetAccessToken(BODY_SEG_APP_KEY, BODY_SEG_APP_SECRET);
        if (string.IsNullOrEmpty(bodySegAccessTokenResult.access_token))
            Debug.LogError("request body_seg access token failure");
 
        Debug.Log("*** request selfie_anime access token...");
        selfieAnimeAccessTokenResult = GetAccessToken(SELFIE_ANIME_APP_KEY, SELFIE_ANIME_APP_SECRET);
        if (string.IsNullOrEmpty(selfieAnimeAccessTokenResult.access_token))
            Debug.LogError("request selfie_anime access token failure");
    }
 
    //base64转Texture2D
    private Texture2D Base64ToTexture(string base64, int width, int height)
    {
        byte[] imgBytes = Convert.FromBase64String(base64);
        Texture2D new_tex = new Texture2D(width, height, TextureFormat.RGBA32, false);
        new_tex.LoadImage(imgBytes);
        new_tex.Apply();
        return new_tex;
    }
 
    //去掉背景RGB值
    private void StripBackground(Texture2D tex)
    {
        Color32[] colors = tex.GetPixels32();
        for (int i = 0; i < colors.Length; i++)
        {
            Color32 c = colors[i];
            //人像边缘附近的alpha值介于0到1
            //根据文档上的说明，前景的alpha值大于0.5
            if (c.a <= 0.5f)
            {
                c.a = c.r = c.g = c.b = 0;
                colors[i] = c;
            }
        }
        tex.SetPixels32(colors);
        tex.Apply();
    }
 
    //生成一张Alpha图
    private Texture2D GenerateAlphaTexture(Texture2D tex)
    {
        Color32[] colors = tex.GetPixels32();
        for (int i = 0; i < colors.Length; i++)
        {
            Color32 c = colors[i];
            c.r = c.g = c.b = c.a;
            colors[i] = c;
        }
 
        Texture2D alpha_tex = new Texture2D(tex.width, tex.height, TextureFormat.RGBA32, false);
        alpha_tex.SetPixels32(colors);
        alpha_tex.Apply();
 
        return alpha_tex;
    }
 
    //生成裁剪后的人像图
    private Texture2D ClipPersionTexture(Texture2D tex, BodySegResult result)
    {
        if (result.person_info == null || result.person_info.Length == 0)
            return null;
        var rect = result.person_info[0];
        Debug.LogFormat("person_num={0}", result.person_num);
        Debug.LogFormat("(left={0}, top={1}, width={2}, height={3})",
            (int)rect.left, (int)rect.top, (int)rect.width, (int)rect.height);
 
        //默认返回的人体框top不准确，这里进行重新定位
        rect.top = CalculatePersionTop(tex, result);
 
        //Texture的原点坐标在左下角
        int x = (int)rect.left;
        int y = tex.height - (int)rect.top - (int)rect.height;
        y = Mathf.Clamp(y, 0, tex.height - 1);
        int width = (int)rect.width;
        int height = (int)rect.height;
 
        Color[] colors = tex.GetPixels(x, y, width, height);
        Texture2D new_tex = new Texture2D(width, height, TextureFormat.RGBA32, false);
        new_tex.SetPixels(colors);
        new_tex.Apply();
        return new_tex;
    }
 
    //计算人像top
    private int CalculatePersionTop(Texture2D tex, BodySegResult result)
    {
        int top = 0;
        if (result.person_info == null || result.person_info.Length == 0)
            return top;
        var rect = result.person_info[0];
        //从上到下扫描图片
        int y = tex.height - (int)rect.top;
        for (; y > 0; y--)
        {
            for (int x = 0; x < tex.width; x++)
            {
                Color c = tex.GetPixel(x, y);
                //根据百度返回的人体区域Alpha通道概率分数重新定位人体区域top
                if (c.a >= rect.score)
                {
                    top = tex.height - y;
                    goto FOUND;
                }
            }
        }
        FOUND:
        return top;
    }
 
    //计算人像bottom
    private int CalculatePersionBottom(Texture2D tex, BodySegResult result)
    {
        int bottom = 0;
        if (result.person_info == null || result.person_info.Length == 0)
            return bottom;
        var rect = result.person_info[0];
        //从下到上扫描图片
        int topY = tex.height - (int)rect.top;
        int y = tex.height - (int)rect.top - (int)rect.height;
        y = Mathf.Clamp(y, 0, tex.height);
        for (; y < topY; y++)
        {
            for (int x = 0; x < tex.width; x++)
            {
                Color c = tex.GetPixel(x, y);
                if (c.a >= rect.score)
                {
                    bottom = y;
                    goto FOUND;
                }
            }
        }
        FOUND:
        return bottom;
    }
 
    //将人像上下居中
    private Texture2D VeticallyMiddleTexture(Texture2D tex, BodySegResult result)
    {
        int top = CalculatePersionTop(tex, result);
        int bottom = CalculatePersionBottom(tex, result);
        int middleX = 0;
        int middleY = (top + bottom) / 2;
 
        //提取人像区域color
        int x = 0;
        int y = bottom;
        int width = tex.width;
        int height = tex.height - top - bottom;
        Color[] colors = tex.GetPixels(x, y, width, height);
 
        Debug.LogFormat("top={0}, bottom={1}, middleX={2}, middleY={3}, width={4}, height={5}",
            top, bottom, middleX, middleY, width, height);
 
        //将人像上下居中
        Texture2D new_tex = new Texture2D(tex.width, tex.height, TextureFormat.RGBA32, false);
        new_tex.SetPixels(middleX, middleY, width, height, colors);
        Color transparent = new Color(0, 0, 0, 0);
        SetPixelRows(new_tex, middleY + height, tex.height, transparent);
        SetPixelRows(new_tex, 0, middleY, transparent);
        new_tex.Apply();
        return new_tex;
    }
 
    private void SetPixelRows(Texture2D tex, int startY, int endY, Color color)
    {
        for (int i = startY; i <= endY; i++)
            SetPixelRow(tex, i, color);
    }
 
    private void SetPixelRow(Texture2D tex, int y, Color color)
    {
        for (int x=0; x < tex.width; x++)
        {
            if (y < tex.height)
                tex.SetPixel(x, y, color);
        }
    }
 
    // 合并Alpha通道
    public Texture2D CombineAlphaTexture(Texture2D rgbTex, Texture2D alphaTex)
    {
        Texture2D newTex = new Texture2D(rgbTex.width, rgbTex.height, TextureFormat.RGBA32, false);
        for (int y=0; y<rgbTex.height; y++)
        {
            for (int x = 0; x < rgbTex.width; x++)
            {
                Color c = rgbTex.GetPixel(x, y);
                Color alpha = alphaTex.GetPixel(x, y);
                c.a = alpha.a;
                newTex.SetPixel(x, y, c);
            }
        }
        newTex.Apply();
        return newTex;
    }
 
    //获取 Access Token
    public AccessTokenResult GetAccessToken(string appKey, string secretKey)
    {
        String authHost = "https://aip.baidubce.com/oauth/2.0/token";
        HttpClient client = new HttpClient();
        List<KeyValuePair<String, String>> paraList = new List<KeyValuePair<string, string>>();
        paraList.Add(new KeyValuePair<string, string>("grant_type", "client_credentials"));
        paraList.Add(new KeyValuePair<string, string>("client_id", appKey));
        paraList.Add(new KeyValuePair<string, string>("client_secret", secretKey));
 
        HttpResponseMessage response = client.PostAsync(authHost, new FormUrlEncodedContent(paraList)).Result;
        String result = response.Content.ReadAsStringAsync().Result;
        Debug.Log(result);
        var token = JsonUtility.FromJson<AccessTokenResult>(result);
        return token;
    }
 
    //人像分割
    public void AsyncBodySeg(Texture2D tex)
    {
        if (tex == null)
            return;
        StartCoroutine(Post(BODY_SEG_URL, bodySegAccessTokenResult.access_token,
            "foreground", tex, 
            //完成
            (json) => {
                var result = JsonUtility.FromJson<BodySegResult>(json);
                if (!string.IsNullOrEmpty(result.error_msg))
                {
                    BodySegError?.Invoke();
                    return;
                }
                if (result.person_num == 0)
                {
                    OnNotFoundPersionError?.Invoke();
                    return;
                }
                Texture2D new_tex = Base64ToTexture(result.foreground, tex.width, tex.height);
                Debug.LogFormat("Respose image size: width={0}, height={1}", new_tex.width, new_tex.height);
                Texture2D persion_tex;
                if (IsClipPersion)
                {
                    new_tex = ClipPersionTexture(new_tex, result);
                    persion_tex = new_tex;
                }
                else
                {
                    persion_tex = ClipPersionTexture(new_tex, result);
                }
 
                if (IsVeticallyMiddle)
                {
                    new_tex = VeticallyMiddleTexture(new_tex, result);
                }
                Texture2D alpha_tex = GenerateAlphaTexture(new_tex);
                BodySegCompleted?.Invoke(new_tex, alpha_tex, persion_tex);
            },
            //失败
            () => {
                BodySegError?.Invoke();
            }));
        //注意：分割后的图片会将背景区的alpha设为0，同时会保留背景区的RGB值。
    }
 
    //人像动漫化
    public void AsyncSelfieAnime(Texture2D tex)
    {
        if (tex == null)
            return;
 
        StartCoroutine(Post(SELFIE_ANIME_URL, selfieAnimeAccessTokenResult.access_token, 
            "anime", tex,
            //完成
            (json) => {
                var result = JsonUtility.FromJson<SelfieAnimeResult>(json);
                if (!string.IsNullOrEmpty(result.error_msg))
                {
                    SelfieAnimeError?.Invoke();
                    return;
                }
                Texture2D new_tex = Base64ToTexture(result.image, tex.width, tex.height);
                SelfieAnimeCompleted?.Invoke(new_tex);
            },
            //失败
            () => {
                SelfieAnimeError?.Invoke();
            }));
        //注意：返回的图片没有alpha通道
    }
 
    //POST 请求
    private IEnumerator Post(string url, string accessToken, string type, Texture2D tex, Action<string> completedCallback, Action errorCallback)
    {
        yield return null;
        byte[] bytes = tex.EncodeToPNG();
        //byte[] bytes = tex.EncodeToJPG();
        string base64 = Convert.ToBase64String(bytes);
 
        //创建表单
        WWWForm form = new WWWForm();
        form.AddField("type", type);
        form.AddField("image", base64);
 
        url = string.Format("{0}?access_token={1}", url, accessToken);
        //执行请求
        UnityWebRequest request = UnityWebRequest.Post(url, form);
        //使用https需要设置certificateHandler
        request.certificateHandler = new AcceptCertificateHandler();
        request.useHttpContinue = false;
        request.chunkedTransfer = false;
        request.timeout = 10;
        request.downloadHandler = (DownloadHandler)new DownloadHandlerBuffer();
        request.SetRequestHeader("Content-Type", "application/x-www-form-urlencoded");
        request.SendWebRequest();
 
        while (!request.isDone)
        {
            Debug.LogFormat("上传进度: {0}", request.uploadProgress);
            yield return new WaitForSeconds(0.1f);
        }
 
        if (request.isHttpError || request.isNetworkError || request.responseCode != 200)
        {
            Debug.LogErrorFormat("上传出错\n{0}", request.error);
            errorCallback?.Invoke();
            yield break; //跳出协程
        }
 
        Debug.Log("上传成功!");
 
        string json = request.downloadHandler.text;
        //-------常见错误-------
        //{"error_code":18,"error_msg":"Open api qps request limit reached"}
        //原因: 请求次数已达上限
        //解决方法: 控制台领取免费额度，或者充值。
 
        //{"error_code":6,"error_msg":"No permission to access data"}
        //没有请求权限
        //解决方法: 控制台领取免费额度，或者充值。
        //--------End-----------
        Debug.Log(json);
 
        completedCallback?.Invoke(json);
    }
 
    //Access Token 返回结果
    public class AccessTokenResult
    {
        public string refresh_token;
        public string expires_in;
        public string scope;
        public string session_key;
        public string access_token;
        public string session_secret;
 
        public string error = string.Empty;
        public string error_description = string.Empty;
    }
 
    //人像分割返回结果
    public class BodySegResult
    {
        //分割结果图片，base64编码之后的二值图像，需二次处理方能查看分割效果
        public string labelmap;
        /*
         分割后人像前景的scoremap，归一到0-255，不用进行二次处理，直接解码保存图片即可。
         Base64编码后的灰度图文件，图片中每个像素点的灰度值 = 置信度 * 255，
         置信度为原图对应像素点位于人体轮廓内的置信度，取值范围[0, 1]
         */
        public string scoremap;
        /*
         分割后的人像前景抠图，透明背景，Base64编码后的png格式图片，不用进行二次处理，
         直接解码保存图片即可。将置信度大于0.5的像素抠出来，
         并通过image matting技术消除锯齿
         */
        public string foreground;
        //检测到的人体框数目
        public int person_num;
        //人体框信息
        public PersonRect[] person_info;
 
        [Serializable]
        public class PersonRect
        {
            //人体区域的高度，注意当值为0时 数据类型为int
            public float height;
            //人体区域离左边界的距离，注意当值为0时 数据类型为int
            public float left;
            //人体区域离上边界的距离，注意当值为0时 数据类型为int
            public float top;
            //人体区域的宽度，注意当值为0时 数据类型为int
            public float width;
            //人体框的概率分数，取值0-1，，注意当值为0时 数据类型为int
            //alpha值>=score的区域即为人像区域
            public float score;
        }
 
        public int error_code;
        public string error_msg;
    }
 
    //人像动漫化返回
    public class SelfieAnimeResult
    {
        //唯一的log id，用于问题定位
        public UInt64 log_id;
        //处理后图片的Base64编码
        public string image;
 
        public int error_code;
        public string error_msg;
    }
}

原图

分割出来的人像以及动漫化后的人像

标签: Unity3d