Sorry for the weird title, not sure what to title this lol
I’m using Replicate API to try and get a point cloud in Unity, I finally go the API returning a JSON file, and I’m pretty certain it’s returning vector3s of the positions of each point. Yet, when i cast it to a vector3 the values get all out of whack, every number will be like, 6.852349e-43, and even some "NaN"s, when on the API they appear to be all values below 0.
here’s what they look like on Replicate’s page,
"coords": [
[
-0.039118800312280655,
-0.25740715861320496,
0.28050559759140015
],
[
0.13176295161247253,
0.23972494900226593,
-0.06013050675392151
],
Yet when i cast them to a vector 3 in Unity, they’re returning like this,
I’m assuming I just shouldn’t be casting them to a vector3? Replicate’s page says ""coords" is an [N x 3] array of (X,Y,Z) point coordinates" and after extensive Googling, i have no idea what NX3 means. I’ve tried float[,], float[,,], and float[][] instead of vector3, but vector3’s the only one that ever gets assigned properly from the JSON file, (using JsonUtility.FromJson)
Here’s my script calling the api,
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using UnityEngine.Networking;
using SimpleJSON;
public class Prompt
{
public string prompt;
}
public class PostResponse
{
public string completed_at;
public string created_at;
public string error;
public string id;
public Prompt input;
public string[] logs;
public string metrics;
public Output output;
public string started_at;
public string status;
public string version;
}
public class ReplicateAPI : MonoBehaviour
{
private string API_ENDPOINT = "https://api.replicate.com/v1/predictions";
private string REPLICATE_API_TOKEN = "r8_DhOE6C7LuA7edPvLh7Io0ukwbmXCLEY450vc3";
[SerializeField] PointEResponse response;
public string prompt = "dog";
[SerializeField] Material material;
[SerializeField] MeshFilter meshFilter;
private void Start()
{
StartCoroutine(Predict());
}
IEnumerator Predict()
{
string requestData = $@"{{
""version"": ""1a4da7adf0bc84cd786c1df41c02db3097d899f5c159f5fd5814a11117bdf02b"",
""input"": {{
""prompt"": ""{prompt}"",
""output_format"": ""json_file""
}}
}}";
UnityWebRequest request = UnityWebRequest.Put(API_ENDPOINT, requestData);
request.method = "POST";
request.SetRequestHeader("Authorization", "Token " + REPLICATE_API_TOKEN);
request.SetRequestHeader("Content-Type", "application/json");
yield return request.SendWebRequest();
if (request.result == UnityWebRequest.Result.ConnectionError || request.result == UnityWebRequest.Result.ProtocolError)
{
Debug.LogError("Error " + request.responseCode + ": " + request.error);
StartCoroutine(Predict());
}
else
{
Debug.Log(request.downloadHandler.text);
PostResponse response = JsonUtility.FromJson<PostResponse>(request.downloadHandler.text);
StartCoroutine(GetResult(response.id));
}
}
IEnumerator GetResult(string id)
{
yield return new WaitForSeconds(5);
UnityWebRequest request = UnityWebRequest.Get(API_ENDPOINT + "/" + id);
request.SetRequestHeader("Authorization", "Token " + REPLICATE_API_TOKEN);
request.SetRequestHeader("Content-Type", "application/json");
yield return request.SendWebRequest();
if (request.result == UnityWebRequest.Result.ConnectionError || request.result == UnityWebRequest.Result.ProtocolError)
{
Debug.LogError("Error " + request.responseCode + ": " + request.error);
yield break;
}
else
{
response = JsonUtility.FromJson<PointEResponse>(request.downloadHandler.text);
Debug.Log(request.downloadHandler.text);
}
if (response.status != "succeeded") StartCoroutine(GetResult(id));
else
{
if (response.output.json_file.colors != null && response.output.json_file.colors.Length > 0)
{
for (int i = 0; i < response.output.json_file.colors.Length; i++)
{
GameObject g = GameObject.CreatePrimitive(PrimitiveType.Sphere);
g.transform.localScale = Vector3.one / 100;
g.transform.position = new Vector3(response.output.json_file.coords[i].x, response.output.json_file.coords[i].y, response.output.json_file.coords[i].z);
Material m = new Material(material);
m.color = new Color(response.output.json_file.colors[i].x, response.output.json_file.colors[i].y, response.output.json_file.colors[i].z);
g.GetComponent<MeshRenderer>().material = m;
}
}
else
{
StartCoroutine(GetResult(id));
}
}
}
}
And the "response" classes look like this,
using System;
using UnityEngine;
[Serializable]
public class PointEResponse
{
public string id;
public string version;
public Urls urls;
public DateTime created_at;
public DateTime started_at;
public DateTime completed_at;
public string source;
public string status;
public Prompt input;
public Output output;
public object error;
public string[] logs;
public Metrics metrics;
}
[Serializable]
public class Output
{
public JsonFile json_file;
}
[Serializable]
public class JsonFile
{
public Vector3[] coords;
public Vector3[] colors;
}
public class Urls
{
public string get;
public string cancel;
}
public class Metrics
{
public float predict_time;
}
Where the output contains the coordinates and the colours of each coordinate. I’m assuming i’m not supposed to be casting these to a Vector3, but I’ve tried everything else I know of any the only one that even returns any form
Thanks in advance if anyone can give me some insight on this.
>Solution :
The JSON you get here would not match a Vector3 directly.
Afaik a Vector3 is not directly JSON (de)serializable.
"coords": [
[
-0.039118800312280655,
-0.25740715861320496,
0.28050559759140015
],
[
0.13176295161247253,
0.23972494900226593,
-0.06013050675392151
],
means you get something rather looking like
[Serializable]
public class JsonFile
{
public float[][] coords;
// assuming same layout for those
public float[][] colors;
}
If and how exactly this would be laid out in memory into a Vector3 structs is dark magic I guess.
This type of complex collection however is also NOT SPPORTED by the built-in JsonUtility/Unity’s Serializer!
You would rather need a third-party library such as e.g. Newtonsoft Json.NET which comes as a package via Package Manager
using that instead you would go e.g.
using UnityEngine.Scripting;
using Newtonsoft.Json;
...
[Serializable]
public class JsonFile
{
public Vector3[] coords;
public Color[] colors;
[Preserve]
[JsonConstructor]
public JsonFile(float[][] coords, float[][] colors)
{
this.coords = new Vector3[coords.GetLength(0)];
for (var i = 0; i < this.coords.Length; i++)
{
this.coords[i] = new Vector3(coords[i][0], coords[i][1], coords[i][2]);
}
this.colors = new Color[colors.GetLength(0)];
for(var i = 0; i < this.colors.Length; i++)
{
this.colors[i] = new Color(colors[i][0], colors[i][1], colors[i][2]);
}
}
}
and then go
var response = JsonConvert.DeserializeObject<PostResponse>(request.downloadHandler.text);
btw since Vector3 is a struct you an simply do
g.transform.position = response.output.json_file.coords[i];
...
m.color = nresponse.output.json_file.colors[i];