Flatten and Unflatten JSON

[taythebot]

I'm looking to adopt Benthos into our ingestion pipeline, but we require a flatten and unflatten function for JSON. I've read through the documentation and I think Bloblang is the answer.

Could someone help and provide an example for flatten and unflatten JSON? I'm quite lost on how to iterate JSON keys and values.

The desired input and output for the flatten function would be something like below. Unflatten should do the reverse.
Input:

{ "foo": { "bar": "hello world!" } }

Output:

{ "foo.bar": "hello world!" }

Thank you!

[mihaitodor]

Hey @taythebot! For flattening, try using the collapse() method. For example:

{"foo" :{"bar": {"baz": 666}}, "a": ["b", "c"]}

will be collapsed to:

{
  "a.0": "b",
  "a.1": "c",
  "foo.bar.baz": 666
}

To uncollapse the result back to the original input, there's currently no utility for it and it's a bit tricky to implement it since it would need to support arbitrary levels of intermixed objects arrays. And speaking of arrays, this utility would also need to check if there are gaps in the indexes, like having only "foo.0" and "foo.2" without "foo.1", and raise an error if it detects this. I tried to implement it using bloblang and I was able to expand each key into the correct structure, but then merging these resulting elements back into a single nested object isn't trivial to do correctly, especially when there are sibling subobjects and the order of elements in arrays must be preserved... Here's what I have so far (not sure if it helps you):

# Try to convert path elements to actual numbers so we can distinguish between arrays and objects in construct_from_path
map parse_path_element {
  let input = this
  root = match this.number(-1) {
     -1 => $input
     _ => this
  }
}

map get_reversed_fields {
    root = this.split(".").map_each(x -> x.apply("parse_path_element")).enumerated().sort(e -> e.left.index > e.right.index).map_each(e -> e.value)
}

map construct_from_path {
    let input = this
    root = this.path.fold(if this.path.index(0).type() == "string" { {} } else { [] }, node -> match node.value.type() {
      "string"  => { node.value: if $input.path.index(0) == node.value { $input.value } else { node.tally } }
      "number" => if node.tally.type() == "array" { if node.tally.length() > 0 { [ node.tally ] } else { node.tally.append($input.value) } } else { [ node.tally ] }
    })
}

map uncollapse {
    root = this.map_each(i -> {"path": i.key.apply("get_reversed_fields"), "value": i.value}.apply("construct_from_path")).values()
}

root = this.apply("uncollapse")

It should output the following for the collapsed JSON above:

[
  {
    "a": [
      "b"
    ]
  },
  {
    "a": [
      "c"
    ]
  },
  {
    "foo": {
      "bar": {
        "baz": 666
      }
    }
  }
]

Feel free to open an issue linking this discussion if you can provide some use cases where the uncollapse() method is useful.

[taythebot]

Thank you for the examples! Seems like the collapse method isn't exactly what we need. We don't collapse array values into separate key and value pairs. However, I did find out that you can write your own plugins. So we'll be trying that out.

Again thank you for your help!

[mihaitodor]

Try this:

map collapse_rec {
    let path = (this.path + ".") | ""
    let data = this.data
    root = $data.keys().map_each(k -> match $data.get(k) {
        this.type() == "object" => {"path": $path + k, "data": this}.apply("collapse_rec")
        this.type() == "array" => {$path + k: this.map_each(v -> match v.type() {
            "object" => v.apply("collapse")
            _ => v
        })}
        _ => {$path + k: this}
    }).squash()
}

map collapse {
    root = {"data": this}.apply("collapse_rec")
}

root = this.apply("collapse")

Sample input and output:

{
  "foo": {
    "bar": {
      "baz": 666
    },
    "blob": "fish"
  },
  "a": {
    "b": [
      "c",
      {
        "d": {
          "e": {
            "f": 123
          }
        },
        "x": {
          "y": {
            "z": 42
          }
        }
      }
    ]
  }
}

{
  "a.b": [
    "c",
    {
      "d.e.f": 123,
      "x.y.z": 42
    }
  ],
  "foo.bar.baz": 666,
  "foo.blob": "fish"
}

[taythebot]

Try this:

map collapse_rec {
    let path = (this.path + ".") | ""
    let data = this.data
    root = $data.keys().map_each(k -> match $data.get(k) {
        this.type() == "object" => {"path": $path + k, "data": this}.apply("collapse_rec")
        this.type() == "array" => {$path + k: this.map_each(v -> match v.type() {
            "object" => v.apply("collapse")
            _ => v
        })}
        _ => {$path + k: this}
    }).squash()
}

map collapse {
    root = {"data": this}.apply("collapse_rec")
}

root = this.apply("collapse")

Sample input and output:

{
  "foo": {
    "bar": {
      "baz": 666
    },
    "blob": "fish"
  },
  "a": {
    "b": [
      "c",
      {
        "d": {
          "e": {
            "f": 123
          }
        },
        "x": {
          "y": {
            "z": 42
          }
        }
      }
    ]
  }
}

{
  "a.b": [
    "c",
    {
      "d.e.f": 123,
      "x.y.z": 42
    }
  ],
  "foo.bar.baz": 666,
  "foo.blob": "fish"
}

This is awesome! Thank you so much. We'll try this out :)

[mihaitodor]

You're welcome! :)