|
| 1 | +/* |
| 2 | + * Copyright 2025 Google LLC |
| 3 | + * |
| 4 | + * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | + * you may not use this file except in compliance with the License. |
| 6 | + * You may obtain a copy of the License at |
| 7 | + * |
| 8 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | + * |
| 10 | + * Unless required by applicable law or agreed to in writing, software |
| 11 | + * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | + * See the License for the specific language governing permissions and |
| 14 | + * limitations under the License. |
| 15 | + */ |
| 16 | + |
| 17 | +package com.google.cloud.storage; |
| 18 | + |
| 19 | +import com.google.api.client.json.GenericJson; |
| 20 | +import com.google.api.client.json.JsonObjectParser; |
| 21 | +import com.google.api.client.json.gson.GsonFactory; |
| 22 | +import com.google.cloud.storage.UnifiedOpts.NamedField; |
| 23 | +import com.google.common.annotations.VisibleForTesting; |
| 24 | +import com.google.common.collect.ImmutableSet; |
| 25 | +import com.google.common.collect.MapDifference; |
| 26 | +import com.google.common.collect.MapDifference.ValueDifference; |
| 27 | +import com.google.common.collect.Maps; |
| 28 | +import com.google.gson.Gson; |
| 29 | +import com.google.gson.GsonBuilder; |
| 30 | +import com.google.gson.JsonArray; |
| 31 | +import com.google.gson.JsonElement; |
| 32 | +import com.google.gson.JsonNull; |
| 33 | +import com.google.gson.JsonObject; |
| 34 | +import com.google.gson.JsonPrimitive; |
| 35 | +import java.io.IOException; |
| 36 | +import java.io.StringReader; |
| 37 | +import java.util.HashMap; |
| 38 | +import java.util.List; |
| 39 | +import java.util.Map; |
| 40 | +import java.util.Map.Entry; |
| 41 | +import java.util.Set; |
| 42 | +import java.util.regex.Matcher; |
| 43 | +import java.util.regex.Pattern; |
| 44 | +import java.util.stream.Stream; |
| 45 | +import org.checkerframework.checker.nullness.qual.NonNull; |
| 46 | + |
| 47 | +final class JsonUtils { |
| 48 | + |
| 49 | + private static final Gson gson = |
| 50 | + new GsonBuilder() |
| 51 | + // ensure null values are not stripped, they are important to us |
| 52 | + .serializeNulls() |
| 53 | + .setPrettyPrinting() |
| 54 | + .create(); |
| 55 | + @VisibleForTesting static final JsonObjectParser jop = new JsonObjectParser(new GsonFactory()); |
| 56 | + private static final Pattern array_part = Pattern.compile("(.*)\\[(\\d+)]"); |
| 57 | + |
| 58 | + private JsonUtils() {} |
| 59 | + |
| 60 | + /** |
| 61 | + * Given a GenericJson src, and a list of {@code fieldsForOutput} create a new GenericJson where |
| 62 | + * every field specified in {@code fieldsForOutput} is present. If a field exists in {@code src} |
| 63 | + * with a specified name, that value will be used. If the field does not exist in {@code src} it |
| 64 | + * will be set to {@code null}. |
| 65 | + */ |
| 66 | + static <T extends GenericJson> T getOutputJsonWithSelectedFields( |
| 67 | + T src, Set<NamedField> fieldsForOutput) { |
| 68 | + Set<String> fieldPaths = |
| 69 | + fieldsForOutput.stream() |
| 70 | + .map(NamedField::getApiaryName) |
| 71 | + .collect(ImmutableSet.toImmutableSet()); |
| 72 | + try { |
| 73 | + // The datamodel of the apiairy json representation doesn't have a common parent for all |
| 74 | + // field types, rather than writing a significant amount of code to handle all of these types |
| 75 | + // leverage Gson. |
| 76 | + // 1. serialize the object to it's json string |
| 77 | + // 2. load that back with gson |
| 78 | + // 3. use gson's datamodel which is more sane to allow named field traversal and cross |
| 79 | + // selection |
| 80 | + // 4. output the json string of the resulting gson object |
| 81 | + // 5. deserialize the json string to the apiary model class. |
| 82 | + String string = jop.getJsonFactory().toPrettyString(src); |
| 83 | + JsonObject jsonObject = gson.fromJson(string, JsonObject.class); |
| 84 | + JsonObject ret = getOutputJson(jsonObject, fieldPaths); |
| 85 | + String json = gson.toJson(ret); |
| 86 | + Class<? extends GenericJson> aClass = src.getClass(); |
| 87 | + //noinspection unchecked |
| 88 | + Class<T> clazz = (Class<T>) aClass; |
| 89 | + return jop.parseAndClose(new StringReader(json), clazz); |
| 90 | + } catch (IOException e) { |
| 91 | + // StringReader does not throw an IOException |
| 92 | + throw StorageException.coalesce(e); |
| 93 | + } |
| 94 | + } |
| 95 | + |
| 96 | + /** |
| 97 | + * Given the provided {@code inputJson} flatten it to a Map<String, String> where keys are the |
| 98 | + * field path, and values are the string representation of the value. Then, create a |
| 99 | + * Map<String, String> by defining an entry for each value from {@code fieldsInOutput} with a |
| 100 | + * null value. Then, diff the two maps retaining those entries that present in both, and adding |
| 101 | + * entries that only exist in the right. Then, turn that diffed map back into a tree. |
| 102 | + */ |
| 103 | + @VisibleForTesting |
| 104 | + static @NonNull JsonObject getOutputJson(JsonObject inputJson, Set<String> fieldsInOutput) { |
| 105 | + |
| 106 | + Map<String, String> l = flatten(inputJson); |
| 107 | + Map<String, String> r = Utils.setToMap(fieldsInOutput, k -> null); |
| 108 | + |
| 109 | + MapDifference<String, String> diff = Maps.difference(l, r); |
| 110 | + |
| 111 | + // use hashmap so we can have null values |
| 112 | + HashMap<String, String> flat = new HashMap<>(); |
| 113 | + Stream.of( |
| 114 | + diff.entriesInCommon().entrySet().stream(), |
| 115 | + diff.entriesOnlyOnRight().entrySet().stream(), |
| 116 | + // if the key is present in both maps, but has a differing value select the value from |
| 117 | + // the left side, as that is the value from inputJson |
| 118 | + Maps.transformValues(diff.entriesDiffering(), ValueDifference::leftValue) |
| 119 | + .entrySet() |
| 120 | + .stream()) |
| 121 | + // flatten |
| 122 | + .flatMap(x -> x) |
| 123 | + .forEach(e -> flat.put(e.getKey(), e.getValue())); |
| 124 | + |
| 125 | + return treeify(flat); |
| 126 | + } |
| 127 | + |
| 128 | + /** |
| 129 | + * Given a {@link JsonObject} produce a map where keys represent the full field path using json |
| 130 | + * traversal notation ({@code a.b.c.d}) and the value is the string representations of that leaf |
| 131 | + * value. |
| 132 | + * |
| 133 | + * <p>Inverse of {@link #treeify(Map)} |
| 134 | + * |
| 135 | + * @see #treeify |
| 136 | + */ |
| 137 | + @VisibleForTesting |
| 138 | + static Map<String, String> flatten(JsonObject o) { |
| 139 | + // use hashmap so we can have null values |
| 140 | + HashMap<String, String> ret = new HashMap<>(); |
| 141 | + for (Entry<String, JsonElement> e : o.asMap().entrySet()) { |
| 142 | + ret.putAll(flatten(e.getKey(), e.getValue())); |
| 143 | + } |
| 144 | + return ret; |
| 145 | + } |
| 146 | + |
| 147 | + /** |
| 148 | + * Given a map where keys represent json field paths and values represent values, produce a {@link |
| 149 | + * JsonObject} with the tree structure matching those paths and values. |
| 150 | + * |
| 151 | + * <p>Inverse of {@link #flatten(JsonObject)} |
| 152 | + * |
| 153 | + * @see #flatten(JsonObject) |
| 154 | + */ |
| 155 | + @VisibleForTesting |
| 156 | + static JsonObject treeify(Map<String, String> m) { |
| 157 | + JsonObject o = new JsonObject(); |
| 158 | + for (Entry<String, String> e : m.entrySet()) { |
| 159 | + String key = e.getKey(); |
| 160 | + String[] splits = key.split("\\."); |
| 161 | + String leaf = splits[splits.length - 1]; |
| 162 | + |
| 163 | + JsonElement curr = o; |
| 164 | + int currIdx = -1; |
| 165 | + for (int i = 0, splitsEnd = splits.length, leafIdx = splitsEnd - 1; i < splitsEnd; i++) { |
| 166 | + final String name; |
| 167 | + final int idx; |
| 168 | + { |
| 169 | + String split = splits[i]; |
| 170 | + Matcher matcher = array_part.matcher(split); |
| 171 | + if (matcher.matches()) { |
| 172 | + name = matcher.group(1); |
| 173 | + String idxString = matcher.group(2); |
| 174 | + idx = Integer.parseInt(idxString); |
| 175 | + } else { |
| 176 | + idx = -1; |
| 177 | + name = split; |
| 178 | + } |
| 179 | + } |
| 180 | + |
| 181 | + if (curr.isJsonObject()) { |
| 182 | + if (i != leafIdx) { |
| 183 | + curr = |
| 184 | + curr.getAsJsonObject() |
| 185 | + .asMap() |
| 186 | + .computeIfAbsent( |
| 187 | + name, |
| 188 | + s -> { |
| 189 | + if (idx > -1) { |
| 190 | + return new JsonArray(); |
| 191 | + } |
| 192 | + return new JsonObject(); |
| 193 | + }); |
| 194 | + } else if (idx > -1) { |
| 195 | + curr = curr.getAsJsonObject().asMap().computeIfAbsent(name, s -> new JsonArray()); |
| 196 | + } |
| 197 | + if (currIdx == -1) { |
| 198 | + currIdx = idx; |
| 199 | + } else { |
| 200 | + currIdx = -1; |
| 201 | + } |
| 202 | + } |
| 203 | + |
| 204 | + if (curr.isJsonArray()) { |
| 205 | + JsonArray a = curr.getAsJsonArray(); |
| 206 | + int size = a.size(); |
| 207 | + int nullElementsToAdd = 0; |
| 208 | + if (size < currIdx) { |
| 209 | + nullElementsToAdd = currIdx - size; |
| 210 | + } |
| 211 | + |
| 212 | + for (int j = 0; j < nullElementsToAdd; j++) { |
| 213 | + a.add(JsonNull.INSTANCE); |
| 214 | + } |
| 215 | + } |
| 216 | + |
| 217 | + if (i == leafIdx) { |
| 218 | + String v = e.getValue(); |
| 219 | + if (curr.isJsonObject()) { |
| 220 | + curr.getAsJsonObject().addProperty(leaf, v); |
| 221 | + } else if (curr.isJsonArray()) { |
| 222 | + JsonArray a = curr.getAsJsonArray(); |
| 223 | + JsonElement toAdd; |
| 224 | + if (idx != currIdx) { |
| 225 | + JsonObject tmp = new JsonObject(); |
| 226 | + tmp.addProperty(leaf, v); |
| 227 | + toAdd = tmp; |
| 228 | + } else { |
| 229 | + toAdd = v == null ? JsonNull.INSTANCE : new JsonPrimitive(v); |
| 230 | + } |
| 231 | + |
| 232 | + if (a.size() == currIdx) { |
| 233 | + a.add(toAdd); |
| 234 | + } else { |
| 235 | + List<JsonElement> l = a.asList(); |
| 236 | + l.add(currIdx, toAdd); |
| 237 | + // the add above will push all values after it down an index, we instead want to |
| 238 | + // replace it. Remove the next index so we have the same overall size of array. |
| 239 | + l.remove(currIdx + 1); |
| 240 | + } |
| 241 | + } |
| 242 | + } |
| 243 | + } |
| 244 | + } |
| 245 | + return o; |
| 246 | + } |
| 247 | + |
| 248 | + private static Map<String, String> flatten(String k, JsonElement e) { |
| 249 | + HashMap<String, String> ret = new HashMap<>(); |
| 250 | + if (e.isJsonObject()) { |
| 251 | + JsonObject o = e.getAsJsonObject(); |
| 252 | + for (Entry<String, JsonElement> oe : o.asMap().entrySet()) { |
| 253 | + String prefix = k + "." + oe.getKey(); |
| 254 | + ret.putAll(flatten(prefix, oe.getValue())); |
| 255 | + } |
| 256 | + } else if (e.isJsonArray()) { |
| 257 | + List<JsonElement> asList = e.getAsJsonArray().asList(); |
| 258 | + for (int i = 0, asListSize = asList.size(); i < asListSize; i++) { |
| 259 | + JsonElement ee = asList.get(i); |
| 260 | + ret.putAll(flatten(k + "[" + i + "]", ee)); |
| 261 | + } |
| 262 | + } else if (e.isJsonNull()) { |
| 263 | + ret.put(k, null); |
| 264 | + } else { |
| 265 | + ret.put(k, e.getAsString()); |
| 266 | + } |
| 267 | + return ret; |
| 268 | + } |
| 269 | +} |
0 commit comments