001package com.astrolabsoftware.FinkBrowser.Januser; 002 003import com.Lomikel.Utils.Pair; 004import com.Lomikel.Utils.LomikelException; 005import com.Lomikel.Januser.GremlinRecipies; 006import com.Lomikel.Januser.ModifyingGremlinClient; 007import com.astrolabsoftware.FinkBrowser.HBaser.FinkHBaseClient; 008 009// Tinker Pop 010import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; 011import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; 012import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.outV; 013import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.V; 014import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.fold; 015import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.has; 016import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.not; 017import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.unfold; 018import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.out; 019import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.repeat; 020import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.values; 021import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.count; 022import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.addV; 023import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.label; 024import static org.apache.tinkerpop.gremlin.process.traversal.P.within; 025import org.apache.tinkerpop.gremlin.structure.Edge; 026import org.apache.tinkerpop.gremlin.structure.Vertex; 027import org.apache.tinkerpop.gremlin.structure.Direction; 028 029// Java 030import java.lang.Math; 031import java.util.Arrays; 032import java.util.Iterator; 033import java.util.ArrayList; 034import java.util.List; 035import java.util.Map; 036import java.util.HashMap; 037import java.util.SortedSet; 038import java.util.TreeSet; 039import java.util.Set; 040import java.util.HashSet; 041import java.util.Date; 042import java.text.SimpleDateFormat; 043 044// Log4J 045import org.apache.logging.log4j.Logger; 046import org.apache.logging.log4j.LogManager; 047 048/** Fink-specific graph mutation and lifecycle recipes. 049 * 050 * <p>The package documentation defines the graph model, label-mirror 051 * invariant, and transaction ownership shared by this class and the Groovy 052 * traversal recipes.</p> 053 * @opt attributes 054 * @opt operations 055 * @opt types 056 * @opt visibility 057 * @author <a href="mailto:Julius.Hrivnac@cern.ch">J.Hrivnac</a> */ 058// Client-dependent preconditions are checked by the operation that needs them. 059public class FinkGremlinRecipies extends GremlinRecipies { 060 061 /** Create and attach to {@link GraphTraversalSource}. 062 * @param g The attached {@link GraphTraversalSource}. */ 063 public FinkGremlinRecipies(GraphTraversalSource g) { 064 super(g); 065 } 066 067 /** Create and attach to {@link ModifyingGremlinClient}. 068 * @param client The attached {@link ModifyingGremlinClient}. */ 069 public FinkGremlinRecipies(ModifyingGremlinClient client) { 070 super(client); 071 } 072 073 /* 074 * The deprecated processOCol/fillOCol orchestration was intentionally 075 * removed from the compiled API. It mixed source discovery (HBase/Fink 076 * Portal), classification, and correlation generation in one operation. 077 * Supported callers select object IDs explicitly, call classifySources or 078 * classifySource, and invoke generateCorrelations as a separate lifecycle 079 * step. 080 */ 081 082 /** Classify <em>object</em> . 083 * @param classifiers The {@link Classifier}s to be used. 084 * @param oids The {@link Set} of {@code objectId}s of <em>object</em> to be added. 085 * @param hbaseUrl The url of HBase with alerts as {@code ip:port:table:schema}. 086 * @throws LomikelException If anything fails. */ 087 public void classifySources(Classifier[] classifiers, 088 Set<String> oids, 089 String hbaseUrl) throws LomikelException { 090 int size = oids.size(); 091 int n = 0; 092 long dt; 093 double freq; 094 long startTime = System.currentTimeMillis(); 095 // loop over all objects 096 for (String oid : oids) { 097 log.info(oid + " (" + n + " of " + size + "):"); 098 for (Classifier classifier : classifiers) { 099 try { 100 classifySource(classifier, oid, hbaseUrl); 101 } 102 catch (LomikelException e) { 103 log.error("Cannot get classification for " + oid, e); 104 } 105 } 106 n++; 107 dt = (System.currentTimeMillis() - startTime) / 1000; 108 freq = (double)n / (double)dt; 109 log.info("\t\twith " + String.format("%.2f", freq) + " Hz"); 110 } 111 } 112 113 /** Classify <em>object</em>. 114 * @param classifier The {@link Classifier} to be used. 115 * @param objectId The {@code objectId} of <em>object</em> to be added. 116 * @param hbaseUrl The url of HBase with alerts as {@code ip:port:table:schema}. 117 * @throws LomikelException If anything fails. */ 118 public void classifySource(Classifier classifier, 119 String objectId, 120 String hbaseUrl) throws LomikelException { 121 fhclient(hbaseUrl); 122 classifySource(classifier, objectId); 123 } 124 125 /** Classify <em>object</em>. 126 * @param classifier The {@link Classifier} to be used. 127 * @param objectId The {@code objectId} of <em>object</em> to be added. 128 * @throws LomikelException If anything fails. */ 129 public void classifySource(Classifier classifier, 130 String objectId) throws LomikelException { 131 if (!supportsTransactions()) { 132 throw new UnsupportedOperationException("Atomic classification requires rollback-capable transactions"); 133 } 134 if (_classificationTransaction.get()) { 135 throw new IllegalStateException("Nested classification transactions are not supported"); 136 } 137 _classificationTransaction.set(true); 138 try { 139 Iterator<Vertex> objects = g().V().has("lbl", "object"). 140 has("objectId", objectId). 141 limit(1); 142 if (objects.hasNext()) { 143 Vertex object = objects.next(); 144 // Cleanup intentionally follows the indexed lbl property. Registration 145 // additionally requires the native label, so only cache an endpoint 146 // that standalone registration would select as well. 147 if ("object".equals(object.label())) { 148 _classificationObjects.get().put(objectId, object); 149 } 150 List<Edge> edges = g().V(object).inE("deepcontains"). 151 where(outV(). 152 has("lbl", "OCol"). 153 has("survey", classifier.survey()). 154 has("classifier", classifier.name() ). 155 has("flavor", classifier.flavor())). 156 toList(); 157 for (Edge edge : edges) { 158 edge.remove(); 159 } 160 } 161 classifier.classify(this, objectId); 162 commit(); 163 } 164 catch (LomikelException | RuntimeException e) { 165 try { 166 rollback(); 167 } 168 catch (RuntimeException rollbackFailure) { 169 e.addSuppressed(rollbackFailure); 170 } 171 throw e; 172 } 173 finally { 174 _classificationObjects.remove(); 175 _classificationTransaction.remove(); 176 } 177 } 178 179 /** Register <em>object</em> in <em>OCol</em>. 180 * @param classifier The {@link Classifier} to be used. 181 * @param cls The type (class) of <em>OCol</em> {@link Vertex}. 182 * It will be created if not yet exists. 183 * @param objectId The objectId of the new <em>Source</em> {@link Vertex}. 184 * It will be created if not yet exists. 185 * @param weight The weight of the connection. 186 * Usualy the number of <em>Alerts</em> of this type. 187 * @param instancesS The <em>jd</em> of related <em>Alerts</em> as strings separated by comma. 188 * Potential square brackets are removed. 189 * May be {@code null} or empty. 190 * @param weightsS The corresponding per-alert weights, separated by comma. */ 191 public void registerOCol(Classifier classifier, 192 String cls, 193 String objectId, 194 double weight, 195 String instancesS, 196 String weightsS) { 197 List<String> instances = new ArrayList<>(); 198 List<Double> weights = new ArrayList<>(); 199 if (instancesS != null && !instancesS.trim().equals("")) { 200 for (String instance : instancesS.replaceAll("\\[", "").replaceAll("]", "").split(",")) { 201 instances.add(instance); 202 } 203 } 204 if (weightsS != null && !weightsS.trim().equals("")) { 205 for (String weighs : weightsS.replaceAll("\\[", "").replaceAll("]", "").split(",")) { 206 weights.add(Double.valueOf(weighs.trim())); 207 } 208 } 209 registerOCol(classifier, cls, objectId, weight, instances, weights); 210 } 211 212 /** Register <em>object</em> in <em>OCol</em>. Replace possible existing registration. 213 * @param classifier The {@link Classifier} to be used. 214 * @param cls The type (class) of <em>OCol</em> {@link Vertex}. 215 * It will be created if not yet exists. 216 * @param objectId The objectId of the new <em>Source</em> {@link Vertex}. 217 * It will be created if not yet exists. 218 * @param weight The total weight of the connection. 219 * Usualy the number of <em>Alerts</em> of this type. 220 * @param instances The <em>jd</em> of related <em>Alerts</em>. 221 * @param weights The weights of related <em>Alerts</em>. */ 222 public void registerOCol(Classifier classifier, 223 String cls, 224 String objectId, 225 double weight, 226 List<String> instances, 227 List<Double> weights) { 228 validateWeight(weight, "aggregate weight"); 229 if (instances == null || weights == null || instances.size() != weights.size()) { 230 throw new IllegalArgumentException("Instances and weights must have equal lengths"); 231 } 232 for (Double instanceWeight : weights) { 233 if (instanceWeight == null) { 234 throw new IllegalArgumentException("Per-instance weight must not be null"); 235 } 236 validateWeight(instanceWeight, "per-instance weight"); 237 } 238 Map<String, Object> attributes = new HashMap<>(); 239 attributes.put("weight", weight); 240 attributes.put("instances", instances.toString().replaceFirst("\\[", "").replaceAll("]", "")); 241 attributes.put("weights", weights.toString().replaceFirst("\\[", "").replaceAll("]", "")); 242 registerOCol(classifier, cls, objectId, attributes, true); 243 } 244 245 /** Register <em>object</em> in <em>OCol</em>. 246 * @param classifier The {@link Classifier} to be used. 247 * @param cls The type (class) of <em>OCol</em> {@link Vertex}. 248 * It will be created if not yet exists. 249 * @param objectId The objectId of the new <em>Source</em> {@link Vertex}. 250 * It will be created if not yet exists. 251 * @param attributes The additional {@link Edge} attributes. 252 * @param replace Whether to replace existing resistration. */ 253 public synchronized void registerOCol(Classifier classifier, 254 String cls, 255 String objectId, 256 Map<String, ?> attributes, 257 boolean replace) { 258 Object weightValue = attributes == null ? null : attributes.get("weight"); 259 double weight; 260 if (weightValue instanceof Number) { 261 weight = ((Number)weightValue).doubleValue(); 262 } 263 else if (weightValue instanceof String) { 264 try { 265 weight = Double.parseDouble(((String)weightValue).trim()); 266 } 267 catch (NumberFormatException e) { 268 throw new IllegalArgumentException("Registration weight must be numeric", e); 269 } 270 } 271 else { 272 throw new IllegalArgumentException("Registration weight must be numeric"); 273 } 274 validateWeight(weight, "aggregate weight"); 275 Map<String, Object> validatedAttributes = new HashMap<>(); 276 validatedAttributes.putAll(attributes); 277 validatedAttributes.remove("lbl"); 278 validatedAttributes.put("weight", weight); 279 boolean rollbackOnFailure = !_classificationTransaction.get() && supportsTransactions(); 280 try { 281 //log.info("\tregistering " + objectId + " as " + classifier + " / " + cls + " with attributes " + attributes + ", replace = " + replace); 282 log.info("\tregistering " + objectId + " as " + classifier + " / " + cls + " with weight = " + weight + ", replace = " + replace); 283 String importDate = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS").format(new Date()); 284 Vertex ocol = g().V().has("lbl", "OCol" ). 285 has("survey", classifier.survey()). 286 has("classifier", classifier.name() ). 287 has("flavor", classifier.flavor()). 288 has("cls", cls ). 289 filter(label().is("OCol")). 290 fold(). 291 coalesce(unfold(), 292 addV("OCol"). 293 property("lbl", "OCol"). 294 property("survey", classifier.survey()). 295 property("classifier", classifier.name() ). 296 property("flavor", classifier.flavor()). 297 property("cls", cls )). 298 next(); 299 boolean classificationTransaction = _classificationTransaction.get(); 300 Vertex s = classificationTransaction ? _classificationObjects.get().get(objectId) : null; 301 boolean cacheObject = false; 302 if (s == null) { 303 s = g().V().has("lbl", "object"). 304 has("objectId", objectId). 305 filter(label().is("object")). 306 fold(). 307 coalesce(unfold(), 308 addV("object"). 309 property("lbl", "object"). 310 property("objectId", objectId)). 311 next(); 312 cacheObject = classificationTransaction; 313 } 314 s.property("importDate", importDate); 315 if (cacheObject) { 316 _classificationObjects.get().put(objectId, s); 317 } 318 if (replace) { 319 replaceRegistrationEdge(ocol, s, validatedAttributes); 320 } 321 else { 322 createRegistrationEdge(ocol, s, validatedAttributes); 323 } 324 if (!_classificationTransaction.get()) { 325 commit(); 326 } 327 } 328 catch (RuntimeException e) { 329 if (rollbackOnFailure) { 330 try { 331 rollback(); 332 } 333 catch (RuntimeException rollbackFailure) { 334 e.addSuppressed(rollbackFailure); 335 } 336 } 337 throw e; 338 } 339 } 340 341 /** Replace all parallel registrations with one fully prepared edge. */ 342 private void replaceRegistrationEdge(Vertex ocol, 343 Vertex object, 344 Map<String, Object> attributes) { 345 List<Edge> existingEdges = getEdge(ocol, object, "deepcontains"); 346 if (supportsTransactions()) { 347 for (Edge edge : existingEdges) { 348 edge.remove(); 349 } 350 createRegistrationEdge(ocol, object, attributes); 351 return; 352 } 353 354 Edge replacement = createRegistrationEdge(ocol, object, attributes); 355 try { 356 for (Edge edge : existingEdges) { 357 edge.remove(); 358 } 359 } 360 catch (RuntimeException failure) { 361 try { 362 replacement.remove(); 363 } 364 catch (RuntimeException cleanupFailure) { 365 failure.addSuppressed(cleanupFailure); 366 } 367 throw failure; 368 } 369 } 370 371 /** Create a complete registration edge or remove its partial state on failure. */ 372 private Edge createRegistrationEdge(Vertex ocol, 373 Vertex object, 374 Map<String, Object> attributes) { 375 Edge edge = ocol.addEdge("deepcontains", object); 376 try { 377 edge.property("lbl", "deepcontains"); 378 for (Map.Entry<String, Object> attribute : attributes.entrySet()) { 379 edge.property(attribute.getKey(), attribute.getValue()); 380 } 381 return edge; 382 } 383 catch (RuntimeException failure) { 384 try { 385 edge.remove(); 386 } 387 catch (RuntimeException cleanupFailure) { 388 failure.addSuppressed(cleanupFailure); 389 } 390 throw failure; 391 } 392 } 393 394 /** Validate scientific registration weights before graph mutation. */ 395 private static void validateWeight(double weight, String description) { 396 if (!Double.isFinite(weight) || weight < 0.0) { 397 throw new IllegalArgumentException("Invalid " + description + ": " + weight); 398 } 399 } 400 401 /** Clean tree under <em>OCol</em>. 402 * Drop alerts. Alerts are dropped even if they have other 403 * {@link Edge}s. 404 * @param classifier The {@link Classifier} to be used. 405 * All its <em>flavors</em> are handled. 406 * @param cls The type (class) of <em>OCol</em>. 407 * @throws LomikelException If anything goes wrong. */ 408 public void cleanOCol(Classifier classifier, 409 String cls) throws LomikelException { 410 log.info("Cleaning " + cls + " OCol"); 411 g().V().has("lbl", "OCol"). 412 has("survey", classifier.survey()). 413 has("classifier", classifier.name() ). 414 has("flavor", classifier.flavor()). 415 has("cls", cls ). 416 out("deepcontains"). 417 out(). 418 drop(). 419 iterate(); 420 commit(); 421 } 422 423 /** Generate <em>overlaps</em> Edges between <em>OCol</em>. 424 * Possibly between two {@link Classifier}s. 425 * @param classifiers The {@link Classifier}s to be used. */ 426 public void generateCorrelations(Classifier... classifiers) { 427 if (!supportsTransactions()) { 428 throw new UnsupportedOperationException("Atomic correlation regeneration requires rollback-capable transactions"); 429 } 430 try { 431 generateCorrelationsInTransaction(classifiers); 432 } 433 catch (RuntimeException e) { 434 try { 435 rollback(); 436 } 437 catch (RuntimeException rollbackFailure) { 438 e.addSuppressed(rollbackFailure); 439 } 440 throw e; 441 } 442 } 443 444 /** Generate correlations inside one caller-owned transaction. */ 445 private void generateCorrelationsInTransaction(Classifier... classifiers) { 446 log.info("Generating correlations for OCol of " + Arrays.asList(classifiers)); 447 // Clean all correlations 448 g().E().has("lbl", "overlaps"). 449 drop(). 450 iterate(); 451 Set<String> classifierScopes = new HashSet<>(); 452 List<Vertex> scopedOCols = new ArrayList<>(); 453 List<Vertex> malformedScopedOCols = new ArrayList<>(); 454 for (Classifier classifier : classifiers) { 455 classifierScopes.add(correlationScope(classifier.survey(), classifier.name(), classifier.flavor())); 456 scopedOCols.addAll(g().V().has("lbl", "OCol" ). 457 has("survey", classifier.survey()). 458 has("classifier", classifier.name() ). 459 has("flavor", classifier.flavor()). 460 has("cls"). 461 toList()); 462 malformedScopedOCols.addAll(g().V().has("lbl", "OCol" ). 463 has("survey", classifier.survey()). 464 has("classifier", classifier.name() ). 465 has("flavor", classifier.flavor()). 466 not(has("cls")). 467 toList()); 468 } 469 // Accumulate correlations and sizes 470 Map<OCol, Double> weights0 = new HashMap<>(); // cls -> weight (for one objext) 471 Map<Pair<OCol, OCol>, Double> corrS = new HashMap<>(); // [cls1, cls2] -> weight (for all object between OCol-OCol) 472 Map<OCol, Double> sizeS = new HashMap<>(); // cls -> total (for all objects of OCol) 473 SortedSet<OCol> types0 = new TreeSet<>(); // [cls] (for one object) 474 SortedSet<OCol> types = new TreeSet<>(); // [cls] (for all objects) 475 Vertex object; 476 Iterator<Edge> deepcontainsIt; 477 Edge deepcontains; 478 Edge overlaps; 479 double weight; 480 double weight1; 481 double weight2; 482 double cor; 483 Vertex ocol; 484 Vertex ocol1; 485 Vertex ocol2; 486 OCol cls; 487 Pair<OCol, OCol> rel; 488 Map<OCol, Vertex> uniqueOCols = new HashMap<>(); 489 Set<OCol> duplicateOCols = new HashSet<>(); 490 for (Vertex scopedOCol : scopedOCols) { 491 cls = new OCol(scopedOCol); 492 if (uniqueOCols.putIfAbsent(cls, scopedOCol) != null) { 493 duplicateOCols.add(cls); 494 } 495 } 496 // Loop over objets and accumulated weights to each object 497 GraphTraversal<Vertex, Vertex> objectT = g().V().has("lbl", "object"); 498 while (objectT.hasNext()) { 499 weights0.clear(); 500 types0.clear(); 501 object = objectT.next(); 502 deepcontainsIt = object.edges(Direction.IN, "deepcontains"); 503 // Get weights only from OCols in the requested classifier scope. 504 while (deepcontainsIt.hasNext()) { 505 deepcontains = deepcontainsIt.next(); 506 ocol1 = deepcontains.outVertex(); 507 if (!ocol1.property("lbl").isPresent() || 508 !"OCol".equals(ocol1.property("lbl").value()) || 509 !ocol1.property("survey").isPresent() || 510 !ocol1.property("classifier").isPresent() || 511 !ocol1.property("flavor").isPresent() || 512 !ocol1.property("cls").isPresent()) { 513 continue; 514 } 515 if (!classifierScopes.contains(correlationScope(ocol1.property("survey").value().toString(), 516 ocol1.property("classifier").value().toString(), 517 ocol1.property("flavor").value().toString()))) { 518 continue; 519 } 520 if (!deepcontains.property("weight").isPresent()) { 521 throw new IllegalArgumentException("Scoped deepcontains edge has no weight"); 522 } 523 try { 524 weight = Double.parseDouble(deepcontains.property("weight").value().toString()); 525 } 526 catch (NumberFormatException e) { 527 throw new IllegalArgumentException("Scoped deepcontains edge has a non-numeric weight", e); 528 } 529 if (!Double.isFinite(weight) || weight < 0.0) { 530 throw new IllegalArgumentException("Scoped deepcontains edge has an invalid weight: " + weight); 531 } 532 cls = new OCol(ocol1); 533 types0.add(cls); 534 types.add(cls); 535 weights0.put(cls, weight); 536 } 537 // Double loop over accumulated weights and fill weights between OCols 538 for (OCol cls1 : types0) { 539 weight1 = weights0.get(cls1); 540 for (OCol cls2 : types0) { 541 weight2 = weights0.get(cls2); 542 rel = Pair.of(cls1, cls2); 543 // OCol-OCol 544 if (!corrS.containsKey(rel)) { 545 corrS.put(rel, 0.0); 546 } 547 cor = corrS.get(rel); 548 //corrS.put(rel, cor + 1.0); 549 corrS.put(rel, cor + Math.sqrt(weight1 * weight2)); 550 } 551 } 552 } 553 // Fill total sizes 554 double sizeS0; 555 for (OCol cls1 : types) { 556 sizeS0 = 0.0; 557 for (OCol cls2 : types) { 558 if (corrS.containsKey(Pair.of(cls1, cls2))) { 559 sizeS0 += corrS.get(Pair.of(cls1, cls2)); 560 } 561 } 562 sizeS.put(cls1, sizeS0); 563 } 564 // Mutate only after all scoped correlation inputs have been validated. 565 for (Vertex malformedScopedOCol : malformedScopedOCols) { 566 g().V(malformedScopedOCol).drop().iterate(); 567 } 568 // Preserve overlaps crossing the requested-scope boundary; those require 569 // both classifier scopes to be regenerated together. 570 Set<Object> scopedIds = new HashSet<>(); 571 for (Vertex scopedOCol : scopedOCols) { 572 scopedIds.add(scopedOCol.id()); 573 } 574 Set<Object> internalOverlapIds = new HashSet<>(); 575 for (Vertex scopedOCol : scopedOCols) { 576 Iterator<Edge> overlapsIt = scopedOCol.edges(Direction.BOTH, "overlaps"); 577 while (overlapsIt.hasNext()) { 578 Edge oldOverlap = overlapsIt.next(); 579 if (scopedIds.contains(oldOverlap.outVertex().id()) && 580 scopedIds.contains(oldOverlap.inVertex().id())) { 581 internalOverlapIds.add(oldOverlap.id()); 582 } 583 } 584 } 585 if (!internalOverlapIds.isEmpty()) { 586 g().E(internalOverlapIds.toArray()).drop().iterate(); 587 } 588 // Create overlaps 589 int ns = 0; 590 // Loop over OCol-OCol and create overlaps Edge OCol-OCol if non empty 591 OCol cls1; 592 OCol cls2; 593 for (Map.Entry<Pair<OCol, OCol>, Double> entry : corrS.entrySet()) { 594 rel = entry.getKey(); 595 cls1 = rel.first(); 596 cls2 = rel.second(); 597 weight = entry.getValue(); 598 ocol1 = duplicateOCols.contains(cls1) ? findOCol(cls1) : uniqueOCols.get(cls1); 599 ocol2 = duplicateOCols.contains(cls2) ? findOCol(cls2) : uniqueOCols.get(cls2); 600 overlaps = ocol1.addEdge("overlaps", ocol2); 601 overlaps.property("lbl", "overlaps"); 602 overlaps.property("intersection", weight); 603 overlaps.property("sizeIn", sizeS.get(cls1)); 604 overlaps.property("sizeOut", sizeS.get(cls2)); 605 ns++; 606 } 607 commit(); 608 log.info("" + ns + " object-object correlations generated"); 609 } 610 611 /** Find the endpoint selected by the existing indexed OCol lookup. 612 * @param cls The logical OCol identity. 613 * @return The first matching physical OCol endpoint. */ 614 protected Vertex findOCol(OCol cls) { 615 return g().V().has("lbl", "OCol" ). 616 has("survey", cls.survey() ). 617 has("classifier", cls.classifier()). 618 has("flavor", cls.flavor() ). 619 has("cls", cls.cls() ). 620 next(); 621 } 622 623 /** Create a new {@link FinkHBaseClient}. Singleton when url unchanged. 624 * @param hbaseUrl The HBase url as {@code ip:port:table[:schema]}. 625 * @return The corresponding {@link FinkHBaseClient}, created and initialised if needed. 626 * @throws LomikelException If cannot be created. */ 627 public synchronized FinkHBaseClient fhclient(String hbaseUrl) throws LomikelException { 628 if (hbaseUrl == null) { 629 return _fhclient; 630 } 631 if (hbaseUrl.equals(_fhclientUrl) && _fhclient != null) { 632 return _fhclient; 633 } 634 String[] url = hbaseUrl.split(":", -1); 635 if ((url.length != 3 && url.length != 4) || 636 url[0].isEmpty() || url[1].isEmpty() || url[2].isEmpty()) { 637 throw new LomikelException("Invalid HBase URL (expected ip:port:table[:schema]): " + hbaseUrl); 638 } 639 String ip = url[0]; 640 String port = url[1]; 641 String table = url[2]; 642 String schema = url.length == 4 ? url[3] : ""; 643 FinkHBaseClient candidate = null; 644 try { 645 candidate = new FinkHBaseClient(ip, port); 646 candidate.connect(table, schema); 647 } 648 catch (RuntimeException | LomikelException failure) { 649 if (candidate != null) { 650 candidate.close(); 651 } 652 if (failure instanceof LomikelException) { 653 throw (LomikelException)failure; 654 } 655 throw new LomikelException("Cannot initialise FinkHBaseClient for " + hbaseUrl, failure); 656 } 657 FinkHBaseClient previous = _fhclient; 658 _fhclient = candidate; 659 _fhclientUrl = hbaseUrl; 660 if (previous != null) { 661 previous.close(); 662 } 663 return _fhclient; 664 } 665 666 /** Get existing {@link FinkHBaseClient}. 667 * @return The corresponding {@link FinkHBaseClient}. 668 * @throws LomikelException If not yet created. */ 669 public FinkHBaseClient fhclient() throws LomikelException { 670 if (_fhclient == null) { 671 throw new LomikelException("FinkHBaseClient not initialised"); 672 } 673 return _fhclient; 674 } 675 676 /** Give HBase url. 677 * @return The HBase url as {@code ip:port:table[:schema]}. */ 678 public String hbaseUrl() { 679 return _fhclientUrl; 680 } 681 682 /** Build an unambiguous identity for a correlation-generation scope. */ 683 private static String correlationScope(String survey, 684 String classifier, 685 String flavor) { 686 return survey + "\u0000" + classifier + "\u0000" + flavor; 687 } 688 689 private FinkHBaseClient _fhclient; 690 691 private String _fhclientUrl; 692 693 /** Whether this thread's registration participates in a classification transaction. */ 694 private ThreadLocal<Boolean> _classificationTransaction = ThreadLocal.withInitial(() -> false); 695 696 /** Objects already selected or created by this thread's classification transaction. */ 697 private ThreadLocal<Map<String, Vertex>> _classificationObjects = ThreadLocal.withInitial(HashMap::new); 698 699 700 private static String FINK_OBJECTS_WS = "https://api.ztf.fink-portal.org/api/v1/objects"; 701 private static String FINK_LATESTS_WS = "https://api.ztf.fink-portal.org/api/v1/latests"; 702 private static String FINK_ANOMALY_WS = "https://api.ztf.fink-portal.org/api/v1/anomaly"; 703 704 /** Logging . */ 705 private static Logger log = LogManager.getLogger(FinkGremlinRecipies.class); 706 707 }