001package com.astrolabsoftware.FinkBrowser.Januser;
002
003import com.Lomikel.Utils.Pair;
004import com.Lomikel.Utils.LomikelException;
005import com.Lomikel.Januser.GremlinRecipies;
006import com.Lomikel.Januser.ModifyingGremlinClient;
007import com.astrolabsoftware.FinkBrowser.HBaser.FinkHBaseClient;
008
009// Tinker Pop
010import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
011import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
012import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.outV;
013import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.V;
014import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.fold;
015import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.has;
016import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.not;
017import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.unfold;
018import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.out;
019import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.repeat;
020import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.values;
021import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.count;
022import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.addV;
023import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.label;
024import static org.apache.tinkerpop.gremlin.process.traversal.P.within;
025import org.apache.tinkerpop.gremlin.structure.Edge;
026import org.apache.tinkerpop.gremlin.structure.Vertex;
027import org.apache.tinkerpop.gremlin.structure.Direction;
028
029// Java
030import java.lang.Math;
031import java.util.Arrays;
032import java.util.Iterator;
033import java.util.ArrayList;
034import java.util.List;
035import java.util.Map;
036import java.util.HashMap;
037import java.util.SortedSet;
038import java.util.TreeSet;
039import java.util.Set;
040import java.util.HashSet;
041import java.util.Date;
042import java.text.SimpleDateFormat;
043
044// Log4J
045import org.apache.logging.log4j.Logger;
046import org.apache.logging.log4j.LogManager;
047
048/** Fink-specific graph mutation and lifecycle recipes.
049  *
050  * <p>The package documentation defines the graph model, label-mirror
051  * invariant, and transaction ownership shared by this class and the Groovy
052  * traversal recipes.</p>
053  * @opt attributes
054  * @opt operations
055  * @opt types
056  * @opt visibility
057  * @author <a href="mailto:Julius.Hrivnac@cern.ch">J.Hrivnac</a> */
058// Client-dependent preconditions are checked by the operation that needs them.
059public class FinkGremlinRecipies extends GremlinRecipies {
060    
061  /** Create and attach to {@link GraphTraversalSource}.
062    * @param g The attached {@link GraphTraversalSource}. */
063  public FinkGremlinRecipies(GraphTraversalSource g) {
064    super(g);
065    }
066    
067  /** Create and attach to {@link ModifyingGremlinClient}.
068    * @param client The attached  {@link ModifyingGremlinClient}. */
069  public FinkGremlinRecipies(ModifyingGremlinClient client) {
070    super(client);
071    }
072    
073  /*
074   * The deprecated processOCol/fillOCol orchestration was intentionally
075   * removed from the compiled API. It mixed source discovery (HBase/Fink
076   * Portal), classification, and correlation generation in one operation.
077   * Supported callers select object IDs explicitly, call classifySources or
078   * classifySource, and invoke generateCorrelations as a separate lifecycle
079   * step.
080   */
081    
082  /** Classify <em>object</em> .
083    * @param classifiers The {@link Classifier}s to be used.
084    * @param oids        The {@link Set} of {@code objectId}s of <em>object</em> to be added.
085    * @param hbaseUrl    The url of HBase with alerts as {@code ip:port:table:schema}.
086    * @throws LomikelException If anything fails. */
087  public void classifySources(Classifier[] classifiers,
088                              Set<String>  oids,
089                              String       hbaseUrl) throws LomikelException {
090    int size = oids.size();
091    int n = 0;
092    long dt;
093    double freq;
094    long startTime = System.currentTimeMillis();
095    // loop over all objects
096    for (String oid : oids) {
097      log.info(oid + " (" + n + " of " + size + "):");
098      for (Classifier classifier : classifiers) {
099        try {
100          classifySource(classifier, oid, hbaseUrl);
101          }
102        catch (LomikelException e) {
103          log.error("Cannot get classification for " + oid, e);
104          }
105        }
106      n++;
107      dt = (System.currentTimeMillis() - startTime) / 1000;
108      freq = (double)n / (double)dt;
109      log.info("\t\twith " + String.format("%.2f", freq) + " Hz");
110      }
111    }
112    
113   /** Classify <em>object</em>.
114    * @param classifier The {@link Classifier} to be used.
115    * @param objectId   The {@code objectId} of <em>object</em> to be added.
116    * @param hbaseUrl   The url of HBase with alerts as {@code ip:port:table:schema}.
117    * @throws LomikelException If anything fails. */
118  public void classifySource(Classifier classifier,
119                             String     objectId,
120                             String     hbaseUrl) throws LomikelException {  
121    fhclient(hbaseUrl);
122    classifySource(classifier, objectId);
123    }
124   
125  /** Classify <em>object</em>.
126    * @param classifier The {@link Classifier} to be used.
127    * @param objectId   The {@code objectId} of <em>object</em> to be added.
128    * @throws LomikelException If anything fails. */
129  public void classifySource(Classifier classifier,
130                             String     objectId) throws LomikelException {
131    if (!supportsTransactions()) {
132      throw new UnsupportedOperationException("Atomic classification requires rollback-capable transactions");
133      }
134    if (_classificationTransaction.get()) {
135      throw new IllegalStateException("Nested classification transactions are not supported");
136      }
137    _classificationTransaction.set(true);
138    try {
139      Iterator<Vertex> objects = g().V().has("lbl", "object").
140                                        has("objectId", objectId).
141                                        limit(1);
142      if (objects.hasNext()) {
143        Vertex object = objects.next();
144        // Cleanup intentionally follows the indexed lbl property. Registration
145        // additionally requires the native label, so only cache an endpoint
146        // that standalone registration would select as well.
147        if ("object".equals(object.label())) {
148          _classificationObjects.get().put(objectId, object);
149          }
150        List<Edge> edges = g().V(object).inE("deepcontains").
151                              where(outV().
152                                has("lbl",        "OCol").
153                                has("survey",     classifier.survey()).
154                                has("classifier", classifier.name()  ).
155                                has("flavor",     classifier.flavor())).
156                              toList();
157        for (Edge edge : edges) {
158          edge.remove();
159          }
160        }
161      classifier.classify(this, objectId);
162      commit();
163      }
164    catch (LomikelException | RuntimeException e) {
165      try {
166        rollback();
167        }
168      catch (RuntimeException rollbackFailure) {
169        e.addSuppressed(rollbackFailure);
170        }
171      throw e;
172      }
173    finally {
174      _classificationObjects.remove();
175      _classificationTransaction.remove();
176      }
177    }
178       
179  /** Register  <em>object</em> in <em>OCol</em>.
180    * @param classifier The {@link Classifier} to be used.
181    * @param cls        The type (class) of <em>OCol</em> {@link Vertex}.
182    *                   It will be created if not yet exists.
183    * @param objectId   The objectId of the new <em>Source</em> {@link Vertex}.
184    *                   It will be created if not yet exists.
185    * @param weight     The weight of the connection.
186    *                   Usualy the number of <em>Alerts</em> of this type. 
187    * @param instancesS The <em>jd</em> of related <em>Alerts</em> as strings separated by comma.
188    *                   Potential square brackets are removed.
189    *                   May be {@code null} or empty.
190    * @param weightsS   The corresponding per-alert weights, separated by comma. */
191  public void registerOCol(Classifier classifier,
192                           String     cls,
193                           String     objectId,
194                           double     weight,
195                           String     instancesS,
196                           String     weightsS) {   
197    List<String> instances = new ArrayList<>();
198    List<Double> weights   = new ArrayList<>();
199    if (instancesS != null && !instancesS.trim().equals("")) {
200      for (String instance : instancesS.replaceAll("\\[", "").replaceAll("]", "").split(",")) {
201        instances.add(instance);
202        }
203      }
204    if (weightsS != null && !weightsS.trim().equals("")) {
205      for (String weighs : weightsS.replaceAll("\\[", "").replaceAll("]", "").split(",")) {
206        weights.add(Double.valueOf(weighs.trim()));
207        }
208      }
209    registerOCol(classifier, cls, objectId, weight, instances, weights);
210    }
211    
212  /** Register <em>object</em> in <em>OCol</em>. Replace possible existing registration.
213    * @param classifier The {@link Classifier} to be used.
214    * @param cls        The type (class) of <em>OCol</em> {@link Vertex}.
215    *                   It will be created if not yet exists.
216    * @param objectId   The objectId of the new <em>Source</em> {@link Vertex}.
217    *                   It will be created if not yet exists.
218    * @param weight     The total weight of the connection.
219    *                   Usualy the number of <em>Alerts</em> of this type. 
220    * @param instances  The <em>jd</em> of related <em>Alerts</em>.
221    * @param weights    The weights of related <em>Alerts</em>. */
222  public void registerOCol(Classifier   classifier,
223                           String       cls,
224                           String       objectId,
225                           double       weight,
226                           List<String> instances,
227                           List<Double> weights) { 
228    validateWeight(weight, "aggregate weight");
229    if (instances == null || weights == null || instances.size() != weights.size()) {
230      throw new IllegalArgumentException("Instances and weights must have equal lengths");
231      }
232    for (Double instanceWeight : weights) {
233      if (instanceWeight == null) {
234        throw new IllegalArgumentException("Per-instance weight must not be null");
235        }
236      validateWeight(instanceWeight, "per-instance weight");
237      }
238    Map<String, Object> attributes = new HashMap<>();
239    attributes.put("weight",    weight);
240    attributes.put("instances", instances.toString().replaceFirst("\\[", "").replaceAll("]", ""));
241    attributes.put("weights",   weights.toString().replaceFirst("\\[", "").replaceAll("]", ""));
242    registerOCol(classifier, cls, objectId, attributes, true);
243    }
244    
245  /** Register <em>object</em> in <em>OCol</em>.
246    * @param classifier The {@link Classifier} to be used.
247    * @param cls        The type (class) of <em>OCol</em> {@link Vertex}.
248    *                   It will be created if not yet exists.
249    * @param objectId   The objectId of the new <em>Source</em> {@link Vertex}.
250    *                   It will be created if not yet exists.
251    * @param attributes The additional {@link Edge} attributes.    
252    * @param replace    Whether to replace existing resistration. */
253  public synchronized void registerOCol(Classifier          classifier,
254                                        String              cls,
255                                        String              objectId,
256                                        Map<String, ?> attributes,
257                                        boolean             replace) {
258    Object weightValue = attributes == null ? null : attributes.get("weight");
259    double weight;
260    if (weightValue instanceof Number) {
261      weight = ((Number)weightValue).doubleValue();
262      }
263    else if (weightValue instanceof String) {
264      try {
265        weight = Double.parseDouble(((String)weightValue).trim());
266        }
267      catch (NumberFormatException e) {
268        throw new IllegalArgumentException("Registration weight must be numeric", e);
269        }
270      }
271    else {
272      throw new IllegalArgumentException("Registration weight must be numeric");
273      }
274    validateWeight(weight, "aggregate weight");
275    Map<String, Object> validatedAttributes = new HashMap<>();
276    validatedAttributes.putAll(attributes);
277    validatedAttributes.remove("lbl");
278    validatedAttributes.put("weight", weight);
279    boolean rollbackOnFailure = !_classificationTransaction.get() && supportsTransactions();
280    try {
281    //log.info("\tregistering " + objectId + " as " + classifier + " / " + cls + " with attributes " + attributes + ", replace = " + replace);
282    log.info("\tregistering " + objectId + " as " + classifier + " / " + cls + " with weight = " + weight + ", replace = " + replace);
283    String importDate = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS").format(new Date());
284    Vertex ocol = g().V().has("lbl",        "OCol"             ).
285                          has("survey",     classifier.survey()).
286                          has("classifier", classifier.name()  ).
287                          has("flavor",     classifier.flavor()).
288                          has("cls",        cls                ).
289                          filter(label().is("OCol")).
290                          fold().
291                          coalesce(unfold(), 
292                                  addV("OCol").
293                                  property("lbl",        "OCol").
294                                  property("survey",     classifier.survey()).
295                                  property("classifier", classifier.name()  ).
296                                  property("flavor",     classifier.flavor()).
297                                  property("cls",        cls                )).
298                         next();
299    boolean classificationTransaction = _classificationTransaction.get();
300    Vertex s = classificationTransaction ? _classificationObjects.get().get(objectId) : null;
301    boolean cacheObject = false;
302    if (s == null) {
303      s = g().V().has("lbl",      "object").
304                  has("objectId", objectId).
305                  filter(label().is("object")).
306                  fold().
307                  coalesce(unfold(),
308                           addV("object").
309                           property("lbl",      "object").
310                           property("objectId", objectId)).
311                  next();
312      cacheObject = classificationTransaction;
313      }
314    s.property("importDate", importDate);
315    if (cacheObject) {
316      _classificationObjects.get().put(objectId, s);
317      }
318    if (replace) {
319      replaceRegistrationEdge(ocol, s, validatedAttributes);
320      }
321    else {
322      createRegistrationEdge(ocol, s, validatedAttributes);
323      }
324    if (!_classificationTransaction.get()) {
325      commit();
326      }
327      }
328    catch (RuntimeException e) {
329      if (rollbackOnFailure) {
330        try {
331          rollback();
332          }
333        catch (RuntimeException rollbackFailure) {
334          e.addSuppressed(rollbackFailure);
335          }
336        }
337      throw e;
338      }
339    }
340
341  /** Replace all parallel registrations with one fully prepared edge. */
342  private void replaceRegistrationEdge(Vertex              ocol,
343                                       Vertex              object,
344                                       Map<String, Object> attributes) {
345    List<Edge> existingEdges = getEdge(ocol, object, "deepcontains");
346    if (supportsTransactions()) {
347      for (Edge edge : existingEdges) {
348        edge.remove();
349        }
350      createRegistrationEdge(ocol, object, attributes);
351      return;
352      }
353
354    Edge replacement = createRegistrationEdge(ocol, object, attributes);
355    try {
356      for (Edge edge : existingEdges) {
357        edge.remove();
358        }
359      }
360    catch (RuntimeException failure) {
361      try {
362        replacement.remove();
363        }
364      catch (RuntimeException cleanupFailure) {
365        failure.addSuppressed(cleanupFailure);
366        }
367      throw failure;
368      }
369    }
370
371  /** Create a complete registration edge or remove its partial state on failure. */
372  private Edge createRegistrationEdge(Vertex              ocol,
373                                      Vertex              object,
374                                      Map<String, Object> attributes) {
375    Edge edge = ocol.addEdge("deepcontains", object);
376    try {
377      edge.property("lbl", "deepcontains");
378      for (Map.Entry<String, Object> attribute : attributes.entrySet()) {
379        edge.property(attribute.getKey(), attribute.getValue());
380        }
381      return edge;
382      }
383    catch (RuntimeException failure) {
384      try {
385        edge.remove();
386        }
387      catch (RuntimeException cleanupFailure) {
388        failure.addSuppressed(cleanupFailure);
389        }
390      throw failure;
391      }
392    }
393
394  /** Validate scientific registration weights before graph mutation. */
395  private static void validateWeight(double weight, String description) {
396    if (!Double.isFinite(weight) || weight < 0.0) {
397      throw new IllegalArgumentException("Invalid " + description + ": " + weight);
398      }
399    }
400   
401  /** Clean tree under <em>OCol</em>.
402    * Drop alerts. Alerts are dropped even if they have other
403    * {@link Edge}s.
404    * @param classifier The {@link Classifier} to be used.
405    *                   All its <em>flavors</em> are handled.
406    * @param cls        The type (class) of <em>OCol</em>.
407    * @throws LomikelException If anything goes wrong. */
408  public void cleanOCol(Classifier classifier,
409                        String     cls) throws LomikelException {
410    log.info("Cleaning " + cls + " OCol");
411    g().V().has("lbl",        "OCol").
412            has("survey",     classifier.survey()).
413            has("classifier", classifier.name()  ).
414            has("flavor",     classifier.flavor()).
415            has("cls",        cls                ).
416            out("deepcontains").
417            out().
418            drop().
419            iterate();
420    commit();
421    }
422        
423  /** Generate <em>overlaps</em> Edges between <em>OCol</em>.
424    * Possibly between two {@link Classifier}s.
425    * @param classifiers The {@link Classifier}s to be used. */
426  public void generateCorrelations(Classifier... classifiers) {
427    if (!supportsTransactions()) {
428      throw new UnsupportedOperationException("Atomic correlation regeneration requires rollback-capable transactions");
429      }
430    try {
431      generateCorrelationsInTransaction(classifiers);
432      }
433    catch (RuntimeException e) {
434      try {
435        rollback();
436        }
437      catch (RuntimeException rollbackFailure) {
438        e.addSuppressed(rollbackFailure);
439        }
440      throw e;
441      }
442    }
443
444  /** Generate correlations inside one caller-owned transaction. */
445  private void generateCorrelationsInTransaction(Classifier... classifiers) {
446    log.info("Generating correlations for OCol of " + Arrays.asList(classifiers));
447    // Clean all correlations 
448    g().E().has("lbl", "overlaps").
449            drop().
450            iterate();
451    Set<String> classifierScopes = new HashSet<>();
452    List<Vertex> scopedOCols = new ArrayList<>();
453    List<Vertex> malformedScopedOCols = new ArrayList<>();
454    for (Classifier classifier : classifiers) {
455      classifierScopes.add(correlationScope(classifier.survey(), classifier.name(), classifier.flavor()));
456      scopedOCols.addAll(g().V().has("lbl",        "OCol"             ).
457                                has("survey",     classifier.survey()).
458                                has("classifier", classifier.name()  ).
459                                has("flavor",     classifier.flavor()).
460                                has("cls").
461                                toList());
462      malformedScopedOCols.addAll(g().V().has("lbl",        "OCol"             ).
463                                          has("survey",     classifier.survey()).
464                                          has("classifier", classifier.name()  ).
465                                          has("flavor",     classifier.flavor()).
466                                          not(has("cls")).
467                                          toList());
468      }
469    // Accumulate correlations and sizes
470    Map<OCol, Double>             weights0 = new HashMap<>(); // cls -> weight (for one objext)
471    Map<Pair<OCol, OCol>, Double> corrS    = new HashMap<>(); // [cls1, cls2] -> weight (for all object between OCol-OCol)
472    Map<OCol, Double>             sizeS    = new HashMap<>(); // cls -> total (for all objects of OCol)
473    SortedSet<OCol>               types0   = new TreeSet<>(); // [cls] (for one object)
474    SortedSet<OCol>               types    = new TreeSet<>(); // [cls] (for all objects)
475    Vertex object;
476    Iterator<Edge> deepcontainsIt;
477    Edge deepcontains;
478    Edge overlaps;
479    double weight;
480    double weight1;
481    double weight2;
482    double cor;
483    Vertex ocol;
484    Vertex ocol1;
485    Vertex ocol2;
486    OCol cls;
487    Pair<OCol, OCol> rel;
488    Map<OCol, Vertex> uniqueOCols = new HashMap<>();
489    Set<OCol> duplicateOCols = new HashSet<>();
490    for (Vertex scopedOCol : scopedOCols) {
491      cls = new OCol(scopedOCol);
492      if (uniqueOCols.putIfAbsent(cls, scopedOCol) != null) {
493        duplicateOCols.add(cls);
494        }
495      }
496    // Loop over objets and accumulated weights to each object
497    GraphTraversal<Vertex, Vertex> objectT = g().V().has("lbl", "object");
498    while (objectT.hasNext()) {
499      weights0.clear();
500      types0.clear();
501      object = objectT.next();
502      deepcontainsIt = object.edges(Direction.IN, "deepcontains");
503      // Get weights only from OCols in the requested classifier scope.
504      while (deepcontainsIt.hasNext()) {
505        deepcontains = deepcontainsIt.next();
506        ocol1 = deepcontains.outVertex();
507        if (!ocol1.property("lbl").isPresent() ||
508            !"OCol".equals(ocol1.property("lbl").value()) ||
509            !ocol1.property("survey").isPresent() ||
510            !ocol1.property("classifier").isPresent() ||
511            !ocol1.property("flavor").isPresent() ||
512            !ocol1.property("cls").isPresent()) {
513          continue;
514          }
515        if (!classifierScopes.contains(correlationScope(ocol1.property("survey").value().toString(),
516                                                         ocol1.property("classifier").value().toString(),
517                                                         ocol1.property("flavor").value().toString()))) {
518          continue;
519          }
520        if (!deepcontains.property("weight").isPresent()) {
521          throw new IllegalArgumentException("Scoped deepcontains edge has no weight");
522          }
523        try {
524          weight = Double.parseDouble(deepcontains.property("weight").value().toString());
525          }
526        catch (NumberFormatException e) {
527          throw new IllegalArgumentException("Scoped deepcontains edge has a non-numeric weight", e);
528          }
529        if (!Double.isFinite(weight) || weight < 0.0) {
530          throw new IllegalArgumentException("Scoped deepcontains edge has an invalid weight: " + weight);
531          }
532        cls = new OCol(ocol1);
533        types0.add(cls);
534        types.add(cls);
535        weights0.put(cls, weight);
536        }
537      // Double loop over accumulated weights and fill weights between OCols
538      for (OCol cls1 : types0) {
539        weight1 = weights0.get(cls1);
540        for (OCol cls2 : types0) {
541          weight2 = weights0.get(cls2);
542          rel = Pair.of(cls1, cls2);
543          // OCol-OCol
544          if (!corrS.containsKey(rel)) {
545            corrS.put(rel, 0.0);
546            }
547          cor = corrS.get(rel);
548          //corrS.put(rel, cor + 1.0);
549          corrS.put(rel, cor + Math.sqrt(weight1 * weight2));
550          }
551        }
552      }
553    // Fill total sizes
554    double sizeS0;
555    for (OCol cls1 : types) {
556      sizeS0 = 0.0;
557      for (OCol cls2 : types) {
558        if (corrS.containsKey(Pair.of(cls1, cls2))) {
559          sizeS0 += corrS.get(Pair.of(cls1, cls2));
560          }
561        }
562      sizeS.put(cls1, sizeS0);
563      }
564    // Mutate only after all scoped correlation inputs have been validated.
565    for (Vertex malformedScopedOCol : malformedScopedOCols) {
566      g().V(malformedScopedOCol).drop().iterate();
567      }
568    // Preserve overlaps crossing the requested-scope boundary; those require
569    // both classifier scopes to be regenerated together.
570    Set<Object> scopedIds = new HashSet<>();
571    for (Vertex scopedOCol : scopedOCols) {
572      scopedIds.add(scopedOCol.id());
573      }
574    Set<Object> internalOverlapIds = new HashSet<>();
575    for (Vertex scopedOCol : scopedOCols) {
576      Iterator<Edge> overlapsIt = scopedOCol.edges(Direction.BOTH, "overlaps");
577      while (overlapsIt.hasNext()) {
578        Edge oldOverlap = overlapsIt.next();
579        if (scopedIds.contains(oldOverlap.outVertex().id()) &&
580            scopedIds.contains(oldOverlap.inVertex().id())) {
581          internalOverlapIds.add(oldOverlap.id());
582          }
583        }
584      }
585    if (!internalOverlapIds.isEmpty()) {
586      g().E(internalOverlapIds.toArray()).drop().iterate();
587      }
588    // Create overlaps
589    int ns = 0;
590    // Loop over OCol-OCol and create overlaps Edge OCol-OCol if non empty 
591    OCol cls1;
592    OCol cls2;
593    for (Map.Entry<Pair<OCol, OCol>, Double> entry : corrS.entrySet()) {
594      rel = entry.getKey();
595      cls1 = rel.first();
596      cls2 = rel.second();
597      weight = entry.getValue();
598      ocol1 = duplicateOCols.contains(cls1) ? findOCol(cls1) : uniqueOCols.get(cls1);
599      ocol2 = duplicateOCols.contains(cls2) ? findOCol(cls2) : uniqueOCols.get(cls2);
600      overlaps = ocol1.addEdge("overlaps", ocol2);
601      overlaps.property("lbl",          "overlaps");
602      overlaps.property("intersection", weight);
603      overlaps.property("sizeIn",       sizeS.get(cls1));
604      overlaps.property("sizeOut",      sizeS.get(cls2));
605      ns++;
606      }
607    commit();
608    log.info("" + ns + " object-object correlations generated");
609    }
610
611  /** Find the endpoint selected by the existing indexed OCol lookup.
612    * @param cls The logical OCol identity.
613    * @return The first matching physical OCol endpoint. */
614  protected Vertex findOCol(OCol cls) {
615    return g().V().has("lbl",        "OCol"          ).
616                   has("survey",     cls.survey()    ).
617                   has("classifier", cls.classifier()).
618                   has("flavor",     cls.flavor()    ).
619                   has("cls",        cls.cls()       ).
620                   next();
621    }
622    
623  /** Create a new {@link FinkHBaseClient}. Singleton when url unchanged.
624    * @param hbaseUrl The HBase url as {@code ip:port:table[:schema]}.
625    * @return         The corresponding {@link FinkHBaseClient}, created and initialised if needed.
626    * @throws LomikelException If cannot be created. */
627  public synchronized FinkHBaseClient fhclient(String hbaseUrl) throws LomikelException {
628    if (hbaseUrl == null) {
629      return _fhclient;
630      }
631    if (hbaseUrl.equals(_fhclientUrl) && _fhclient != null) {
632      return _fhclient;
633      }
634    String[] url = hbaseUrl.split(":", -1);
635    if ((url.length != 3 && url.length != 4) ||
636        url[0].isEmpty() || url[1].isEmpty() || url[2].isEmpty()) {
637      throw new LomikelException("Invalid HBase URL (expected ip:port:table[:schema]): " + hbaseUrl);
638      }
639    String ip     = url[0];
640    String port   = url[1];
641    String table  = url[2];
642    String schema = url.length == 4 ? url[3] : "";
643    FinkHBaseClient candidate = null;
644    try {
645      candidate = new FinkHBaseClient(ip, port);
646      candidate.connect(table, schema);
647      }
648    catch (RuntimeException | LomikelException failure) {
649      if (candidate != null) {
650        candidate.close();
651        }
652      if (failure instanceof LomikelException) {
653        throw (LomikelException)failure;
654        }
655      throw new LomikelException("Cannot initialise FinkHBaseClient for " + hbaseUrl, failure);
656      }
657    FinkHBaseClient previous = _fhclient;
658    _fhclient = candidate;
659    _fhclientUrl = hbaseUrl;
660    if (previous != null) {
661      previous.close();
662      }
663    return _fhclient;
664    }
665    
666  /** Get existing {@link FinkHBaseClient}.
667    * @return The corresponding {@link FinkHBaseClient}.
668    * @throws LomikelException If not yet created. */
669  public FinkHBaseClient fhclient() throws LomikelException {
670    if (_fhclient == null) {
671      throw new LomikelException("FinkHBaseClient not initialised");
672      }
673    return _fhclient;
674    }
675    
676  /** Give HBase url.
677    * @return The HBase url as {@code ip:port:table[:schema]}. */
678  public String hbaseUrl() {
679    return _fhclientUrl;
680    }
681    
682  /** Build an unambiguous identity for a correlation-generation scope. */
683  private static String correlationScope(String survey,
684                                         String classifier,
685                                         String flavor) {
686    return survey + "\u0000" + classifier + "\u0000" + flavor;
687    }
688
689  private FinkHBaseClient _fhclient;
690  
691  private String _fhclientUrl;
692
693  /** Whether this thread's registration participates in a classification transaction. */
694  private ThreadLocal<Boolean> _classificationTransaction = ThreadLocal.withInitial(() -> false);
695
696  /** Objects already selected or created by this thread's classification transaction. */
697  private ThreadLocal<Map<String, Vertex>> _classificationObjects = ThreadLocal.withInitial(HashMap::new);
698   
699 
700  private static String FINK_OBJECTS_WS = "https://api.ztf.fink-portal.org/api/v1/objects";
701  private static String FINK_LATESTS_WS = "https://api.ztf.fink-portal.org/api/v1/latests";
702  private static String FINK_ANOMALY_WS = "https://api.ztf.fink-portal.org/api/v1/anomaly";
703
704  /** Logging . */
705  private static Logger log = LogManager.getLogger(FinkGremlinRecipies.class);
706
707  }