001package com.astrolabsoftware.FinkBrowser.Januser;
002
003import com.Lomikel.HBaser.HBaseClient;
004import com.Lomikel.Utils.LomikelException;
005import com.astrolabsoftware.FinkBrowser.FinkPortalClient.FPC;
006import com.astrolabsoftware.FinkBrowser.HBaser.Clusteriser.ClusterFinder;
007
008// org.json
009import org.json.JSONArray;
010import org.json.JSONObject;
011
012// Java
013import java.util.Arrays;
014import java.util.Set;
015import java.util.TreeSet;
016import java.util.Map;
017import java.util.TreeMap;
018import java.util.List;
019import java.util.ArrayList;
020import java.io.IOException;
021
022// Log4J
023import org.apache.logging.log4j.Logger;
024import org.apache.logging.log4j.LogManager;
025
026/** <code>FeaturesClassifier</code> classifies sources according to
027  * HBase <tt>lc_features_*</tt> field.
028  * <em>flavor</em> points to resource carrying model. 
029  * @opt attributes
030  * @opt operations
031  * @opt types
032  * @opt visibility
033  * @author <a href="mailto:Julius.Hrivnac@cern.ch">J.Hrivnac</a> */
034// BUG: jd should be String or long
035public class FeaturesClassifier extends Classifier {
036
037  @Override
038  public void classify(FinkGremlinRecipies recipies,
039                       String              oid) throws LomikelException {
040    String jd;
041    String cl;
042    Map<String, String> value;
043    String[] featuresS;
044    double[] featuresD;
045    String fg;
046    String fr;
047    Map<String, Set<String>> allInstances; // cl -> [jd]
048    Map<String, Double>      allWeights;   // jd -> w
049    Set<String> jds;
050    String key;
051    Map<String, Map<String, String>> alerts = recipies.fhclient().scan(null,
052                                                                       "key:key:" + oid + ":prefix",
053                                                                       "i:jd,d:lc_features_g,d:lc_features_r",
054                                                                       0,
055                                                                       0,
056                                                                       false,
057                                                                       false);
058    allInstances = new TreeMap<>();
059    allWeights   = new TreeMap<>();
060    // get all alerts (jd) and their classses
061    boolean isClassified = false;
062    for (Map.Entry<String, Map<String, String>> entry : alerts.entrySet()) {
063      value = entry.getValue();
064      jd = value.get("i:jd");
065      if (value.containsKey("d:lc_features_g") &&
066          value.containsKey("d:lc_features_r")) {
067        fg = value.get("d:lc_features_g").replaceFirst("\\[", "").replaceAll("]$", "");
068        fr = value.get("d:lc_features_r").replaceFirst("\\[", "").replaceAll("]$", "");
069        // BUG: some models replace by mean
070        featuresS = (fg + "," + fr).replaceAll("null", "0.0").
071                                    replaceAll("NaN",  "0.0").
072                                    split(",");
073        featuresD = Arrays.stream(featuresS).
074                           mapToDouble(Double::parseDouble).
075                           toArray();
076        cl = String.valueOf(finder().transformAndPredict(featuresD));                  
077        if (!cl.equals("-1")) {
078          if (allInstances.containsKey(cl)) {
079            jds = allInstances.get(cl);
080            jds.add(jd);
081            }
082          else {
083            jds = new TreeSet<String>();
084            jds.add(jd);
085            allInstances.put(cl, jds);
086            }
087          allWeights.put(jd, 1.0);
088          }
089        isClassified = true;
090        }
091      else {
092        //log.warn("Alert " + entry.getKey() + " has no features");
093        }
094      }
095    // rearrange instances and weights and register
096    double weight;
097    double totalWeight;
098    double w;
099    totalWeight = 0;
100    List<String> instancesL;
101    List<Double> weightsL;
102    for (Map.Entry<String, Set<String>> cls : allInstances.entrySet()) {
103      for (String instance : cls.getValue()) {
104        totalWeight += allWeights.get(instance);
105        }
106      }
107    for (Map.Entry<String, Set<String>> cls : allInstances.entrySet()) {
108      key = "FC-" + cls.getKey();
109      instancesL = new ArrayList<String>(cls.getValue());
110      weightsL   = new ArrayList<Double>();
111      w = 0;
112      for (String instance : instancesL) {
113        weightsL.add(allWeights.get(instance));
114        w += allWeights.get(instance);
115        }
116      weight = w / totalWeight;
117      recipies.registerSoI(this, key, oid, weight, instancesL, weightsL);
118      }
119    if (!isClassified) {
120      log.warn("Source " + oid + " cannot be classified because his alerts have no LC features");
121      }
122    }
123    
124  /** Give {@link ClusterFinder} to current database. Singleton.
125    * @return The corresponding {@link ClusterFinder}. 
126    * @throws LomikelExceltion If {@link ClusterFinder} cannot be created. */
127  private ClusterFinder finder() throws LomikelException {
128    if (_finder == null) {
129      try {
130         ClassLoader classLoader = getClass().getClassLoader();
131        _finder = new ClusterFinder(classLoader.getResource(flavor() + "/scaler_params.json"),
132                                    classLoader.getResource(flavor() + "/pca_params.json"),
133                                    classLoader.getResource(flavor() + "/cluster_centers.json"));
134        }
135      catch (IOException | IllegalArgumentException e) {
136        throw new LomikelException("Cannot create Cluster Finder for " + flavor(), e);
137        }
138      }
139    return _finder;
140    }
141  
142  private ClusterFinder _finder;
143
144  /** Logging . */
145  private static Logger log = LogManager.getLogger(FeaturesClassifier.class);
146  
147  }
148           
149