SPARC queries

SELECT DISTINCT
?p
# ?pp
# ?po
WHERE {
  ?dataset a sparc:Dataset .
  ?dataset ?p ?o .
  # ?p ?pp ?po .
} ORDER BY ?p

SELECT DISTINCT
?d
?label
?approach
WHERE {
  VALUES ?match { "electrophysiology" }
  ?d a sparc:Dataset ;
  rdfs:label ?l ;
  TEMP:hasExperimentalApproach ?approach .
  FILTER (str(?approach) = str(?match))
  BIND(substr(str(?l), 0, 40) AS ?label)
  # TODO lift and pull filetypes
}

SELECT DISTINCT
?dataset
?ut
(substr(str(?l), 0, 40) as ?label)
WHERE {
  ?dataset a sparc:Dataset .
  ?dataset TEMP:contentsWereUpdatedAtTime ?ut .
  ?dataset rdfs:label ?l .
} ORDER BY DESC(?ut)

import rdflib
from pyontutils.core import OntResIri
from pyontutils.namespaces import sparc, TEMP, dc, rdfs

ori = OntResIri('https://cassava.ucsd.edu/sparc/preview/exports/curation-export.ttl')
g = ori.graph
gns = g.namespace_manager

def fmt(s, u):
    return f'[[{u}][{s.n3(gns)}]]'

species = set([fmt(do, urih) for s, p, o in g
              if isinstance(o, rdflib.Literal) and
              ('human' in o.lower() or 'homo' in o.lower()) and
              p == sparc.animalSubjectIsOfSpecies
              for do in g[s:TEMP.hasDerivedInformationAsParticipant]
              for urih in g[do:TEMP.hasUriHuman]])

hlabel = set([fmt(s, urih) for s, p, o in g
             if isinstance(o, rdflib.Literal) and
             ('human' in o.lower() or 'homo' in o.lower()) and
             p == rdfs.label
             for urih in g[s:TEMP.hasUriHuman]])

htitle = set([fmt(s, urih) for s, p, o in g
              if isinstance(o, rdflib.Literal) and
              ('human' in o.lower() or 'homo' in o.lower()) and
              p == dc.title
              for urih in g[s:TEMP.hasUriHuman]])

htd = set([fmt(s, urih) for s, p, o in g
           if isinstance(o, rdflib.Literal) and
           ('human' in o.lower() or 'homo' in o.lower()) and
           (p == dc.title or p == dc.description)
           for urih in g[s:TEMP.hasUriHuman]])

counts = dict(species=len(human),
              label=len(hlabel),
              title=len(htitle),
              title_and_desc=len(htd))

[print(_ + r' \\') for _ in ['species n= ' + str(counts['species'])] +
sorted(species) +
['label n= ' + str(counts['label'])] +
sorted(hlabel) +
['title n= ' + str(counts['title'])] +
sorted(htitle) +
['td n= ' + str(counts['title_and_desc'])] +
sorted(htd)]

select distinct
?protocol
?n
(sample(?dataset) AS ?datasetx)
?doi_protocol
(sample(bound(?urih_protocol)) as ?hasUri)
(sample(bound(?some_child)) as ?hasComplexAnnos)
where {
  ?protocol a sparc:Protocol .
  optional { ?protocol TEMP:hasNumberOfProtcurAnnotations ?n }
  optional { ?dataset TEMP:hasProtocol ?protocol }
  optional { ?protocol TEMP:hasDoi ?doi_protocol }
  optional { ?protocol TEMP:hasUriHuman ?urih_protocol }
  optional { ?protocol ?some_predicate ?annotation .
             ?annotation TEMP:protcurChildren ?some_child }
}
group by ?protocol ?n ?doi_protocol ?hasUri
order by desc(bound(?datasetx)) asc(?hasUri) desc(?n)

SELECT (COUNT(DISTINCT ?protocol) AS ?count)
WHERE { ?protocol a sparc:Protocol . }

SELECT (COUNT(*) as ?count) {
  <<sparql-protocols-at-least-one-protc>>
}}

SELECT (COUNT(*) as ?count) {
  <<sparql-protocols-pio-api-at-least-one-protc>>
}

select distinct
?protocol ?anno_count
where {
  values ?some_predicate {
    TEMP:protocolEmploysTechnique
    TEMP:protocolInvolvesAction
    TEMP:protocolInvolvesAspect
    TEMP:protocolInvolvesBlackBox
    TEMP:protocolInvolvesBlackBoxComponent
    TEMP:protocolInvolvesInput
    TEMP:protocolInvolvesInvariant
  }
  ?protocol a sparc:Protocol ;
    ?some_predicate ?annotation ;  # this line an the filter should wind up with the same number TODO add a test to check
    TEMP:hasNumberOfProtcurAnnotations ?anno_count .
  FILTER (?anno_count > 0 )
  FILTER CONTAINS(str(?protocol), "protocols.io/api")
  #FILTER CONTAINS(str(?protocol), "protocols.io")
}

grep -oE 'TEMP:protocol[^\ ]+' protcur.ttl | sort -u

select distinct
?protocol ?anno_count
where {
  values ?some_predicate {
    TEMP:protocolEmploysTechnique
    TEMP:protocolInvolvesAction
    TEMP:protocolInvolvesAspect
    TEMP:protocolInvolvesBlackBox
    TEMP:protocolInvolvesBlackBoxComponent
    TEMP:protocolInvolvesInput
    TEMP:protocolInvolvesInvariant
  }
  ?protocol a sparc:Protocol ;
    ?some_predicate ?annotation ;  # this line an the filter should wind up with the same number TODO add a test to check
    TEMP:hasNumberOfProtcurAnnotations ?anno_count .
  FILTER (?anno_count > 0 )

SELECT (COUNT(*) as ?count) {
  <<sparql-complex-protocols>>
}

SELECT (COUNT(*) as ?count) {
  <<sparql-complex-protocols-published>>
}

<<sparql-complex-protocols>>
    ?protocol TEMP:hasDoi ?doi .
}

select distinct
?protocol ?anno_count
where {
    ?annotation TEMP:protcurChildren ?some_child .
    ?protocol a sparc:Protocol ;
              ?some_predicate ?annotation ;
              TEMP:hasNumberOfProtcurAnnotations ?anno_count .

WHERE {
    ?startsubj TEMP:hasDerivedInformationAsParticipant ?dataset .
    ?subj  TEMP:hasDerivedInformationAsParticipant ?dataset .
    ?subj  TEMP:hasAssignedGroup ?group .
}

SELECT DISTINCT
?dataset
?group
?subj
WHERE {
    ?startsubj TEMP:hasDerivedInformationAsParticipant ?dataset .
    ?subj  TEMP:hasDerivedInformationAsParticipant ?dataset .
    ?subj  TEMP:hasAssignedGroup ?group .
}
ORDER BY ASC(?dataset) ASC(?group) LIMIT ?limit

SELECT DISTINCT
?group
?subj
WHERE {
    ?startsubj TEMP:hasDerivedInformationAsParticipant ?dataset .
    ?subj  TEMP:hasDerivedInformationAsParticipant ?dataset .
    ?subj  TEMP:hasAssignedGroup ?group .
}
ORDER BY ASC(?group) LIMIT ?limit

SELECT DISTINCT
?dataset
?technique
# ?protocol
# ?technique_p
WHERE {
  { ?dataset a sparc:Dataset .
    ?dataset TEMP:protocolEmploysTechnique ?technique .
    # ?technique rdfs:label ?tl
  }
  UNION
  { ?dataset TEMP:hasProtocol ?protocol .
    ?protocol TEMP:protocolEmploysTechnique ?technique .
    # ?technique rdfs:label ?tl
  }
}
LIMIT ?limit

SELECT DISTINCT
?dataset
?completeness
WHERE {
  ?dataset a sparc:Dataset .
  ?dataset TEMP:completenessOfDataset ?completeness .
}
LIMIT ?limit

SELECT DISTINCT
?title
?dataset
WHERE {
    ?startdataset TEMP:collectionTitle ?title .
    ?dataset  TEMP:collectionTitle ?title .
}
ORDER BY ASC(?title)
LIMIT ?limit

SELECT DISTINCT
?dataset
?subject
WHERE {
  ?dataset a sparc:Dataset .
  ?dataset TEMP:isAboutParticipant ?subject .
  ?subject a sparc:Subject .
}
LIMIT ?limit

SELECT DISTINCT
(COUNT(DISTINCT ?subject) as ?count_subject)
WHERE {
  ?dataset a sparc:Dataset .
  ?dataset TEMP:isAboutParticipant ?subject .
  ?subject a sparc:Subject .
}

SELECT DISTINCT
?local_id

?assigned_group

?l_species
?strain
?l_sex

?age_category
?age_value
?age_unit

?mass_value
?mass_unit

WHERE {
  ?subject a sparc:Subject .
  ?subject TEMP:localId ?local_id .
  ?subject sparc:animalSubjectIsOfSpecies ?species . OPTIONAL { ?species rdfs:label ?l_species . }
  OPTIONAL { ?subject sparc:animalSubjectIsOfStrain ?strain . } # ?strain rdfs:label ?l_strain .
  OPTIONAL { ?subject TEMP:hasBiologicalSex ?sex . ?sex rdfs:label ?l_sex . }
  OPTIONAL { ?subject TEMP:hasAgeCategory ?age_category . }
  OPTIONAL { ?subject TEMP:hasAssignedGroup ?assigned_group . }
  # OPTIONAL { ?subject TEMP:participantInPerformanceOf ?protocol . }

  OPTIONAL {
  # mass
  ?subject sparc:animalSubjectHasWeight ?bn_mass .
  ?bn_mass a sparc:Measurement .
  ?bn_mass TEMP:hasUnit ?mass_unit .
  ?bn_mass rdf:value ?mass_value .
}
  OPTIONAL {
  # age
  ?subject TEMP:hasAge ?bn_age .
  ?bn_age a sparc:Measurement .
  ?bn_age TEMP:hasUnit ?age_unit .
  ?bn_age rdf:value ?age_value .
}
#  VALUES ?l_s {?species ?sex}  # doesn't work, if it did it would duplicate rows
#  ?l_s rdfs:label ?label
} LIMIT ?limit

SELECT DISTINCT
(COUNT(DISTINCT ?subject) as ?count_subject)
WHERE {
  ?dataset a sparc:Dataset .
  ?dataset TEMP:isAboutParticipant ?subject .
  ?subject a sparc:Subject .
}

SELECT DISTINCT
?dataset
?sample
WHERE {
  ?dataset a sparc:Dataset .
  ?dataset TEMP:isAboutParticipant ?sample .
  ?sample a sparc:Sample .
}
LIMIT ?limit

SELECT DISTINCT
(COUNT(DISTINCT ?sample) as ?count_sample)
WHERE {
  ?dataset a sparc:Dataset .
  ?dataset TEMP:isAboutParticipant ?sample .
  ?sample a sparc:Sample .
}

SELECT DISTINCT
?p
WHERE {
  ?sample a sparc:Sample .
  ?sample ?p ?o .
}

SELECT DISTINCT
?subject_lid

?local_id

?assigned_group

(str(?anat_ent_src) as ?aess)

WHERE {
  ?sample a sparc:Sample .
  ?sample TEMP:localId ?local_id .

  ?sample TEMP:wasDerivedFromSubject ?subject .
  ?subject TEMP:localId ?subject_lid .
  ?subject a sparc:Subject .

  OPTIONAL { ?sample TEMP:hasAssignedGroup ?assigned_group . }
  OPTIONAL { ?sample TEMPRAW:wasExtractedFromAnatomicalRegion ?anat_ent_src . }
  # OPTIONAL { ?sample TEMP:hasAge ?abe . }
  # OPTIONAL { ?sample TEMP:participantInPerformanceOf ?protocol . }
}
LIMIT ?limit

SELECT DISTINCT
(COUNT(DISTINCT ?person) as ?cp)
WHERE {
  ?person a sparc:Person .
}
LIMIT ?limit

SELECT DISTINCT
(COUNT(DISTINCT ?person) as ?cp)
#?fn
#?ln
WHERE {
  ?person a sparc:Person .
  ?person sparc:firstName ?fn .
  ?person sparc:lastName ?ln .
  FILTER CONTAINS(str(?person), "orcid.org")
}
LIMIT ?limit

SELECT DISTINCT
?dataset
?file_type
WHERE {
  ?dataset a sparc:Dataset .
  ?dataset TEMP:containsFileWithType ?file_type .  # TODO not 100% on the modelling here
}
LIMIT ?limit

grep -rh 'mimetype":' | sort | uniq -c | sort -h

grep -rh 'basename":.\+\.\([a-z]\+\)' | grep -o '\.[a-z]\+"' | sort | uniq -c | sort -h

SELECT DISTINCT
?dataset
?raw
WHERE {
  ?dataset a sparc:Dataset .
  ?dataset TEMP:containsFolderForRawData ?raw .  # TODO not 100% on the modelling here
}
LIMIT ?limit

SELECT DISTINCT
?dataset
?derived
WHERE {
  ?dataset a sparc:Dataset .
  ?dataset TEMP:containsFolderForDerivedData ?derived .  # TODO not 100% on the modelling here
}
LIMIT ?limit

SELECT DISTINCT
?dataset
?code
WHERE {
  ?dataset a sparc:Dataset .
  ?dataset TEMP:containsFolderForCode ?code .  # TODO not 100% on the modelling here
}
LIMIT ?limit

SELECT DISTINCT
?dataset
?dataset_sim
WHERE {
  ?dataset a sparc:Dataset .
  ?dataset_sim a sparc:Dataset .  # TODO not 100% on this
  # the dataset holds the simulation, and is also what references the other datasets
  # whether we need an explicit type for simulation datasets is not clear, I suspect
  # that we do not, since datasets are just data, the aboutness or typeness probably
  # should come from the fact that the dataset specifies or houses a simulation ...
  # ?dataset_sim a sparc:SimulationDataset . # also not good
  # ?dataset_sim TEMP:isSubstrateForSomeComputationalSimulation true .  # FIXME this is bad bad bad
  VALUES ?p {TEMP:derivesParametersFrom TEMP:derivesValidationDataFrom ilxtr:hasInformationInput} .
  ?dataset_sim ?p ?dataset .
}
LIMIT ?limit

SELECT (COUNT(*) as ?count) {
  <<pds-v2>>
}

SELECT DISTINCT ?s ?dataset (str(?id) as ?local_id) ?award ?species
WHERE {

  ?s a sparc:Subject;
     TEMP:localId ?id;
     TEMP:hasDerivedInformationAsParticipant ?dataset ;
     sparc:animalSubjectIsOfSpecies ?species .
  ?dataset a sparc:Dataset ;
           TEMP:hasAwardNumber ?award .

 { SELECT DISTINCT ?id
    WHERE {
          ?s a sparc:Subject ;
              TEMP:localId ?id ;
              TEMP:hasDerivedInformationAsParticipant ?dataset .
    } GROUP BY ?id HAVING (count(?id) > 1) }

}
ORDER BY ?id

# :results table
# RUN:block
SELECT distinct ?s1 ?s2 ?id ?dataset1 ?dataset2 ?species1 ?species2 ?award1 ?award2
WHERE {

  # Give me all subjects with duplicate IDs with unique datasets
  { SELECT ?id
    WHERE {
          ?s1 a owl:NamedIndividual ;
          TEMP:localId ?id ;
          TEMP:hasDerivedInformationAsParticipant ?datasetx .
          ?s2 a owl:NamedIndividual ;
          TEMP:localId ?id ;
          TEMP:hasDerivedInformationAsParticipant ?datasety .
          filter(?datasetx != ?datasety)
    } GROUP BY ?id having (count(?id) > 1)}

  # filter out the cases where there can't be a match because species doesn't match
  ?s1 TEMP:localId ?id ;
  sparc:animalSubjectIsOfSpecies ?species1;
  TEMP:hasDerivedInformationAsParticipant ?dataset1 .
  ?s2 TEMP:localId ?id ;
  sparc:animalSubjectIsOfSpecies ?species2;
  TEMP:hasDerivedInformationAsParticipant ?dataset2 .
  filter(?s1 != ?s2)
  filter(STR(IRI(?s1)) < STR(IRI(?s2))) # this is a hack to filter out duplicate triples

  # filter to find non-matching awards
  # using contact person doesn't make sense as they are 1:1 with award right now
  ?dataset1 a owl:NamedIndividual ;
  TEMP:hasAwardNumber ?award1 .
  ?dataset2 a owl:NamedIndividual ;
  TEMP:hasAwardNumber ?award2 .
} order by ?id limit ?limit

from pprint import pprint
from pyontutils.scigraph import Cypher, Vocabulary
from pyontutils.scigraph_codegen import moduleDirect
from pyontutils.config import auth
scigraphd = moduleDirect(auth.get('scigraph-api'), 'scigraphd')
from pyontutils.config import auth
from nifstd_tools.simplify import apinat_deblob, sub, pred, obj, ematch
from nifstd_tools.simplify import axon, dend, bag, top, ie, ies, iot, iots, ext, fasIn, endIn, layerIn, onts

ext = onts  # XXX schema change


def query(neupop_id):
    import requests
    url = ('https://scicrunch.org/api/1/sparc-scigraph/scigraph/dynamic/'
           f'demos/apinat/neru-7/{neupop_id}?limit=9999999&key={auth.get("scigraph-api-key")}')
    resp = requests.get(url, headers={'Accept': 'application/json'})
    blob = resp.json()
    return blob


def find_terminals(blob, type):
    return [es for es in blob['edges']
            if
            pred(es, iots) and
            obj(es, type) and
            ematch(blob,
                   (lambda e, m:
                    sub(e, m) and
                    pred(e, top) and
                    obj(e, bag)),
                   sub(es))]


def find_region(blob, edge):
    collect = []
    def select(e, m, collect=collect):
        if sub(e, m):
            #print(e)
            if pred(e, layerIn) or pred(e, fasIn) or pred(e, endIn):
                return ematch(blob, select, obj(e))
            elif pred(e, onts):
                region = obj(e)
                collect.extend([b for b in blob['nodes'] if b['id'] == region])
                return region

    ematch(blob, select, sub(edge))
    return collect


def find_region_layer(blob, edge, bindex):
    collect = []
    layers = []
    layers_iots = []
    donel = set()
    doner = set()
    def select_term(e, m, layers=layers):
        if sub(e, m):
            pprint(e)
            if (pred(e, onts)
                or pred(e, iots)):
                layer = obj(e)
                if layer not in donel:
                    donel.add(layer)
                    if pred(e, iots):
                        layers_iots.append(bindex[layer])
                    else:
                        layers.append(bindex[layer])
                return layer

    def select(e, m, collect=collect):
        if sub(e, m):
            if pred(e, layerIn):
                # we're at a layer
                ematch(blob, select_term, sub(e))
                return ematch(blob, select, obj(e))
            elif (pred(e, fasIn)
                  or pred(e, endIn)):
                return ematch(blob, select, obj(e))
            elif pred(e, onts):
                region = obj(e)
                if region not in doner:
                    doner.add(region)
                    collect.append(bindex[region])
                return region
    ematch(blob, select, sub(edge))
    if collect and not layers and not layers_iots:
        layers = [None]
    elif layers_iots and not layers:
        layers = layers_iots

    if len(collect) != len(layers):
        raise ValueError(f'len not matched {len(collect)} {len(layers)}')
    return list(zip(collect, layers))


def find_terminal_regions(blob, type):
    return [region for es in find_terminals(blob, type) for region in find_region(blob, es)]


def find_terminal_region_layers(blob, type, bindex):
    return [(region, layer) for es in find_terminals(blob, type)
            for region, layer in find_region_layer(blob, es, bindex)]


def main(blob=None, blob_string=None, neupop_id=None, remove_converge=remove):
    # do not call this function unless it is from an org babel block, there are free variables
    if blob is None and blob_string is not None:
        import json
        blob = json.loads(blob_string)

    neupop_id = neupop_id if neupop_id else 'ilxtr:neuron-type-keast-6'

    if blob is None:
        blob = query(neupop_id)

    blob, edges, somas, externals, ordering_edges = v = apinat_deblob(
        blob, remove_converge=remove_converge)

    test_call = False  # change to True to test the #+call: expression
    if test_call:
        pprint(v)
    return v
intIn = 'apinatomy:internalIn'

def isLayer(blob, s):
    return ematch(blob, (lambda e, m: sub(e, m) and pred(e, layerIn)), s)

def reclr(blob, start_link):
    # recurse up the hierarchy until fasIn endIn intIn terminates
    #print('starting reclr')
    collect = []
    layer = []
    col = True
    def select_ext(e, m, collect=collect):
        nonlocal col
        nonlocal layer
        if sub(e, m):
            if pred(e, 'apinatomy:cloneOf'):  # should be zapped during simplify
                return ematch(blob, select_ext, obj(e))
            if pred(e, onts) or pred(e, iot) or pred(e, iots):
                #print('select_ext', e)
                external = obj(e)
                #print('se', col, layer)
                if col:
                    if layer:
                        if len(layer) > 1:
                            l, *layer = layer
                        else:
                            l = layer.pop()
                    else:
                        l = None
                    r = [b for b in blob['nodes'] if b['id'] == external][0]['id']  # if this is empty we are in big trouble
                    collect.append((l, r))
                else:
                    l = [b for b in blob['nodes'] if b['id'] == external][0]['id']
                    layer.append(l)
                #print(layer)
                return external

    def select(e, m):
        nonlocal col
        if sub(e, m):
            #print(e)
            if pred(e, layerIn) or pred(e, fasIn) or pred(e, endIn) or pred(e, intIn):
                #print('select', e)
                col = not isLayer(blob, obj(e))
                #print('s', col)
                ematch(blob, select_ext, obj(e))
                ematch(blob, select, obj(e))

    ematch(blob, select, start_link)

    return collect

def lay_reg(blob, start):
    direct = [obj(t) for t in
              ematch(blob, (lambda e, m: sub(e, m)
                            and (pred(e, intIn) or  # XXX why isn't this getting pulled in correctly for somas? answer: missing somas in starts? surely not, ARGH a lyph internalIn a layer
                                 pred(e, endIn) or
                                 pred(e, fasIn))),
                     start)]

    layers = [obj(t) for d in direct for t in
              ematch(blob, (lambda e, m: sub(e, m)
                            and isLayer(blob, m)
                            and (pred(e, iot) or
                                 pred(e, iots) or
                                 pred(e, onts))),
                     d)]

    lregs = []
    if layers:
        ldir = [obj(t) for d in direct for t in
                ematch(blob, (lambda e, m: sub(e, m)
                              and pred(e, layerIn)),
                       d)]

        lregs = [obj(t) for d in ldir for t in
                 ematch(blob, (lambda e, m: sub(e, m)
                               and not isLayer(blob, m)
                               and (pred(e, iot) or
                                    pred(e, iots) or  # XXX ie seems to have no hits now
                                    pred(e, onts))),
                        d)]

    regions = [obj(t) for d in direct for t in
               ematch(blob, (lambda e, m: sub(e, m)
                             and not isLayer(blob, m)
                             and (pred(e, iot) or
                                  pred(e, iots) or
                                  pred(e, onts))),
                      d)]

    #if start == 'kblad:snl16':
        #print(start, direct, layers, regions)

    # FIXME TODO nested regions and layers
    lrs = reclr(blob, start)  # FIXME doesn't pair regions and layers
    # FIXME SIGH SIGH SIGH start can be either part of a next >_<
    #print(lrs)

    assert not (lregs and regions), (lregs, regions)  # not both
    regions = lregs if lregs else regions
    #out = start, layers[0] if layers else None, regions[0] if regions else None, tuple([l for l in lrs])
    out = start, tuple(lrs)
    if out:
      return out

def render_blob(blob):
    starts = [obj(e) for e in blob['edges'] if pred(e, 'apinatomy:lyphs')]

    nxt = 'apinatomy:next'
    nxts = 'apinatomy:next*'

    nexts = [(sub(t), obj(t)) for start in starts for t in
             ematch(blob, (lambda e, m: pred(e, nxt) or pred(e, nxts)), None)]

    #pprint(nexts)

    connected_pairs = sorted(set([tuple([lay_reg(blob, e) for e in p]) for p in nexts]))
    #pprint(connected_pairs)
    return connected_pairs
sgv = Vocabulary()
sgd = scigraphd.Dynamic(cache=True, verbose=True, do_error=True)
sgc = scigraphd.Cypher(cache=True, verbose=True)
q_neru_6 = """
MATCH (neupop:Class{iri: $neupop_id})
-[a:apinatomy:annotates]->(neugrp:NamedIndividual{`https://apinatomy.org/uris/readable/description`: "dynamic"}) // FIXME HACK

, (neugrp)
-[:apinatomy:links]->(link)
-[c:apinatomy:fasciculatesIn|apinatomy:endsIn*0..1]->(lyph_or_layer) // real lyphs convey things, layers do not
-[d:apinatomy:layerIn*0..1]->(lyph)
-[:apinatomy:conveys*0..1]->() // make sure we are at a real lyph

WITH neugrp, link, lyph, a, c, d
OPTIONAL MATCH layer_ext = (lyph)
<-[d*1]-(layer)-[:apinatomy:cloneOf*0..1]->()-[:apinatomy:inheritedExternal]->()

WITH neugrp, link, lyph, a, c, d, layer_ext
OPTIONAL MATCH more = (lyph)
-[:apinatomy:layerIn|apinatomy:endsIn|apinatomy:fasciculatesIn|apinatomy:internalIn|apinatomy:cloneOf*1..]->()
-[:apinatomy:ontologyTerms|apinatomy:inheritedExternal]->()

WITH neugrp, link, lyph, a, c, d, layer_ext, more // there is a difference here because the previous match does not require lyphs to have external ids
MATCH (lyph)
-[e:apinatomy:ontologyTerms]->(region)

, p2 = (link)
-[:apinatomy:conveyingLyph]->(cl)
-[:apinatomy:topology]->()

, (cl)
-[x:apinatomy:inheritedExternal*0..1]->()

// use apinatomy:next to extract ordering information
, (link)
-[f:apinatomy:next|apinatomy:nextChainStartLevels*0..]->()
-[g:apinatomy:target*0..1]->()
-[h:apinatomy:rootOf*0..1]->()
-[i:apinatomy:levels*0..1]->()
<-[:apinatomy:links]-(neugrp)

// publications
WITH neugrp, a, c, d, e, f, g,h,i, p2, x, layer_ext, more
OPTIONAL MATCH path = (neugrp)
-[:apinatomy:references]->(pub)
-[:type]->(:Class{iri: "https://apinatomy.org/uris/elements/Reference"}) // cannot be curied, dynamic endpoints will not expand it

RETURN a, null as b, c, d, e, f, g,h,i, path, p2, x, layer_ext, more

UNION

// this part usually only returns the soma housing lyph
MATCH (neupop:Class{iri: $neupop_id})
-[a:apinatomy:annotates]->(neugrp:NamedIndividual{`https://apinatomy.org/uris/readable/description`: "dynamic"}) // FIXME HACK
-[b:apinatomy:lyphs]->(lyph)
-[c:apinatomy:internalIn]->()
-[d:apinatomy:ontologyTerms*0..1]->(region)
, p2 = (lyph)
-[:apinatomy:conveys]->(soma_link)
-[:apinatomy:source|apinatomy:target]->(soma_node)
-[:apinatomy:sourceOf]->(chain_link)
, (chain_link)
-[:apinatomy:levelIn]->(chain)
, (soma_node)
-[:apinatomy:rootOf]->(chain)

WITH lyph, a, b, c, d, p2
OPTIONAL MATCH more = (lyph)
-[:apinatomy:layerIn|apinatomy:endsIn|apinatomy:fasciculatesIn|apinatomy:internalIn|apinatomy:cloneOf*1..]->()
-[:apinatomy:ontologyTerms|apinatomy:inheritedExternal]->()

RETURN a, b, c, d, null AS e, null AS f, null AS g, null AS h, null AS i, null AS path, p2, null as x, null as layer_ext, more
"""
def neupop_blob(neupop_id):
    # XXX requires a patch to Cypher and CypherBase to support arbitrary kwargs
    blob = sgc.execute(query=q_neru_6, limit=None, neupop_id=neupop_id, output='application/json')
    return blob

def sigh():
    blobs = [neupop_blob(f"ilxtr:neuron-type-keast-{npid}") for npid in range(1, 21)]
    debs = [main(blob=blob)[0] for blob in blobs]
    simples = [render_blob(deb) for deb in debs]
    #[pprint(simple) for simple in simples[:1]]

def fbid(id):
    if id:
        result = sgv.findById(id)
        if result['labels']:
            return result['labels'][0]

def fbids(ids):
    return [(id, fbid(id)) for id in ids]

def rendlab(pairs):
    for pair in pairs:
        yield tuple((start, *(tuple(fbids(hp)) for hp in housings)) for start, housings in pair)
    #for (*tstart, sid, sasdf), (*tend, eid, easdf) in pairs:
        #yield ((*tstart, sid, fbid(sid), fbids(sasdf)) if sid else (*tstart, sid, fbids(sasdf)),
               #(*tend, eid, fbid(eid), fbids(easdf)) if eid else (*tend, eid, fbids(easdf)))
from collections import defaultdict
q_cnap = """
OPTIONAL MATCH (start:Ontology)
<-[:isDefinedBy]-(graph:NamedIndividual)
-[:type]->({iri: "https://apinatomy.org/uris/elements/Graph"})
, (start)
<-[:isDefinedBy]-(external:Class)
-[:subClassOf*]->(:Class {iri: "http://uri.interlex.org/tgbugs/uris/readable/NeuronEBM"})
return external
"""

def get_label(region):
    return id_label[region]

def get_labels(regions):
    return [get_label(region) for region in regions]

d = sgc.execute(query=q_cnap, limit=99999, output='application/json')
ids = [n['id'] for n in d['nodes']]
blobs = [neupop_blob(id) for id in ids]
id_label = {n['id']:n['lbl'] if 'lbl' in n and n['lbl'] else n['id'] for blob in blobs for n in blob['nodes']}

_ext_all = sorted(set([obj(e) for b in blobs for e in b['edges'] if pred(e, 'apinatomy:ontologyTerms')]))
ext_pop = {i:set([obj(e) for e in blob['edges'] if pred(e, 'apinatomy:ontologyTerms')]) for i, blob in zip(ids, blobs)}

# TODO popuation, region, process type, terminal
pop_by_reg_ads_trm = []

pop_by_reg = defaultdict(list)
for i, regs in ext_pop.items():
    for reg in regs:
        pop_by_reg[reg].append(i)

def key_allc(region):
    count = 0
    for i, id in enumerate(ids):
        if region in ext_pop[id]:
            count += 1
    return count

_ext_allc = sorted(_ext_all, key=key_allc)

def key_pop(pop):
    return -len(ext_pop[pop])

ids_ord = sorted(ids, key=key_pop)

def key_all(region):
    first = None
    count = 0
    regs = []
    iths = []
    for i, id in enumerate(ids_ord):
        if region in ext_pop[id]:
            regs.append(region)
            iths.append(i)
            #count += 1  # wow =+ 1 no error wat
            #if first is None:
            #first = i

    return iths[0], *[-i for i in iths[1:]]

ext_all = sorted(_ext_all, key=key_all)
n_region = len(ext_all)
n_pop = len(ext_pop)
header = [['neupop', *ext_all]]  # TODO -> label
xl = get_labels(header[0][1:])
rows = header + [[id, *[region in ext_pop[id] for region in ext_all]] for id in ids_ord]

def jac(a, b):
    anb, aub = zip(*[(ar and br, ar or br) for ar, br in zip(a, b)])
    sanb, saub = sum(anb) , sum(aub)
    if not sanb and not saub:
        return sanb  # 0
    elif not sanb:
        return saub
    elif not saub:
        return sanb
    else:
        return sanb / saub

def sumjac(rows, norm=False):
    # actually want distance from .5 ?
    mat = jac_mat(rows, switch=True)
    #pprint(mat)
    return [(sum(r[:-1]) * sum(rows[r[-1] + 1][1:]) if norm else 1, r[-1]) for i, r in enumerate(mat)]

def sumjac_norm(rows):
    return sumjac(rows, norm=True)

def jac_mat(rows, switch=False, reverse=False):
    asdf = []
    for i, (_, *rowa) in enumerate(rows[1:]):
        jacr = []
        for j, (_, *rowb) in enumerate(rows[1:]):
            jci = jac(rowa, rowb)
            if switch:
                jacr.append(jci)
            else:
                jacr.append((jci, j))
        if switch:
            jacr.append(i)
            asdf.append(jacr)
        else:
            minimized = [i for j, i in sorted(jacr)][1:]  # 1: to skip self
            minimized.append(i)
            asdf.append(minimized)

    return asdf

def jmc(rows):
    return jac_mat(rows, switch=True)

def jmcr(rows):  # XXX broken
    return jac_mat(rows, switch=True, reverse=True)

def jac_sigh(rows):
    sigh = jmc(rows)
    return [(np.std(s[:-1]), np.mean(s[:-1]), s[-1]) for s in sigh]

def jac_sigh(rows):
    sigh = jmc(rows)
    #return [(np.std(s[:-1]), np.mean(s[:-1]), s[-1]) for s in sigh]
    return [(*[((x - .5) ** 2) ** .5 for x in s[:-1]], s[-1]) for s in sigh]

def resort(rows, sortf, reverse=False):
    asdf_ord = sorted(sortf(rows), reverse=not reverse) # don't think too hard about it
    ind_rank = {j:i for i, (*_, j) in enumerate(asdf_ord)}

    def srt(row):
        ind = rows.index(row)
        if ind == 0:
            return 0
        else:
            return ind_rank[ind - 1]

    return sorted(rows, key=srt)

def sumthm(rows):
    return [[-sum(row[1:]), i] for i, row in enumerate(rows[1:])]

def sumth(rows):
    return [[sum(row[1:]), i] for i, row in enumerate(rows[1:])]

def blank(rows):
    return [[*row[1:], i] for i, row in enumerate(rows[1:])]

def sumoth(rows):
    crank = [sum(col) for col in list(zip(*rows[1:]))[1:]]
    return [[*
             #[crank[j] if c and crank[j] == max([crank[x] for x, v in enumerate(row[1:]) if v]) else 0
             [crank[j] if c else 0
             for j, c in enumerate(row[1:])], i] for i, row in enumerate(rows[1:])]

def sumothm(rows):
    crank = [sum(col) for col in list(zip(*rows[1:]))[1:]]
    return [[*
             #[crank[j] if c and crank[j] == max([crank[x] for x, v in enumerate(row[1:]) if v]) else 0
             [-crank[j] if c else 0
             for j, c in enumerate(row[1:])], i] for i, row in enumerate(rows[1:])]

def maxp(rows):
    crank = [sum(col) for col in list(zip(*rows[1:]))[1:]]
    return [[
             sorted([crank[j] if c else 0 for j, c in enumerate(row[1:])])[0],
             *[-_ for _ in sorted([crank[j] if c else 0 for j, c in enumerate(row[1:])])[1:]]
        , i]
            for i, row in enumerate(rows[1:])]

def jacb(rows):
    a = [row[1:] for row in rows[1:]]
    aT = list(zip(*a))
    # FIXME symmetry
    rd = [[jac(r1, r2) for r2 in a] for r1 in a]
    cd = [[jac(c1, c2) for c2 in aT] for c1 in aT]
    #pprint(rd)
    #pprint(cd)
    # output is a 4d matrix, from rc -> to rc
    dist = np.zeros((len(rd), len(cd), len(rd), len(cd)))
    for i, row_dists in enumerate(rd):
        for k, row_dist in enumerate(row_dists):
            for j, col_dists in enumerate(cd):
                for l, col_dist in enumerate(col_dists):
                    dist[i, j, k, l] = (row_dist ** 2 + col_dist ** 2) ** .5
    #pprint(dist)
    return dist

rows = resort(rows, sumth)
rows = list(zip(*resort(list(zip(*rows)), blank, reverse=True)))
rows = resort(rows, jmc)

if False:
    import seaborn
    seaborn.set(rc={'figure.figsize':(50, 50)})
    #seaborn.set(rc={'figure.figsize':(5, 5)})
    axes = seaborn.heatmap(
        [r[1:] for r in rows[1:]],
        xticklabels=xl,
        yticklabels=[r[0] for r in rows[1:]],
        cbar=False,
        square=True,
    )
    # axes.get_figure().savefig('heatmap-pop-reg-test.svg')

import csv
csv_path = 'heatmap-pop-reg.csv'
with open(csv_path, 'wt') as f:
    csv.writer(f).writerows(rows)

MATCH (neupop:Class{iri: $neupop_id})
-[a:apinatomy:annotates]->(neugrp:NamedIndividual{`https://apinatomy.org/uris/readable/description`: "dynamic"}) // FIXME HACK

, (neugrp)
-[:apinatomy:links]->(link)
-[c:apinatomy:fasciculatesIn|apinatomy:endsIn*0..1]->(lyph_or_layer) // real lyphs convey things, layers do not
-[d:apinatomy:layerIn*0..1]->(lyph)
-[:apinatomy:conveys*0..1]->() // make sure we are at a real lyph

WITH neugrp, link, lyph, a, c, d
OPTIONAL MATCH layer_ext = (lyph)
<-[d*1]-(layer)-[:apinatomy:cloneOf*0..1]->()-[:apinatomy:inheritedOntologyTerms]->()

WITH neugrp, link, lyph, a, c, d, layer_ext
OPTIONAL MATCH more = (lyph)
-[:apinatomy:layerIn|apinatomy:endsIn|apinatomy:fasciculatesIn|apinatomy:internalIn|apinatomy:cloneOf*1..]->()
-[:apinatomy:ontologyTerms|apinatomy:inheritedOntologyTerms]->()

WITH neugrp, link, lyph, a, c, d, layer_ext, more // there is a difference here because the previous match does not require lyphs to have external ids
MATCH (lyph)
-[e:apinatomy:ontologyTerms]->(region)

, p2 = (link)
-[:apinatomy:conveyingLyph]->(cl)
-[:apinatomy:topology]->()

, (cl)
-[x:apinatomy:inheritedOntologyTerms*0..1]->()

// use apinatomy:next to extract ordering information
, (link)
-[f:apinatomy:next|apinatomy:nextChainStartLevels*0..]->()
-[g:apinatomy:target*0..1]->()
-[h:apinatomy:rootOf*0..1]->()
-[i:apinatomy:levels*0..1]->()
<-[:apinatomy:links]-(neugrp)

// publications
WITH neugrp, a, c, d, e, f, g,h,i, p2, x, layer_ext, more
OPTIONAL MATCH path = (neugrp)
-[:apinatomy:references]->(pub)
-[:type]->(:Class{iri: "https://apinatomy.org/uris/elements/Reference"}) // cannot be curied, dynamic endpoints will not expand it

RETURN a, null as b, c, d, e, f, g,h,i, path, p2, x, layer_ext, more

UNION

// this part usually only returns the soma housing lyph
MATCH (neupop:Class{iri: $neupop_id})
-[a:apinatomy:annotates]->(neugrp:NamedIndividual{`https://apinatomy.org/uris/readable/description`: "dynamic"}) // FIXME HACK
-[b:apinatomy:lyphs]->(lyph)
-[c:apinatomy:internalIn]->()
-[d:apinatomy:ontologyTerms*0..1]->(region)
, p2 = (lyph)
-[:apinatomy:conveys]->(soma_link)
-[:apinatomy:source|apinatomy:target]->(soma_node)
-[:apinatomy:sourceOf]->(chain_link)
, (chain_link)
-[:apinatomy:levelIn]->(chain)
, (soma_node)
-[:apinatomy:rootOf]->(chain)

WITH lyph, a, b, c, d, p2
OPTIONAL MATCH more = (lyph)
-[:apinatomy:layerIn|apinatomy:endsIn|apinatomy:fasciculatesIn|apinatomy:internalIn|apinatomy:cloneOf*1..]->()
-[:apinatomy:ontologyTerms|apinatomy:inheritedOntologyTerms]->()

RETURN a, b, c, d, null AS e, null AS f, null AS g, null AS h, null AS i, null AS path, p2, null as x, null as layer_ext, more

MATCH (neupop:Class{iri: $neupop_id})
-[a:apinatomy:annotates]->(neugrp:NamedIndividual{`https://apinatomy.org/uris/readable/description`: "dynamic"}) // FIXME HACK

, (neugrp)
-[:apinatomy:links]->(link)
-[c:apinatomy:fasciculatesIn|apinatomy:endsIn*0..1]->(lyph_or_layer) // real lyphs convey things, layers do not
-[d:apinatomy:layerIn*0..1]->(lyph)
-[:apinatomy:conveys*0..1]->() // make sure we are at a real lyph

WITH neugrp, link, lyph, a, c, d
OPTIONAL MATCH layer_ext = (lyph)
<-[d*1]-(layer)-[:apinatomy:cloneOf*0..1]->()-[:apinatomy:inheritedExternal]->()

WITH neugrp, link, lyph, a, c, d, layer_ext
OPTIONAL MATCH more = (lyph)
-[:apinatomy:layerIn|apinatomy:endsIn|apinatomy:fasciculatesIn|apinatomy:internalIn|apinatomy:cloneOf*1..]->()
-[:apinatomy:ontologyTerms|apinatomy:inheritedExternal]->()

WITH neugrp, link, lyph, a, c, d, layer_ext, more // there is a difference here because the previous match does not require lyphs to have external ids
MATCH (lyph)
-[e:apinatomy:ontologyTerms]->(region)

, p2 = (link)
-[:apinatomy:conveyingLyph]->(cl)
-[:apinatomy:topology]->()

, (cl)
-[x:apinatomy:inheritedExternal*0..1]->()

// use apinatomy:next to extract ordering information
, (link)
-[f:apinatomy:next|apinatomy:nextChainStartLevels*0..]->()
-[g:apinatomy:target*0..1]->()
-[h:apinatomy:rootOf*0..1]->()
-[i:apinatomy:levels*0..1]->()
<-[:apinatomy:links]-(neugrp)

// publications
WITH neugrp, a, c, d, e, f, g,h,i, p2, x, layer_ext, more
OPTIONAL MATCH path = (neugrp)
-[:apinatomy:references]->(pub)
-[:type]->(:Class{iri: "https://apinatomy.org/uris/elements/Reference"}) // cannot be curied, dynamic endpoints will not expand it

RETURN a, null as b, c, d, e, f, g,h,i, path, p2, x, layer_ext, more

UNION

// this part usually only returns the soma housing lyph
MATCH (neupop:Class{iri: $neupop_id})
-[a:apinatomy:annotates]->(neugrp:NamedIndividual{`https://apinatomy.org/uris/readable/description`: "dynamic"}) // FIXME HACK
-[b:apinatomy:lyphs]->(lyph)
-[c:apinatomy:internalIn]->()
-[d:apinatomy:ontologyTerms*0..1]->(region)
, p2 = (lyph)
-[:apinatomy:conveys]->(soma_link)
-[:apinatomy:source|apinatomy:target]->(soma_node)
-[:apinatomy:sourceOf]->(chain_link)
, (chain_link)
-[:apinatomy:levelIn]->(chain)
, (soma_node)
-[:apinatomy:rootOf]->(chain)

WITH lyph, a, b, c, d, p2
OPTIONAL MATCH more = (lyph)
-[:apinatomy:layerIn|apinatomy:endsIn|apinatomy:fasciculatesIn|apinatomy:internalIn|apinatomy:cloneOf*1..]->()
-[:apinatomy:ontologyTerms|apinatomy:inheritedExternal]->()

RETURN a, b, c, d, null AS e, null AS f, null AS g, null AS h, null AS i, null AS path, p2, null as x, null as layer_ext, more

MATCH (neupop:Class{iri: $neupop_id})
-[a:apinatomy:annotates]->(neugrp:NamedIndividual{`https://apinatomy.org/uris/readable/description`: "dynamic"}) // FIXME HACK

, (neugrp)
-[:apinatomy:links]->(link)
-[c:apinatomy:fasciculatesIn|apinatomy:endsIn*0..1]->(lyph_or_layer) // real lyphs convey things, layers do not
-[d:apinatomy:layerIn*0..1]->(lyph)
-[:apinatomy:conveys*0..1]->() // make sure we are at a real lyph

WITH neugrp, link, lyph, a, c, d
OPTIONAL MATCH layer_ext = (lyph)
<-[d*1]-(layer)-[:apinatomy:cloneOf]->()-[:apinatomy:inheritedOntologyTerms]->()

WITH neugrp, link, lyph, a, c, d, layer_ext // there is a difference here because the previous match does not require lyphs to have external ids
MATCH (lyph)
-[e:apinatomy:ontologyTerms]->(region)

, p2 = (link)
-[:apinatomy:conveyingLyph]->(cl)
-[:apinatomy:topology]->()

, (cl)
-[x:apinatomy:inheritedOntologyTerms*0..1]->()

// use apinatomy:next to extract ordering information
, (link)
-[f:apinatomy:next*0..]->()
-[g:apinatomy:target*0..1]->()
-[h:apinatomy:rootOf*0..1]->()
-[i:apinatomy:levels*0..1]->()
<-[:apinatomy:links]-(neugrp)

// publications
WITH neugrp, a, c, d, e, f, g,h,i, p2, x, layer_ext
OPTIONAL MATCH path = (neugrp)
-[:apinatomy:references]->(pub)
-[:type]->(:Class{iri: "https://apinatomy.org/uris/elements/Publication"}) // cannot be curied, dynamic endpoints will not expand it

RETURN a, null as b, c, d, e, f, g,h,i, path, p2, x, layer_ext

UNION

// this part usually only returns the soma housing lyph
MATCH (neupop:Class{iri: $neupop_id})
-[a:apinatomy:annotates]->(neugrp:NamedIndividual{`https://apinatomy.org/uris/readable/description`: "dynamic"}) // FIXME HACK
-[b:apinatomy:lyphs]->(lyph)
-[c:apinatomy:internalIn]->()
-[d:apinatomy:ontologyTerms*0..1]->(region)
, p2 = (lyph)
-[:apinatomy:conveys]->(soma_link)
-[:apinatomy:source|apinatomy:target]->(soma_node)
-[:apinatomy:sourceOf]->(chain_link)
, (chain_link)
-[:apinatomy:levelIn]->(chain)
, (soma_node)
-[:apinatomy:rootOf]->(chain)

RETURN a, b, c, d, null AS e, null AS f, null AS g, null AS h, null AS i, null AS path, p2, null as x, null as layer_ext

MATCH (neupop:Class{iri: $neupop_id})
-[a:apinatomy:annotates]->(neugrp:NamedIndividual{`https://apinatomy.org/uris/readable/description`: "dynamic"}) // FIXME HACK

// publications
WITH neugrp, a
OPTIONAL MATCH path = (neugrp)
-[:apinatomy:references]->(pub)
-[:type]->(:Class{iri: "https://apinatomy.org/uris/elements/Publication"}) // cannot be curied, dynamic endpoints will not expand it

WITH neugrp, a, path
MATCH (neugrp)
-[b:apinatomy:links]->(link)
-[c:apinatomy:fasciculatesIn|apinatomy:endsIn*0..1]->(lyph_or_layer) // real lyphs convey things, layers do not
-[d:apinatomy:layerIn*0..1]->(lyph)
-[:apinatomy:conveys*0..1]->() // make sure we are at a real lyph

WITH neugrp, link, lyph, a, b, c, d, path // MATCH vs , not all things that match as lyphs have externals
MATCH (lyph)
-[e:apinatomy:ontologyTerms]->(region)

// use apinatomy:next to extract ordering information
WITH neugrp, link, a, b, c, d, e, path
MATCH p2 = (link)
-[:apinatomy:conveyingLyph]->(cl)
-[:apinatomy:topology]->()

WITH neugrp, link, a, b, c, d, e, path, p2, cl
MATCH (cl)
-[x:apinatomy:inheritedOntologyTerms*0..1]->()

WITH neugrp, link, a, b, c, d, e, path, p2, x
MATCH (link)
-[f:apinatomy:next*0..]->()
//-[f:apinatomy:next|apinatomy:nextChainStartLevels*0..]->()
// FIXME these should be collapsing into a single relationship
-[g:apinatomy:target*0..1]->()
-[h:apinatomy:rootOf*0..1]->()
-[i:apinatomy:levels*0..1]->()
<-[:apinatomy:links]-(neugrp)

RETURN a, b, c, d, e, f, g,h,i, path, p2, x

UNION

// this part usually only returns the soma housing lyph
MATCH (neupop:Class{iri: $neupop_id})
-[a:apinatomy:annotates]->(neugrp:NamedIndividual{`https://apinatomy.org/uris/readable/description`: "dynamic"}) // FIXME HACK
-[b:apinatomy:lyphs]->(lyph)
-[c:apinatomy:internalIn]->()
-[d:apinatomy:ontologyTerms*0..1]->(region)
// this variant shows the dead end lyphs that correspond to the fasciculatesIn links above
//-[c:apinatomy:internalIn*0..1]->()
//-[d:apinatomy:ontologyTerms*0..1]->(region)
RETURN a, b, c, d, null AS e, null AS f, null AS g, null AS h, null AS i, null AS path, null as p2, null as x

MATCH (neupop:Class{iri: $neupop_id})
-[a:apinatomy:annotates]->(neugrp:NamedIndividual{`https://apinatomy.org/uris/readable/description`: "dynamic"}) // FIXME HACK

// publications
WITH neugrp, a
OPTIONAL MATCH path = (neugrp)
-[:apinatomy:references]->(pub)
-[:type]->(:Class{iri: "https://apinatomy.org/uris/elements/Publication"}) // cannot be curied, dynamic endpoints will not expand it

WITH neugrp, a, path
MATCH (neugrp)
-[b:apinatomy:links]->(link)
-[c:apinatomy:fasciculatesIn|apinatomy:endsIn*0..1]->(lyph_or_layer) // real lyphs convey things, layers do not
-[d:apinatomy:layerIn*0..1]->(lyph)
-[:apinatomy:conveys*0..1]->() // make sure we are at a real lyph

WITH neugrp, link, lyph, a, b, c, d, path
MATCH (lyph)
-[e:apinatomy:ontologyTerms]->(region)

// use apinatomy:next to extract ordering information
WITH neugrp, link, a, b, c, d, e, path
MATCH (link)
-[f:apinatomy:next*0..]->()
//-[f:apinatomy:next|apinatomy:nextChainStartLevels*0..]->()
// FIXME these should be collapsing into a single relationship
-[g:apinatomy:target*0..1]->()
-[h:apinatomy:rootOf*0..1]->()
-[i:apinatomy:levels*0..1]->()
<-[:apinatomy:links]-(neugrp)

RETURN a, b, c, d, e, f, g,h,i, path

UNION

// this part usually only returns the soma housing lyph
MATCH (neupop:Class{iri: $neupop_id})
-[a:apinatomy:annotates]->(neugrp:NamedIndividual{`https://apinatomy.org/uris/readable/description`: "dynamic"}) // FIXME HACK
-[b:apinatomy:lyphs]->(lyph)
-[c:apinatomy:internalIn]->(e) // e is a hack to get columns to match
-[d:apinatomy:ontologyTerms*0..1]->(region)
// this variant shows the dead end lyphs that correspond to the fasciculatesIn links above
//-[c:apinatomy:internalIn*0..1]->(e)
//-[d:apinatomy:ontologyTerms*0..1]->(region)
RETURN a, b, c, d, null AS e, null AS f, null AS g, null AS h, null AS i, null AS path

MATCH path = (neupop:Class{iri: $neupop_id})
-[:apinatomy:annotates]->(neugrp:NamedIndividual{`https://apinatomy.org/uris/readable/description`: "dynamic"})
-[:apinatomy:links]->()
-[:apinatomy:next*0..]->()
// -[:apinatomy:nextChainStartLevels*0..]-()
//-[:apinatomy:prevChainEndLevels*0..]-()
-[:apinatomy:levelIn*0..]->()
RETURN path

MATCH (neupop:Class{iri: $neupop_id})
-[a:apinatomy:annotates]->(neugrp:NamedIndividual{`https://apinatomy.org/uris/readable/description`: "dynamic"}) // FIXME HACK

// publications
WITH neugrp, a
OPTIONAL MATCH path = (neugrp)
-[:apinatomy:references]->(pub)
-[:type]->(:Class{iri: "https://apinatomy.org/uris/elements/Publication"}) // cannot be curied, dynamic endpoints will not expand it

WITH neugrp, a, path
MATCH (neugrp)
-[b:apinatomy:links]->(link)
-[c:apinatomy:fasciculatesIn|apinatomy:endsIn*0..1]->(lyph_or_layer) // real lyphs convey things, layers do not
-[d:apinatomy:layerIn*0..1]->(lyph)
-[:apinatomy:conveys*0..1]->() // make sure we are at a real lyph

WITH lyph, a, b, c, d, path
MATCH (lyph)
-[e:apinatomy:ontologyTerms]->(region)
RETURN a, b, c, d, e, path

UNION

// this part usually only returns the soma housing lyph
MATCH (neupop:Class{iri: $neupop_id})
-[a:apinatomy:annotates]->(neugrp:NamedIndividual{`https://apinatomy.org/uris/readable/description`: "dynamic"}) // FIXME HACK
-[b:apinatomy:lyphs]->(lyph)
-[c:apinatomy:internalIn]->(e) // e is a hack to get columns to match
-[d:apinatomy:ontologyTerms*0..1]->(region)
// this variant shows the dead end lyphs that correspond to the fasciculatesIn links above
//-[c:apinatomy:internalIn*0..1]->(e)
//-[d:apinatomy:ontologyTerms*0..1]->(region)
return a, b, c, d, null AS e, null AS path

OPTIONAL MATCH (start:Ontology)
<-[:isDefinedBy]-(graph:NamedIndividual)
-[:type]->({iri: "https://apinatomy.org/uris/elements/Graph"})
, (start)
<-[:isDefinedBy]-(external:Class)
-[:subClassOf*]->(:Class {iri: "http://uri.interlex.org/tgbugs/uris/readable/NeuronEBM"})
return external

MATCH (start:Ontology)
<-[:isDefinedBy]-(external:Class)
-[:subClassOf*]->(:Class {iri: "http://uri.interlex.org/tgbugs/uris/readable/NeuronEBM"}) // FIXME
, (start:Ontology)
<-[:apinat:hasGraph]-(graph:Class {iri: ""})
RETURN external

MATCH (start:Ontology {iri: $model_id})
<-[:isDefinedBy]-(external:Class)
-[:subClassOf*]->(:Class {iri: "http://uri.interlex.org/tgbugs/uris/readable/NeuronEBM"}) // FIXME
RETURN external

MATCH (start:Ontology {iri: $model_id})
<-[:isDefinedBy]-(external:Class)
-[:subClassOf*]->(:Class {iri: "http://uri.interlex.org/tgbugs/uris/readable/NeuronEBM"}) // FIXME
,
(external)
-[e:type]->()
RETURN e
UNION
OPTIONAL MATCH (start:Ontology {iri: $model_id})
<-[:isDefinedBy]-(graph:NamedIndividual)
-[:type]->({iri: "https://apinatomy.org/uris/elements/Graph"}) // elements don't have a superclass right now
,
(graph)
-[:apinatomy:references]->(pub)
-[e:type]->(:Class{iri: "https://apinatomy.org/uris/elements/Reference"})
RETURN e

MATCH (region)
-[:apinatomy:annotates]->(lyph)
<-[:apinatomy:lyphs]-
(neugrp:NamedIndividual{`https://apinatomy.org/uris/readable/description`: "dynamic"})
RETURN region

MATCH (neupop:Class{iri: $neupop_id})
-[:apinatomy:annotates]->(neugrp:NamedIndividual{`https://apinatomy.org/uris/readable/description`: "dynamic"})
-[:apinatomy:lyphs]->(end_lyph)
-[t:apinatomy:topology]->(top)
WHERE top.iri IN ["https://apinatomy.org/uris/readable/BAG", "https://apinatomy.org/uris/readable/BAG2"]
WITH neugrp, end_lyph, t
MATCH (type) // FIXME we need more information in the models
<-[x:apinatomy:inheritedOntologyTerms*0..1]-(end_lyph)
-[a:apinatomy:conveys]->(link)
-[b:apinatomy:fasciculatesIn|apinatomy:endsIn*0..1]->(lyph_or_layer) // real lyphs convey things, layers do not
-[c:apinatomy:layerIn*0..1]->(lyph)
-[:apinatomy:conveys*0..1]->() // make sure we are at a real lyph
WITH neugrp, lyph, a,b,c,t,x
MATCH (lyph)
-[e:apinatomy:ontologyTerms]->(region)
RETURN e, a,b,c,t,x

MATCH
(neugrp:NamedIndividual{`https://apinatomy.org/uris/readable/description`: "dynamic"})
-[:apinatomy:links]->(link)
-[:apinatomy:fasciculatesIn|apinatomy:endsIn*0..1]->(lyph_or_layer)
-[:apinatomy:layerIn*0..1]->(lyph)
-[:apinatomy:ontologyTerms*0..1]->(region:Class{iri: $region_id})
WITH neugrp
OPTIONAL MATCH (neupop:Class)
-[e:apinatomy:annotates*0..1]->(neugrp)
WITH e, neupop, neugrp, link
MATCH (link)
-[:apinatomy:conveyingLyph]->(cl)
-[:apinatomy:inheritedOntologyTerms]->(:Class {iri: $type_id})
WITH e, neupop, neugrp, cl
MATCH (cl)
-[:apinatomy:topology]-(top)
WHERE top.iri IN ["https://apinatomy.org/uris/readable/BAG", "https://apinatomy.org/uris/readable/BAG2"]
RETURN e, neupop, neugrp

MATCH
(neugrp:NamedIndividual{`https://apinatomy.org/uris/readable/description`: "dynamic"})
-[:apinatomy:links]->(link)
-[:apinatomy:fasciculatesIn|apinatomy:endsIn*0..1]->(lyph_or_layer)
-[:apinatomy:layerIn*0..1]->(lyph)
-[:apinatomy:ontologyTerms*0..1]->(region:Class{iri: $region_id})
WITH neugrp
OPTIONAL MATCH (neupop:Class)
-[e:apinatomy:annotates*0..1]->(neugrp)
WITH e, neupop, neugrp, link
MATCH (link)
-[:apinatomy:conveyingLyph]->(cl)
-[:apinatomy:inheritedOntologyTerms]->(:Class {iri: $type_id})
RETURN e, neupop, neugrp