Skip to content

Commit

Permalink
Update to basic tests to accommodate changes in Feb19 branch (schemao…
Browse files Browse the repository at this point in the history 
…rg#2149)

* Removed duplicate defintion of SurgicalProcedure

* Removed duplicate definition of 'currency'

* Removed duplicate definition of 'educationRequirements'

* Updated files listings

* Updated tests, added utilities only used by tests, removed redundent logging output - influenced by move to term structure.

* Reorderd code
  • Loading branch information
@RichardWallis @danbri
RichardWallis authored and danbri committed Feb 20, 2019
1 parent 2768a43 commit 3d68d98
Show file tree
Hide file tree
Showing 9 changed files with 315 additions and 337 deletions.
167 changes: 62 additions & 105 deletions api.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@

import apirdflib
import apirdfterm
from apirdfterm import VTerm
from sdoutil import *

#from apirdflib import rdfGetTargets, rdfGetSources
Expand Down Expand Up @@ -949,43 +950,14 @@ def GetAllTypes(layers='core'):
#logging.debug("DataCache HIT: %s" % KEY)
return UtilCache.get(KEY,Utc)
else:
#logging.debug("DataCache MISS: %s" % KEY)
mynode = Unit.GetUnit("schema:Thing", True)
subbed = {}
todo = [mynode]
while todo:
current = todo.pop()
subs = GetImmediateSubtypes(current, EVERYLAYER)
if inLayer(layers,current):
subbed[current] = 1
for sc in subs:
if subbed.get(sc.id) == None:
todo.append(sc)
UtilCache.put(KEY,subbed.keys(),Utc)
return subbed.keys()
sorted_all_types = []
types = VTerm.getAllProperties()
for t in types:
sorted_all_types.append(t.getId())
sorted_all_types.sort()
UtilCache.put(KEY,sorted_all_types,Utc)
return sorted_all_types

def oldGetAllTypes(layers='core'):
global Utc
"""Return all types in the graph."""
KEY = "AllTypes:%s" % layers
if UtilCache.get(KEY+'x',Utc):
#logging.debug("DataCache HIT: %s" % KEY)
return UtilCache.get(KEY,Utc)
else:
#logging.debug("DataCache MISS: %s" % KEY)
mynode = Unit.GetUnit("schema:Thing", True)
subbed = {}
todo = [mynode]
while todo:
current = todo.pop()
subs = GetImmediateSubtypes(current, EVERYLAYER)
if inLayer(layers,current):
subbed[current] = 1
for sc in subs:
if subbed.get(sc.id) == None:
todo.append(sc)
UtilCache.put(KEY,subbed.keys(),Utc)
return subbed.keys()

def GetAllDataTypes(layers='core'):
global Utc
Expand All @@ -996,16 +968,15 @@ def GetAllDataTypes(layers='core'):
return UtilCache.get(KEY,Utc)
else:
#logging.debug("DataCache MISS: %s" % KEY)
mynode = Unit.GetUnit("DataType", True)
mynode = apirdfterm.VTerm.getTerm("Datatype")
subbed = {}
todo = [mynode]
while todo:
current = todo.pop()
subs = GetImmediateSubtypes(current, EVERYLAYER)
if inLayer(layers,current):
subbed[current] = 1
subs = current.getSubs()
subbed[current] = 1
for sc in subs:
if subbed.get(sc.id) == None:
if subbed.get(sc.getId()) == None:
todo.append(sc)
UtilCache.put(KEY,subbed.keys(),Utc)
return subbed.keys()
Expand All @@ -1018,20 +989,18 @@ def GetAllEnumerationValues(layers='core'):
return UtilCache.get(KEY,Utc)
else:
#logging.debug("DataCache MISS: %s" % KEY)
mynode = Unit.GetUnit("schema:Enumeration", True)
mynode = apirdfterm.VTerm.getTerm("schema:Enumeration")
enums = {}
subbed = {}
todo = [mynode]
while todo:
current = todo.pop()
subs = GetImmediateSubtypes(current, EVERYLAYER)
subs = current.getSubs()
subbed[current] = 1
for sc in subs:
vals = GetSources( Unit.GetUnit("rdf:type", True), sc, layers=EVERYLAYER)
for val in vals:
if inLayer(layers,val):
enums[val] = 1
if subbed.get(sc.id) == None:
if sc.isEnumerationValue():
enums[sc.getId()] = 1
if subbed.get(sc.getId()) == None:
todo.append(sc)
UtilCache.put(KEY,enums.keys(),Utc)
return enums.keys()
Expand All @@ -1045,14 +1014,11 @@ def GetAllProperties(layers='core'):
#logging.debug("DataCache HIT: %s" % KEY)
return UtilCache.get(KEY,Utc)
else:
#logging.debug("DataCache MISS: %s" % KEY)
mynode = Unit.GetUnit("Thing")
props = GetSources(Unit.GetUnit("rdf:type", True), Unit.GetUnit("rdf:Property", True), layers=EVERYLAYER)
res = []
for prop in props:
if inLayer(layers,prop):
res.append(prop)
sorted_all_properties = sorted(res, key=lambda u: u.id)
sorted_all_properties = []
props = VTerm.getAllProperties()
for p in props:
sorted_all_properties.append(p.getId())
sorted_all_properties.sort()
UtilCache.put(KEY,sorted_all_properties,Utc)
return sorted_all_properties

Expand All @@ -1064,41 +1030,29 @@ def GetAllTerms(layers='core',includeDataTypes=False):
ret.extend(GetAllDataTypes(layers))
return sorted(ret,key=lambda u: u.id)

def GetParentList(start_unit, end_unit=None, path=[], layers='core'):

"""
Returns one or more lists, each giving a path from a start unit to a supertype parent unit.
example:
for path in GetParentList( Unit.GetUnit("Restaurant") ):
pprint.pprint(', '.join([str(x.id) for x in path ]))
'Restaurant, FoodEstablishment, LocalBusiness, Organization, Thing'
'Restaurant, FoodEstablishment, LocalBusiness, Place, Thing'
"""

if not end_unit:
end_unit = Unit.GetUnit("Thing")

arc=Unit.GetUnit("rdfs:subClassOf")
logging.debug("from %s to %s - path length %d" % (start_unit.id, end_unit.id, len(path) ) )
path = path + [start_unit]
if start_unit == end_unit:
return [path]
if not Unit.GetUnit(start_unit.id):
return []
paths = []
for node in GetTargets(arc, start_unit, layers=layers):
if node not in path:
newpaths = GetParentList(node, end_unit, path, layers=layers)
for newpath in newpaths:
paths.append(newpath)
return paths

def HasMultipleBaseTypes(typenode, layers='core'):

def GetParentPathTo(start_term,end_term=None):
#Output paths from start_term to only if end_term in path
if not end_term:
end_term = VTerm.getTerm("Thing")

parentsList = start_term.getParentPaths()
outList = []
for l in parentsList:
if end_term in l:
path = []
for t in l:
path.append(t)
if t == end_term:
break
outList.append(path)
return outList

def HasMultipleBaseTypes(term, layers='core'):
"""True if this unit represents a type with more than one immediate supertype."""
return len( GetTargets( Unit.GetUnit("rdfs:subClassOf", True), typenode, layers ) ) > 1
t = VTerm.getTerm(term)
parents = t.getSupers()
return len(parents) > 1

EXAMPLESMAP = {}
EXAMPLES = {}
Expand Down Expand Up @@ -1265,21 +1219,26 @@ def GetJsonLdContext(layers='core'):
externalines = ""
typins = ""
url = apirdfterm.VTerm.getTerm("schema:URL")
date = apirdfterm.VTerm.getTerm("schema:Date")
datetime = apirdfterm.VTerm.getTerm("schema:DateTime")
for t in apirdfterm.VTerm.getAllTerms(supressSourceLinks=True):
if t.isClass() or t.isEnumeration() or t.isEnumerationValue() or t.isDataType():
line = " \"" + t.getId() + "\": {\"@id\": \"" + t.getPrefixedId() + "\"},"
elif t.isProperty():
ranges = t.getRanges()
range = t.getRanges()

for r in ranges:

if r == url:
typins = ", \"@type\": \"@id\""
break
elif r.isDataType():
typins = ""
else:
typins = ", \"@type\": \"@id\""
type = None

if url in range:
type = "@id"
elif date in range:
type = "Date"
elif datetime in range:
type = "DateTime"

typins = ""
if type:
typins = ", \"@type\": \"" + type + "\""

line = " \"" + t.getId() + "\": { \"@id\": \"" + t.getPrefixedId() + "\"" + typins + "},"

Expand Down Expand Up @@ -1469,8 +1428,8 @@ def load_examples_data(extensions):
ExampleStore.store(EXAMPLES)
ExampleMap.store(EXAMPLESMAP)
memcache.set("ExmplesLoaded",value=True)
else:
load_local_examples_data(extensions)
else:
load_local_examples_data(extensions)

def load_example_sources(files):
if files:
Expand All @@ -1483,9 +1442,7 @@ def load_example_sources(files):


def load_local_examples_data(extensions):
log.info("Skipping examples load")
return

log.info("Loading Local Examples")
load = False
if getInTestHarness():
load = True
Expand Down
32 changes: 14 additions & 18 deletions apirdflib.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,13 +165,13 @@ def rdfGetTriples(id):
targets = []
fullId = id

log.info("rdfgetTriples(%s)" % fullId)
#log.info("rdfgetTriples(%s)" % fullId)
if ':' in id: #Includes full path or namespaces
fullId = id
else:
#fullId = api.SdoConfig.baseUri() + "/" + id
fullId = api.SdoConfig.baseUri() + id
log.info("rdfgetTriples(%s)" % fullId)
#log.info("rdfgetTriples(%s)" % fullId)

first = True
unit = None
Expand All @@ -185,7 +185,7 @@ def rdfGetTriples(id):

res = rdfQueryStore(q,STORE)

log.info("rdfgetTriples RES: %s: %s" % (len(res), res))
#log.info("rdfgetTriples RES: %s: %s" % (len(res), res))
for row in res:
# if source == "http://meta.schema.org/":
# log.info("Triple: %s %s %s %s" % (source, row.p, row.o, row.g))
Expand Down Expand Up @@ -225,7 +225,7 @@ def rdfGetSourceTriples(target):
id = target.id
target.sourced = True
sources = []
log.info("rdfGetSourceTriples(%s)" % id)
#log.info("rdfGetSourceTriples(%s)" % id)
if ':' in id: #Includes full path or namespaces
fullId = id
else:
Expand All @@ -234,16 +234,16 @@ def rdfGetSourceTriples(target):
targ = fullId
if fullId.startswith('http://') or fullId.startswith('https://'):
targ = "<%s>" % fullId
log.info("rdfGetSourceTriples(%s)" % targ)
#log.info("rdfGetSourceTriples(%s)" % targ)

q = "SELECT ?g ?s ?p WHERE {GRAPH ?g {?s ?p %s }}" % targ
log.info("%s" % q)
#log.info("%s" % q)

res = rdfQueryStore(q,STORE)
log.info("rdfGetSourceTriples: res: %s %s" % (len(res),res))
#log.info("rdfGetSourceTriples: res: %s %s" % (len(res),res))

for row in res:
log.info("SUB: %s PRED: %s OBJ: %s" % (stripID(row.s),stripID(row.p),stripID(fullId)))
#log.info("SUB: %s PRED: %s OBJ: %s" % (stripID(row.s),stripID(row.p),stripID(fullId)))
layer = str(getRevNss(str(row.g)))
unit = api.Unit.GetUnit(stripID(row.s),True)
p = stripID(row.p)
Expand Down Expand Up @@ -304,19 +304,15 @@ def rdfgettops():
return TOPSTERMS

def countTypes(extension="ALL",includeAttic=False):
log.info("countTypes()")
#log.info("countTypes()")
filter = countFilter(extension=extension, includeAttic=includeAttic)
log.info("countTypes 1")
query= ('''select (count (?term) as ?cnt) where {
?term a rdfs:Class.
?term rdfs:subClassOf* schema:Thing.
%s
}''') % filter
log.info("countTypes 2")
graph = queryGraph()
log.info("countTypes 3")
count = 0
log.info ("QUERY %s" % query)
res = rdfQueryStore(query,graph)
for row in res:
count = row.cnt
Expand Down Expand Up @@ -392,7 +388,7 @@ def buildSingleTermGraph(node,excludeAttic=True,markdown=True):
full = str(n)
ret = None

log.info("NAME %s %s"% (n,full))
#log.info("NAME %s %s"% (n,full))
atts = None
attic = api.SdoConfig.atticUri()
if attic:
Expand Down Expand Up @@ -427,7 +423,7 @@ def buildSingleTermGraph(node,excludeAttic=True,markdown=True):
}
}
''' % (n,api.SdoConfig.vocabUri())
log.info("Query: %s" % query)
#log.info("Query: %s" % query)

ret = rdfQueryStore(query,q)
for row in ret:
Expand All @@ -448,7 +444,7 @@ def buildSingleTermGraph(node,excludeAttic=True,markdown=True):
}
}
''' % (n,api.SdoConfig.vocabUri())
log.info("Query: %s" % query)
#log.info("Query: %s" % query)
ret = rdfQueryStore(query,q)
for row in ret:
g.add((row.prop,SCHEMA.domainIncludes,row.term))
Expand All @@ -467,7 +463,7 @@ def buildSingleTermGraph(node,excludeAttic=True,markdown=True):
}
}
''' % (n,api.SdoConfig.vocabUri())
log.info("Query: %s" % query)
#log.info("Query: %s" % query)
ret = rdfQueryStore(query,q)
for row in ret:
#log.info("adding %s %s %s" % (row.term,RDFS.subPropertyOf,row.super))
Expand Down Expand Up @@ -545,7 +541,7 @@ def graphFromFiles(files,prefix=None,path=None):
if not "://" in f:
f = full_path(f)

log.info("Trying %s" % f)
#log.info("Trying %s" % f)
try:
g.parse(f,format='json-ld')
msg = ""
Expand Down
Loading

0 comments on commit 3d68d98

Please sign in to comment.