Tasks resolved, bug fixes, code improvements

datadump/data_loader.py

@@ -1,10 +1,11 @@
-import os.path
+import os
 import time
 
 import ujson as json
 from mpds_client import MPDSDataRetrieval, APIError, MPDSDataTypes
 
 
+
 class DataExportMPDS:
     """
     Make requests to MPDS database and save data in files

kickoff/miner_ab_etransport.py

@@ -8,6 +8,7 @@
 
 from etransport_raw import analyze_raw # this is given in the supplied file "etransport_raw.py"
 
+
 # the raw simulation data on the MPDS are in 7z format
 # so we need e.g. the latest dev version of the pylzma package
 # pip install git+https://github.com/fancycode/pylzma

kickoff/miner_bgkmeans.py

@@ -14,6 +14,7 @@
 from element_groups import get_element_group
 
 
+
 client = MPDSDataRetrieval()
 
 dfrm = client.get_dataframe(
@@ -27,7 +28,7 @@
     ]},
     columns=['Formula', 'Elements', 'SG', 'Units', 'Bandgap']
 )
-dfrm = dfrm.filter((dfrm['Units'] == 'eV') & (dfrm['Bandgap'] > 0) & (dfrm['Bandgap'] < 20))
+dfrm = dfrm.filter((dfrm['Units'] == 'eV') & (dfrm['Bandgap'] > 0) & (dfrm['Bandgap'] < 20) & dfrm['Elements'].is_not_null())
 
 # group by 'Formula' and calculate mean Bandgap
 avgbgfrm = dfrm.group_by('Formula').agg(

kickoff/miner_bondlength.py

@@ -9,18 +9,21 @@
 import polars as pl
 
 from mpds_client import MPDSDataRetrieval, MPDSExport
+from ase.neighborlist import NeighborList
 
 
 def calculate_lengths(ase_obj, elA, elB, limit=4):
     assert elA != elB
+    nlist = NeighborList([limit / 2] * len(ase_obj), self_interaction=False, bothways=True)
+    nlist.update(ase_obj)
     lengths = []
     for n, atom in enumerate(ase_obj):
         if atom.symbol == elA:
-            for m, neighbor in enumerate(ase_obj):
-                if neighbor.symbol == elB:
-                    dist = round(ase_obj.get_distance(n, m), 2) # NB occurrence <-> rounding
-                    if dist < limit:
-                        lengths.append(dist)
+            indices, _ = nlist.get_neighbors(n)
+            for m in indices:
+                if ase_obj[m].symbol == elB:
+                    dist = round(ase_obj.get_distance(n,m), 2)
+                    lengths.append(dist)
     return lengths
 
 client = MPDSDataRetrieval()

kickoff/miner_cmp_ab_pr_data.py

@@ -11,6 +11,7 @@
 from mpds_client import MPDSDataRetrieval, MPDSDataTypes
 
 
+
 result_cache = 'mpds_cmp_ab_pr.pkl'
 
 MILLIEV_TO_INVCM = 8.06554
@@ -256,6 +257,8 @@ def get_ab_pr_values(
         'sample.material.phase_id',
         'sample.measurement[0].property.scalar'
     ]}):
+        if not deck:
+            continue
         if ab_prop_massage:
             deck = ab_prop_massage(deck)
             if not deck:
@@ -281,6 +284,8 @@ def get_ab_pr_values(
         'sample.measurement[0].condition[0].name',
         'sample.measurement[0].condition[0].scalar'
     ]}):
+        if not deck:
+            continue
         if pr_prop_massage:
             deck = pr_prop_massage(deck)
             if not deck:

kickoff/miner_liquidus.py

@@ -17,7 +17,8 @@
 plt.switch_backend('agg')
 from mpds_client import MPDSDataRetrieval
 
-from miner_nonformers import pd_svg_to_points
+from miner_nonformers import pl_svg_to_points
+
 
 
 MARGIN_EDGES_COMP = 0.1
@@ -40,7 +41,7 @@
     plt.annotate(elb, xy=(1.05, -0.1), xycoords='axes fraction')
     ymin, ymax = 500, 700
 
-    for pd in api_client.get_data({"props": "phase diagram", "classes": "binary", "elements": "-".join(elements)}, fields={}): # fields={} means all fields
+    for pd in api_client.get_data({"props": "phase diagram", "classes": "binary", "elements": elements}, fields={}): # fields={} means all fields
         # Consider only full-composition diagrams
         if pd['comp_range'] != [0, 100]:
             continue
@@ -68,7 +69,7 @@
                 done_liquidus = True
                 liquidus_line = []
 
-                for point in pd_svg_to_points(area['svgpath']):
+                for point in pl_svg_to_points(area['svgpath']):
 
                     # NB the line out of polygon extraction algorithm must be improved;
                     # this is just a quick and dirty example based on

kickoff/miner_nonformers.py

@@ -18,6 +18,7 @@
 from mpds_client import MPDSDataRetrieval
 
 
+
 # Within this composition tolerance (%), a phase near a pure element
 # will be considered as unary (not a binary) compound
 ELEMENT_TOL = 15

kickoff/miner_pb_ratio.py

@@ -8,6 +8,7 @@
 from mpds_client import MPDSDataRetrieval
 
 
+
 supported_arities = {1: 'unary', 2: 'binary', 3: 'ternary', 4: 'quaternary', 5: 'quinary'}
 mpds_api = MPDSDataRetrieval()
 
@@ -23,7 +24,7 @@ def get_cell_v_for_t(elements, t0=250, t1=350):
     """
     phases_volumes = {}
 
-    for item in mpds_api.get_data(dict(elements='-'.join(elements), classes=supported_arities[len(elements)]), fields={
+    for item in mpds_api.get_data(dict(elements=elements, classes=supported_arities[len(elements)]), fields={
     'P': [
         lambda: 'P',
         'sample.material.phase_id',

kickoff/miner_propstruct.py

@@ -14,6 +14,7 @@
 from ase.data import covalent_radii, chemical_symbols
 from mpds_client import MPDSDataRetrieval
 
+
 def get_APF(ase_obj):
     """
     Example crystal structure descriptor:
@@ -51,6 +52,8 @@ def get_Wiener(ase_obj):
 descriptors = []
 
 for item in answer:
+    if not item:
+        continue
     crystal = MPDSDataRetrieval.compile_crystal(item, 'ase')
     if not crystal:
         continue

kickoff/miner_twofold_props.py

@@ -11,6 +11,7 @@
 import numpy as np
 from mpds_client import MPDSDataRetrieval, MPDSDataTypes
 
+
 mpds_api = MPDSDataRetrieval(dtype=MPDSDataTypes.MACHINE_LEARNING) # NB MPDSDataTypes.ALL
 
 phase_for_formula = {}
@@ -21,6 +22,8 @@
     'sample.material.chemical_formula',
     'sample.measurement[0].property.scalar'
 ]}):
+    if not deck:
+        continue
     if deck[2] > (1800 + 273):
         phase_for_formula[deck[0]] = deck[1]
         phase_for_val_a.setdefault(deck[0], []).append(deck[2]) # why list? each phase might have > 1 value
@@ -30,6 +33,8 @@
     # we don't need *chemical_formula* now, since we have phase_id's
     'sample.measurement[0].property.scalar'
 ]}):
+    if not deck:
+        continue
     phase_for_val_b.setdefault(deck[0], []).append(deck[1] * 1E5) # why list? each phase might have > 1 value
 
 # now we just re-group and show the results (but we can do much more!)
@@ -43,4 +48,4 @@
     )
 
 for item in sorted(results, key=lambda x: x[2]):
-    print("%s T_melt = %.0f C \t alpha^E5 = %.2f" % tuple(item))
+    print("%s T_melt = %.0f C \t alpha^E5 = %.2f" % tuple(item))

notebooks/2_mpds_basic.ipynb

@@ -79,7 +79,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from mpds_client import MPDSDataRetrieval, MPDSDataTypes, APIError"
+     "from mpds_client import MPDSDataRetrieval, MPDSDataTypes, APIError"
    ]
   },
   {
@@ -181,7 +181,7 @@
     "    print(\"Considering %s\" % prop)\n",
     "\n",
     "    try:\n",
-    "        for card in client.get_data({\n",
+     "        for card in client.get_data({\n",
     "            \"props\": prop,\n",
     "            # we defined our props above\n",
     "\n",
@@ -197,7 +197,7 @@
     "\n",
     "            \"years\": \"2010-2019\"\n",
     "            # only recent results (void for MACHINE_LEARNING, as all are 2018)\n",
-    "        }, fields=desired_fields):\n",
+    "        }), fields=desired_fields):\n",
     "\n",
     "            print(\"%s %s %s\" % (card[0], \"-\".join(card[2]), card[3]))\n",
     "\n",
@@ -218,7 +218,7 @@
    "source": [
     "client.dtype = MPDSDataTypes.PEER_REVIEWED\n",
     "\n",
-    "print(client.get_data({\"elements\": \"O\", \"classes\": \"binary\", \"sgs\": \"I4/mmm\"}))"
+     "print(client.get_data({\"elements\": \"O\", \"classes\": \"binary\", \"sgs\": \"I4/mmm\"}))\n"
    ]
   },
   {
@@ -230,7 +230,7 @@
     "import random\n",
     "prop = random.choice(example_props)\n",
     "\n",
-    "print(client.get_data({\"props\": prop, \"elements\": \"O\", \"classes\": \"binary, lanthanoid, non-disordered\"}))"
+     "print(client.get_data({\"props\": prop, \"elements\": \"O\", \"classes\": \"binary, lanthanoid, non-disordered\"}))\n"
    ]
   },
   {
@@ -253,4 +253,4 @@
  "metadata": {},
  "nbformat": 4,
  "nbformat_minor": 2
-}
+}

notebooks/4_eos_fit.ipynb

@@ -145,7 +145,7 @@
     "from numpy.linalg import det\n",
     "from ase.geometry import cellpar_to_cell\n",
     "\n",
-    "from mpds_client import MPDSDataRetrieval"
+     "from mpds_client import MPDSDataRetrieval"
    ]
   },
   {
@@ -164,11 +164,11 @@
    "source": [
     "client = MPDSDataRetrieval()\n",
     "\n",
-    "dfrm_k0p = client.get_dataframe({\"classes\": \"binary\", \"elements\": \"O\", \"props\": \"pressure derivative of isothermal bulk modulus\"})\n",
+     "dfrm_k0p = client.get_dataframe({\"classes\": \"binary\", \"elements\": \"O\", \"props\": \"pressure derivative of isothermal bulk modulus\"})\n",
     "dfrm_k0p = dfrm_k0p[np.isfinite(dfrm_k0p['Phase'])] # only data for the existing distinct phases\n",
     "avg_k0p =  dfrm_k0p.groupby('Phase')['Value'].median().to_frame().reset_index().rename(columns={'Value': 'avg_k0p'})\n",
     "\n",
-    "dfrm_k0 = client.get_dataframe({\"props\": \"isothermal bulk modulus\"}, phases=set(dfrm_k0p['Phase'].tolist()))\n",
+     "dfrm_k0 = client.get_dataframe({\"props\": \"isothermal bulk modulus\"}, phases=set(dfrm_k0p['Phase'].tolist()))\n",
     "avg_k0 =  dfrm_k0.groupby('Phase')['Value'].median().to_frame().reset_index().rename(columns={'Value': 'avg_k0'})\n",
     "avg_k0 =  avg_k0.merge(avg_k0p, how='inner', on='Phase')"
    ]
@@ -206,8 +206,8 @@
    "source": [
     "pvts = {}\n",
     "\n",
-    "for matrix in client.get_data(\n",
-    "    {\"props\": \"cell parameters - pressure diagram\"},\n",
+     "for matrix in client.get_data(\n",
+     "    {\"props\": \"cell parameters - pressure diagram\"},\n",
     "    phases=set(dfrm_k0p['Phase'].tolist()), # only those phases we have experimental bulk modulus\n",
     "    fields={}                               # all fields\n",
     "):\n",
@@ -279,4 +279,4 @@
  "metadata": {},
  "nbformat": 4,
  "nbformat_minor": 2
-}
+}

scripts/validate.py

@@ -23,7 +23,7 @@
 
 try:
     validate(target["out"], schema)
-except ValidationError, e:
+except ValidationError as e:
     raise RuntimeError(
         "The item: \r\n\r\n %s \r\n\r\n has an issue: \r\n\r\n %s" % (
             e.instance, e.context