Merge pull request #63 from felix5572/devel

amcadmus · web-flow · commit 8ef6758b3db0 · 2019-11-29T16:12:02.000+08:00
support MultiSystems from_dir &amp;&amp; add  more test cases for quip/gap/xyz
diff --git a/README.md b/README.md
@@ -51,16 +51,24 @@ The labels provided in the `OUTCAR`, i.e. energies, forces and virials (if any),
 
 The `System` or `LabeledSystem` can be constructed from the following file formats with the `format key` in the table passed to argument `fmt`:
 
-For `quip/gap xyz` files, single .xyz file may include many different configurations with different atom numbers and atom type.
+The Class `dpdata.MultiSystems`  can read data  from a dir which may contains many files of different systems, or from single xyz file which contains different systems.
 
-The Class `dpdata.MultiSystems`  can read data  this kind of file.
-Now Class dpdata.MultiSystems Only support quip/gap xyz  format file.
+Use `dpdata.MultiSystems.from_dir` to read from a  directory, `dpdata.MultiSystems` will walk in the directory 
+Recursively  and  find all file with specific file_name. Supports all the file formats that `dpdata.LabeledSystem` supports.
+
+Use  `dpdata.MultiSystems.from_file` to read from single file. Now only support quip/gap/xyz  format file.
+
+For example, for `quip/gap xyz` files, single .xyz file may contain many different configurations with different atom numbers and atom type.
 
 The following commands relating to `Class dpdata.MultiSystems` may be useful.
 ```python
 # load data
 
-xyz_multi_systems = dpdata.MultiSystems.from_file('tests/xyz/xyz_unittest.xyz','quip/gap/xyz')
+xyz_multi_systems = dpdata.MultiSystems.from_file(file_name='tests/xyz/xyz_unittest.xyz',fmt='quip/gap/xyz')
+vasp_multi_systems = dpdata.MultiSystems.from_dir(dir_name='./mgal_outcar', file_name='OUTCAR', fmt='vasp/outcar')
+
+# use wildcard
+vasp_multi_systems = dpdata.MultiSystems.from_dir(dir_name='./mgal_outcar', file_name='*OUTCAR', fmt='vasp/outcar')
 
 # print the multi_system infomation
 print(xyz_multi_systems)
diff --git a/dpdata/system.py b/dpdata/system.py
@@ -1,5 +1,6 @@
 #%%
 import os
+import glob
 import numpy as np
 import dpdata.lammps.lmp
 import dpdata.lammps.dump
@@ -950,7 +951,16 @@ def from_file(cls,file_name,fmt):
         multi_systems = cls()
         multi_systems.load_systems_from_file(file_name=file_name,fmt=fmt)
         return multi_systems
-    
+
+    @classmethod
+    def from_dir(cls,dir_name, file_name, fmt='auto'):
+        multi_systems = cls()
+        target_file_list = glob.glob('./{}/**/{}'.format(dir_name, file_name), recursive=True)
+        for target_file in target_file_list:
+            multi_systems.append(LabeledSystem(file_name=target_file, fmt=fmt))
+        return multi_systems
+
+
     def load_systems_from_file(self, file_name=None, fmt=None):
         if file_name is not None:
             if fmt is None:
@@ -1015,10 +1025,10 @@ def check_atom_names(self, system):
             system.add_atom_names(new_in_self)
         system.sort_atom_names()
 
-    def from_quip_gap_xyz_file(self,filename):
-        # quip_gap_xyz_systems = QuipGapxyzSystems(filename)
+    def from_quip_gap_xyz_file(self,file_name):
+        # quip_gap_xyz_systems = QuipGapxyzSystems(file_name)
         # print(next(quip_gap_xyz_systems))
-        for info_dict in QuipGapxyzSystems(filename):
+        for info_dict in QuipGapxyzSystems(file_name):
             system=LabeledSystem(data=info_dict)
             self.append(system)
 
diff --git a/dpdata/xyz/quip_gap_xyz.py b/dpdata/xyz/quip_gap_xyz.py
@@ -1,9 +1,4 @@
 #!/usr/bin/env python3
-
-#%%
-# with open('./test.xyz', 'r') as xyz_file:
-#     lines = xyz_file.readlines()
-#     print(lines)
 #%% 
 import numpy as np
 from collections import OrderedDict
@@ -44,20 +39,16 @@ def handle_single_xyz_frame(lines):
         if len(lines) != atom_num + 2:
             raise RuntimeError("format error, atom_num=={}, {}!=atom_num+2".format(atom_num, len(lines)))
         data_format_line = lines[1].strip('\n').strip()+str(' ')
-        p1 = re.compile(r'(?P<key>\S+)=(?P<quote>[\'\"]?)(?P<value>.*?)(?P=quote)\s+')
-        p2 = re.compile(r'(?P<key>\w+?):(?P<datatype>[a-zA-Z]):(?P<value>\d+)')
-        field_list = [kv_dict.groupdict() for kv_dict in p1.finditer(data_format_line)]
-        field_dict = {}
-        for item in field_list:
-            field_dict[item['key']]=item['value']
-        data_format_line = lines[1]
-        data_format_list= [m.groupdict() for m in p1.finditer(data_format_line)]
+        field_value_pattern= re.compile(r'(?P<key>\S+)=(?P<quote>[\'\"]?)(?P<value>.*?)(?P=quote)\s+')
+        prop_pattern = re.compile(r'(?P<key>\w+?):(?P<datatype>[a-zA-Z]):(?P<value>\d+)')
+
+        data_format_list= [kv_dict.groupdict() for kv_dict in field_value_pattern.finditer(data_format_line)]
         field_dict = {}
         for item in data_format_list:
             field_dict[item['key']]=item['value']
 
         Properties = field_dict['Properties']
-        prop_list = [m.groupdict() for m in p2.finditer(Properties)]
+        prop_list = [kv_dict.groupdict() for kv_dict in prop_pattern.finditer(Properties)]
 
         data_lines = []
         for line in lines[2:]:
@@ -127,6 +118,7 @@ def handle_single_xyz_frame(lines):
             virials = np.array([np.array(list(filter(bool,field_dict['virial'].split(' ')))).reshape(3,3)]).astype('float32')
         else:
             virials = None
+
         info_dict = {}
         info_dict['atom_names'] = list(type_num_array[:,0])
         info_dict['atom_numbs'] = list(type_num_array[:,1].astype(int))
diff --git a/tests/test_quip_gap_xyz.py b/tests/test_quip_gap_xyz.py
@@ -4,7 +4,7 @@
 from context import dpdata
 from comp_sys import CompLabeledSys
 
-class TestQuipGapxyz(unittest.TestCase, CompLabeledSys):
+class TestQuipGapxyz1(unittest.TestCase, CompLabeledSys):
     def setUp (self) :
         self.multi_systems = dpdata.MultiSystems.from_file('xyz/xyz_unittest.xyz','quip/gap/xyz')
         self.system_1 = self.multi_systems.systems['B1C9']
@@ -17,7 +17,7 @@ def setUp (self) :
 class TestQuipGapxyz2(unittest.TestCase, CompLabeledSys):
     def setUp (self) :
         self.system_temp0 = dpdata.MultiSystems.from_file(file_name='xyz/xyz_unittest.xyz', fmt='quip/gap/xyz')
-        self.system_1 = self.system_temp0.systems['B5C7']
+        self.system_1 = self.system_temp0.systems['B5C7'] # .sort_atom_types()
         self.system_temp1 = dpdata.LabeledSystem('xyz/B1C9', fmt='deepmd')
         self.system_temp2 = dpdata.LabeledSystem('xyz/B5C7', fmt='deepmd')
         self.system_temp3 = dpdata.MultiSystems(self.system_temp2, self.system_temp1)
@@ -27,5 +27,54 @@ def setUp (self) :
         self.f_places = 6
         self.v_places = 4
 
+class TestQuipGapxyzsort1(unittest.TestCase, CompLabeledSys):
+    def setUp (self) :
+        self.multi_systems_1 = dpdata.MultiSystems.from_file('xyz/xyz_unittest.sort.xyz','quip/gap/xyz')
+        self.system_1 = self.multi_systems_1.systems['B5C7']
+        self.system_1.sort_atom_types()
+        self.multi_systems_2 = dpdata.MultiSystems.from_file('xyz/xyz_unittest.xyz','quip/gap/xyz')
+        self.system_2 = self.multi_systems_2.systems['B5C7']
+        self.places = 6
+        self.e_places = 6
+        self.f_places = 6
+        self.v_places = 4
+
+class TestQuipGapxyzsort2(unittest.TestCase, CompLabeledSys):
+    def setUp (self) :
+        self.multi_systems_1 = dpdata.MultiSystems.from_file('xyz/xyz_unittest.sort.xyz','quip/gap/xyz')
+        self.system_1 = self.multi_systems_1.systems['B1C9']
+        self.system_1.sort_atom_types()
+        self.multi_systems_2 = dpdata.MultiSystems.from_file('xyz/xyz_unittest.xyz','quip/gap/xyz')
+        self.system_2 = self.multi_systems_2.systems['B1C9']
+        self.places = 6
+        self.e_places = 6
+        self.f_places = 6
+        self.v_places = 4
+
+class TestQuipGapxyzfield(unittest.TestCase, CompLabeledSys):
+    def setUp (self) :
+        self.multi_systems_1 = dpdata.MultiSystems.from_file('xyz/xyz_unittest.field.xyz','quip/gap/xyz')
+        self.system_1 = self.multi_systems_1.systems['B1C9']
+        self.system_1.sort_atom_types()
+        self.multi_systems_2 = dpdata.MultiSystems.from_file('xyz/xyz_unittest.xyz','quip/gap/xyz')
+        self.system_2 = self.multi_systems_2.systems['B1C9']
+        self.places = 6
+        self.e_places = 6
+        self.f_places = 6
+        self.v_places = 4
+
+class TestQuipGapxyzfield2(unittest.TestCase, CompLabeledSys):
+    def setUp (self) :
+        self.multi_systems_1 = dpdata.MultiSystems.from_file('xyz/xyz_unittest.field.xyz','quip/gap/xyz')
+        self.system_1 = self.multi_systems_1.systems['B5C7']
+        self.system_1.sort_atom_types()
+        self.multi_systems_2 = dpdata.MultiSystems.from_file('xyz/xyz_unittest.xyz','quip/gap/xyz')
+        self.system_2 = self.multi_systems_2.systems['B5C7']
+        self.places = 6
+        self.e_places = 6
+        self.f_places = 6
+        self.v_places = 4
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests/xyz/xyz_unittest.field.xyz b/tests/xyz/xyz_unittest.field.xyz
@@ -0,0 +1,41 @@
+# change field structure,table structure add quote and blank line
+12
+energy=0.2397023e+01    virial="159.582203018    1.23282341824e-05    0.187835167689    1.23282341824e-05    164.488614479    9.04070506711e-06    0.187835167689    9.04070506711e-06    300.596373102"    Lattice="2.405388711   0.0   0.0   6.67e-07   8.830761667   0.0   2.145883783   6.3e-08   2.066179013"     Properties=force:R:3:pos:R:3:species:S:1:Z:I:1
+ -7.25565900000  -8.82919400000   4.68743200000   3.19567000000   0.97965000000   1.52250000000  B  5 
+   7.25565700000  -8.82919400000   -4.68743200000  1.35561000000   0.97965000000   0.54368000000  B  5 
+    7.25565900000   8.82919400000   -4.68743200000  1.35561000000   7.85111000000   0.54368000000  B  5 
+    -7.25565700000   8.82919400000   4.68743200000  3.19567000000   7.85111000000   1.52250000000  B  5 
+ -0.00000000000  0   +0.0   2.27564000000   4.41538000000   1.03309000000  B 5  
+   -14.39490800000 -19.12936600000   -4.67857200000 1.95905000000   6.23505000000   1.53936000000 C  6 
+    14.39490500000 -19.12936600000  4.67857200000   2.59222000000   6.23505000000   0.52682000000 C  6
+   14.39490800000  19.12936600000  4.67857200000     2.59222000000   2.59571000000   0.52682000000 C  6  
+  -14.39490500000  19.12936600000   -4.67857200000    1.95905000000   2.59571000000   1.53936000000 C  6   
+   -0.00000000000  -4.51935800000  0   2.27564000000   1.56002000000   1.03309000000 C  6    
+    0.00000000000   4.51935800000  0   2.27564000000   7.27074000000   1.03309000000 C  6   
+    -0.00000000000  -0.00000000000 0    2.27564000000   0.00000000000   1.03309000000 C  6  
+ 
+
+  10   
+virial=" -3.68535041825    1.63204257089e-06    -4.28008468355    1.63204257089e-06    18.1630123797    0.0    -4.28008468355    0.0    3.03073243091 "    Lattice=" 9.217080809   0.0   0.0   4.86e-07   6.431267224   0.0   4.201562981   4.1e-08   2.205334915 "  Properties='species:S:1:pos:R:3:Z:I:1:force:R:3 ' energy='-58.342497      '
+B   12.38023000000   3.21563000000   2.13103000000   5    -9.35551000000  -0.00000000000  -0.22364600000
+C   12.96783000000   4.32879000000   2.14172000000   6     7.05653600000  11.19171000000   1.80087100000
+C   12.96783000000   2.10247000000   2.14172000000   6     7.05653500000 -11.19171000000   1.80087100000
+C    7.95424000000   1.03482000000   2.10290000000   6     1.71010500000  -3.80357800000  -0.12402100000
+C    7.95424000000   5.39645000000   2.10290000000   6     1.71010600000   3.80357800000  -0.12402100000
+C    0.48970000000   5.37868000000   0.03690000000   6     0.43541500000   0.75339500000  -0.62086800000
+C    0.48970000000   1.05259000000   0.03690000000   6     0.43541500000  -0.75339500000  -0.62086800000
+C    2.71900000000   1.71197000000   0.17210000000   6    -3.43041200000   0.05665700000  -0.44035700000
+C    2.71900000000   4.71930000000   0.17210000000   6    -3.43041200000  -0.05665700000  -0.44035700000
+C    9.69981000000   3.21563000000   0.55395000000   6    -2.18778000000  -0.00000000000  -1.00760300000
+10
+energy=-56.397425    virial="-5.61261501333    -6.34809444383e-07    6.34809444383e-07    -6.34809444383e-07    -5.61261501333    6.34809444383e-07    6.34809444383e-07    6.34809444383e-07    4.91130356636"    Lattice="5.512073672   0.0   0.0   4.16e-07   5.512073672   0.0   2.53e-07   2.53e-07   3.347159197"     Properties=species:S:1:pos:R:3:Z:I:1:force:R:3
+B    4.08581000000   4.57623000000   2.94238000000   5     0.03989300000  -1.87257900000   0.97202400000
+C    1.42627000000   0.93584000000   2.94238000000   5    -0.03989300000   1.87257900000   0.97202400000
+C    0.93584000000   4.08581000000   2.94238000000   5     1.87257900000   0.03989300000   0.97202400000
+C    4.57623000000   1.42627000000   2.94238000000   5    -1.87257900000  -0.03989300000   0.97202400000
+C    2.75604000000   2.75604000000   0.32076000000   5    -0.00000000000  -0.00000000000  -1.59707500000
+C    2.94936000000   1.19711000000   3.32090000000   6     0.74133300000  -2.43841400000  -2.33370300000
+C    2.56271000000   4.31496000000   3.32090000000   6    -0.74133400000   2.43841400000  -2.33370300000
+C    4.31496000000   2.94936000000   3.32090000000   6     2.43841400000   0.74133400000  -2.33370300000
+C    1.19711000000   2.56271000000   3.32090000000   6    -2.43841400000  -0.74133400000  -2.33370300000
+C    2.75604000000   2.75604000000   1.72073000000   6     0.00000100000   0.00000100000   7.04379200000
diff --git a/tests/xyz/xyz_unittest.sort.xyz b/tests/xyz/xyz_unittest.sort.xyz
@@ -0,0 +1,38 @@
+12
+energy=2.397023    virial="159.582203018    1.23282341824e-05    0.187835167689    1.23282341824e-05    164.488614479    9.04070506711e-06    0.187835167689    9.04070506711e-06    300.596373102"    Lattice="2.405388711   0.0   0.0   6.67e-07   8.830761667   0.0   2.145883783   6.3e-08   2.066179013"     Properties=species:S:1:pos:R:3:Z:I:1:force:R:3
+C    1.95905000000   6.23505000000   1.53936000000   6   -14.39490800000 -19.12936600000  -4.67857200000
+B    3.19567000000   0.97965000000   1.52250000000   5    -7.25565900000  -8.82919400000   4.68743200000
+B    1.35561000000   0.97965000000   0.54368000000   5     7.25565700000  -8.82919400000  -4.68743200000
+B    1.35561000000   7.85111000000   0.54368000000   5     7.25565900000   8.82919400000  -4.68743200000
+C    2.59222000000   6.23505000000   0.52682000000   6    14.39490500000 -19.12936600000   4.67857200000
+C    2.59222000000   2.59571000000   0.52682000000   6    14.39490800000  19.12936600000   4.67857200000
+C    1.95905000000   2.59571000000   1.53936000000   6   -14.39490500000  19.12936600000  -4.67857200000
+C    2.27564000000   1.56002000000   1.03309000000   6    -0.00000000000  -4.51935800000   0.00000000000
+C    2.27564000000   7.27074000000   1.03309000000   6     0.00000000000   4.51935800000   0.00000000000
+B    3.19567000000   7.85111000000   1.52250000000   5    -7.25565700000   8.82919400000   4.68743200000
+C    2.27564000000   0.00000000000   1.03309000000   6    -0.00000000000  -0.00000000000   0.00000000000
+B    2.27564000000   4.41538000000   1.03309000000   5    -0.00000000000  -0.00000000000   0.00000000000
+10
+energy=-58.342497    virial="-3.68535041825    1.63204257089e-06    -4.28008468355    1.63204257089e-06    18.1630123797    0.0    -4.28008468355    0.0    3.03073243091"    Lattice="9.217080809   0.0   0.0   4.86e-07   6.431267224   0.0   4.201562981   4.1e-08   2.205334915"     Properties=species:S:1:pos:R:3:Z:I:1:force:R:3
+C   12.96783000000   4.32879000000   2.14172000000   6     7.05653600000  11.19171000000   1.80087100000
+C   12.96783000000   2.10247000000   2.14172000000   6     7.05653500000 -11.19171000000   1.80087100000
+C    7.95424000000   1.03482000000   2.10290000000   6     1.71010500000  -3.80357800000  -0.12402100000
+C    7.95424000000   5.39645000000   2.10290000000   6     1.71010600000   3.80357800000  -0.12402100000
+B   12.38023000000   3.21563000000   2.13103000000   5    -9.35551000000  -0.00000000000  -0.22364600000
+C    0.48970000000   5.37868000000   0.03690000000   6     0.43541500000   0.75339500000  -0.62086800000
+C    0.48970000000   1.05259000000   0.03690000000   6     0.43541500000  -0.75339500000  -0.62086800000
+C    2.71900000000   1.71197000000   0.17210000000   6    -3.43041200000   0.05665700000  -0.44035700000
+C    2.71900000000   4.71930000000   0.17210000000   6    -3.43041200000  -0.05665700000  -0.44035700000
+C    9.69981000000   3.21563000000   0.55395000000   6    -2.18778000000  -0.00000000000  -1.00760300000
+10
+energy=-56.397425    virial="-5.61261501333    -6.34809444383e-07    6.34809444383e-07    -6.34809444383e-07    -5.61261501333    6.34809444383e-07    6.34809444383e-07    6.34809444383e-07    4.91130356636"    Lattice="5.512073672   0.0   0.0   4.16e-07   5.512073672   0.0   2.53e-07   2.53e-07   3.347159197"     Properties=species:S:1:pos:R:3:Z:I:1:force:R:3
+B    4.08581000000   4.57623000000   2.94238000000   5     0.03989300000  -1.87257900000   0.97202400000
+C    1.42627000000   0.93584000000   2.94238000000   5    -0.03989300000   1.87257900000   0.97202400000
+C    0.93584000000   4.08581000000   2.94238000000   5     1.87257900000   0.03989300000   0.97202400000
+C    4.57623000000   1.42627000000   2.94238000000   5    -1.87257900000  -0.03989300000   0.97202400000
+C    2.75604000000   2.75604000000   0.32076000000   5    -0.00000000000  -0.00000000000  -1.59707500000
+C    2.94936000000   1.19711000000   3.32090000000   6     0.74133300000  -2.43841400000  -2.33370300000
+C    2.56271000000   4.31496000000   3.32090000000   6    -0.74133400000   2.43841400000  -2.33370300000
+C    4.31496000000   2.94936000000   3.32090000000   6     2.43841400000   0.74133400000  -2.33370300000
+C    1.19711000000   2.56271000000   3.32090000000   6    -2.43841400000  -0.74133400000  -2.33370300000
+C    2.75604000000   2.75604000000   1.72073000000   6     0.00000100000   0.00000100000   7.04379200000
