Corey Morris
commited on
Commit
•
f228d38
1
Parent(s):
74822dd
moved methods to better match flow
Browse files
details_data_processor.py
CHANGED
@@ -16,7 +16,14 @@ class DetailsDataProcessor:
|
|
16 |
# self.data = self.process_data()
|
17 |
# self.ranked_data = self.rank_data()
|
18 |
|
19 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
20 |
|
21 |
# download a file from a single url and save it to a local directory
|
22 |
@staticmethod
|
@@ -52,14 +59,7 @@ class DetailsDataProcessor:
|
|
52 |
return constructed_url
|
53 |
|
54 |
|
55 |
-
|
56 |
-
matching_files = [] # List to hold matching filenames
|
57 |
-
for root, dirs, files in os.walk(directory):
|
58 |
-
for basename in files:
|
59 |
-
if fnmatch.fnmatch(basename, pattern):
|
60 |
-
filename = os.path.join(root, basename)
|
61 |
-
matching_files.append(filename) # Append the matching filename to the list
|
62 |
-
return matching_files # Return the list of matching filenames
|
63 |
|
64 |
|
65 |
def pipeline(self):
|
|
|
16 |
# self.data = self.process_data()
|
17 |
# self.ranked_data = self.rank_data()
|
18 |
|
19 |
+
def _find_files(self, directory='results', pattern='results*.json'):
|
20 |
+
matching_files = [] # List to hold matching filenames
|
21 |
+
for root, dirs, files in os.walk(directory):
|
22 |
+
for basename in files:
|
23 |
+
if fnmatch.fnmatch(basename, pattern):
|
24 |
+
filename = os.path.join(root, basename)
|
25 |
+
matching_files.append(filename) # Append the matching filename to the list
|
26 |
+
return matching_files # Return the list of matching filenames
|
27 |
|
28 |
# download a file from a single url and save it to a local directory
|
29 |
@staticmethod
|
|
|
59 |
return constructed_url
|
60 |
|
61 |
|
62 |
+
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
63 |
|
64 |
|
65 |
def pipeline(self):
|