import os,json import sqlite3 from io import BytesIO,StringIO import uuid import tempfile from datetime import datetime import mimetypes mimetypes.add_type("video/webm" ,'.mkv') mimetypes.add_type("audio/flac",".flac") DEFAULT_BLOCK_SIZE = 1024*1024*8 class FileS: def __init__(self, meta, conn): self.size = meta['length'] self.create = meta['created'] self.modified = meta['modified'] self.mimetype = meta['mimetype'] self.encoding = meta['encoding'] self.parts = json.loads(meta['parts']) self.conn = conn self.position = 0 self.buffer = [b'', -1, -1] def read(self, size=-1): if size < 0: size = self.size - self.position data = b'' while size > 0: if size < DEFAULT_BLOCK_SIZE and self.buffer[1] <= self.position < self.buffer[2]: chunk = self.buffer[0] start = self.position - self.buffer[1] end = min(start + size, self.buffer[2] - self.buffer[1]) data += chunk[start:end] size -= end - start self.position += end - start else: part = self._get_next_part() if not part: break cur = self.conn.cursor() cur.execute('SELECT data FROM datas WHERE uuid=?', (part['uuid'],)) chunk = cur.fetchone()[0] if size >= DEFAULT_BLOCK_SIZE: start = self.position % DEFAULT_BLOCK_SIZE end = start + DEFAULT_BLOCK_SIZE data += chunk[start:end] size -= end - start self.position += end - start else: chunk_start = self.position // DEFAULT_BLOCK_SIZE * DEFAULT_BLOCK_SIZE chunk_end = min(chunk_start + DEFAULT_BLOCK_SIZE, part['end']) chunk_pos_start = chunk_start - part['start'] chunk_pos_end = chunk_end - part['start'] self.buffer = [chunk[chunk_pos_start:chunk_pos_end], chunk_start, chunk_end] start = self.position - chunk_start end = min(start + size, chunk_end - chunk_start) data += chunk[chunk_pos_start+start:chunk_pos_start+end] size -= end - start self.position += end - start return data def _get_next_part(self): for part in self.parts: if self.position < part['end'] and self.position >= part['start']: return part return None def seek(self, position): self.position = position self.buffer = [b'', -1, -1] def tell(self): return self.position class FileSQL3: def __init__(self, db_path): self.conn = sqlite3.connect(db_path,check_same_thread=False) self.conn.row_factory = sqlite3.Row self._init_tables() def _init_tables(self): cur = self.conn.cursor() cur.execute('''CREATE TABLE IF NOT EXISTS files ( path TEXT PRIMARY KEY, created TEXT, modified TEXT, length INTEGER, encoding TEXT, mimetype TEXT, description TEXT, parts TEXT)''') cur.execute('''CREATE TABLE IF NOT EXISTS datas ( uuid TEXT PRIMARY KEY, data BLOB, path TEXT, start INTEGER, end INTEGER)''') self.conn.commit() def get(self, file_path): cur = self.conn.cursor() cur.execute('SELECT * FROM files WHERE path=?', (file_path,)) meta = cur.fetchone() if meta: return FileS(dict(meta), self.conn) def putBytes(self, b,p_path,**kws): f=tempfile.NamedTemporaryFile(delete=False) f.write(b) f.close() self.put(f.name,p_path=p_path,**kws) os.unlink(f.name) def put(self, file_path, p_path=None, description=None, block_size=DEFAULT_BLOCK_SIZE): if not p_path: p_path = file_path with open(file_path, "rb") as f: file_size = os.path.getsize(file_path) file_created = datetime.fromtimestamp(os.path.getctime(file_path)).isoformat() file_modified = datetime.fromtimestamp(os.path.getmtime(file_path)).isoformat() parts = [] start = 0 while start < file_size: end = min(start + block_size, file_size) data = f.read(block_size) data_uuid = str(uuid.uuid4()) parts.append({'uuid': data_uuid, 'start': start, 'end': end}) cur = self.conn.cursor() cur.execute('INSERT INTO datas (uuid, data, path, start, end) VALUES (?, ?, ?, ?, ?)', (data_uuid, data, p_path, start, end)) start = end parts_json = json.dumps(parts) try: cur = self.conn.cursor() mt, ec = mimetypes.guess_type(file_path) cur.execute('''INSERT INTO files (path, created, modified, length, encoding, mimetype, description, parts) VALUES (?, ?, ?, ?, ?, ?, ?, ?)''', (p_path, file_created, file_modified, file_size, ec, mt, description, parts_json)) except sqlite3.IntegrityError: cur.execute('DELETE FROM files WHERE path=?', (p_path,)) cur.execute('''INSERT INTO files (path, created, modified, length, encoding, mimetype, description, parts) VALUES (?, ?, ?, ?, ?, ?, ?, ?)''', (p_path, file_created, file_modified, file_size, ec, mt, description, parts_json)) self.conn.commit() def update_files_table(self, path, **fields): cur = self.conn.cursor() query = "UPDATE files SET " query += ', '.join([f"{k} = ?" for k in fields.keys()]) query += " WHERE path = ?" cur.execute(query, (*fields.values(), path)) self.conn.commit() def search(self, search_string): cur = self.conn.cursor() cur.execute('SELECT path FROM files WHERE path LIKE ?', (search_string ,)) return [row['path'] for row in cur.fetchall()] def delete(self, file_path): cur = self.conn.cursor() cur.execute('SELECT parts FROM files WHERE path=?', (file_path,)) parts = cur.fetchone() if parts: for part in json.loads(parts['parts']): cur.execute('DELETE FROM datas WHERE uuid=?', (part['uuid'],)) cur.execute('DELETE FROM files WHERE path=?', (file_path,)) self.conn.commit() """ q=FileSQL3("imgs.sql3") for p in q.search("%"): f=q.get(p) tg=open("imgs/"+p,"wb") tg.write(f.read()) tg.close() """