forked from axiom-team/astrXbian
256 lines
8.2 KiB
Python
256 lines
8.2 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""
|
|
|
|
Copyright (C) 2014-2016 bromix (plugin.video.youtube)
|
|
Copyright (C) 2016-2019 plugin.video.youtube
|
|
|
|
SPDX-License-Identifier: GPL-2.0-only
|
|
See LICENSES/GPL-2.0-only for more information.
|
|
"""
|
|
|
|
from six import PY2
|
|
from six.moves import range
|
|
# noinspection PyPep8Naming
|
|
from six.moves import cPickle as pickle
|
|
|
|
import datetime
|
|
import os
|
|
import sqlite3
|
|
import time
|
|
import traceback
|
|
|
|
from .. import logger
|
|
|
|
|
|
class Storage(object):
|
|
def __init__(self, filename, max_item_count=0, max_file_size_kb=-1):
|
|
self._table_name = 'storage'
|
|
self._filename = filename
|
|
if not self._filename.endswith('.sqlite'):
|
|
self._filename = ''.join([self._filename, '.sqlite'])
|
|
self._file = None
|
|
self._cursor = None
|
|
self._max_item_count = max_item_count
|
|
self._max_file_size_kb = max_file_size_kb
|
|
|
|
self._table_created = False
|
|
self._needs_commit = False
|
|
|
|
def set_max_item_count(self, max_item_count):
|
|
self._max_item_count = max_item_count
|
|
|
|
def set_max_file_size_kb(self, max_file_size_kb):
|
|
self._max_file_size_kb = max_file_size_kb
|
|
|
|
def __del__(self):
|
|
self._close()
|
|
|
|
def _open(self):
|
|
if self._file is None:
|
|
self._optimize_file_size()
|
|
|
|
path = os.path.dirname(self._filename)
|
|
if not os.path.exists(path):
|
|
os.makedirs(path)
|
|
|
|
self._file = sqlite3.connect(self._filename, check_same_thread=False,
|
|
detect_types=0, timeout=1)
|
|
|
|
self._file.isolation_level = None
|
|
self._cursor = self._file.cursor()
|
|
self._cursor.execute('PRAGMA journal_mode=MEMORY')
|
|
self._cursor.execute('PRAGMA busy_timeout=20000')
|
|
# self._cursor.execute('PRAGMA synchronous=OFF')
|
|
self._create_table()
|
|
|
|
def _execute(self, needs_commit, query, values=None):
|
|
if values is None:
|
|
values = []
|
|
if not self._needs_commit and needs_commit:
|
|
self._needs_commit = True
|
|
self._cursor.execute('BEGIN')
|
|
|
|
"""
|
|
Tests revealed that sqlite has problems to release the database in time. This happens no so often, but just to
|
|
be sure, we try at least 3 times to execute out statement.
|
|
"""
|
|
for tries in range(3):
|
|
try:
|
|
return self._cursor.execute(query, values)
|
|
except TypeError:
|
|
return None
|
|
except:
|
|
time.sleep(0.1)
|
|
else:
|
|
return None
|
|
|
|
def _close(self):
|
|
if self._file is not None:
|
|
self.sync()
|
|
self._file.commit()
|
|
self._cursor.close()
|
|
self._cursor = None
|
|
self._file.close()
|
|
self._file = None
|
|
|
|
def _optimize_file_size(self):
|
|
# do nothing - only we have given a size
|
|
if self._max_file_size_kb <= 0:
|
|
return
|
|
|
|
# do nothing - only if this folder exists
|
|
path = os.path.dirname(self._filename)
|
|
if not os.path.exists(path):
|
|
return
|
|
|
|
if not os.path.exists(self._filename):
|
|
return
|
|
|
|
try:
|
|
file_size_kb = (os.path.getsize(self._filename) // 1024)
|
|
if file_size_kb >= self._max_file_size_kb:
|
|
os.remove(self._filename)
|
|
except OSError:
|
|
pass
|
|
|
|
def _create_table(self):
|
|
self._open()
|
|
if not self._table_created:
|
|
query = 'CREATE TABLE IF NOT EXISTS %s (key TEXT PRIMARY KEY, time TIMESTAMP, value BLOB)' % self._table_name
|
|
self._execute(True, query)
|
|
self._table_created = True
|
|
|
|
def sync(self):
|
|
if self._cursor is not None and self._needs_commit:
|
|
self._needs_commit = False
|
|
return self._execute(False, 'COMMIT')
|
|
|
|
def _set(self, item_id, item):
|
|
def _encode(obj):
|
|
return sqlite3.Binary(pickle.dumps(obj, protocol=pickle.HIGHEST_PROTOCOL))
|
|
|
|
if self._max_file_size_kb < 1 and self._max_item_count < 1:
|
|
self._optimize_item_count()
|
|
else:
|
|
self._open()
|
|
now = datetime.datetime.now() + datetime.timedelta(microseconds=1) # add 1 microsecond, required for dbapi2
|
|
query = 'REPLACE INTO %s (key,time,value) VALUES(?,?,?)' % self._table_name
|
|
self._execute(True, query, values=[item_id, now, _encode(item)])
|
|
self._close()
|
|
self._optimize_item_count()
|
|
|
|
def _optimize_item_count(self):
|
|
if self._max_item_count < 1:
|
|
if not self._is_empty():
|
|
self._clear()
|
|
else:
|
|
self._open()
|
|
query = 'SELECT key FROM %s ORDER BY time DESC LIMIT -1 OFFSET %d' % (self._table_name, self._max_item_count)
|
|
result = self._execute(False, query)
|
|
if result is not None:
|
|
for item in result:
|
|
self._remove(item[0])
|
|
self._close()
|
|
|
|
def _clear(self):
|
|
self._open()
|
|
query = 'DELETE FROM %s' % self._table_name
|
|
self._execute(True, query)
|
|
self._create_table()
|
|
self._close()
|
|
self._open()
|
|
self._execute(False, 'VACUUM')
|
|
self._close()
|
|
|
|
def _is_empty(self):
|
|
self._open()
|
|
query = 'SELECT exists(SELECT 1 FROM %s LIMIT 1);' % self._table_name
|
|
result = self._execute(False, query)
|
|
is_empty = True
|
|
if result is not None:
|
|
for item in result:
|
|
is_empty = item[0] == 0
|
|
break
|
|
self._close()
|
|
return is_empty
|
|
|
|
def _get_ids(self, oldest_first=True):
|
|
self._open()
|
|
# self.sync()
|
|
query = 'SELECT key FROM %s' % self._table_name
|
|
if oldest_first:
|
|
query = '%s ORDER BY time ASC' % query
|
|
else:
|
|
query = '%s ORDER BY time DESC' % query
|
|
|
|
query_result = self._execute(False, query)
|
|
|
|
result = []
|
|
if query_result:
|
|
for item in query_result:
|
|
result.append(item[0])
|
|
|
|
self._close()
|
|
return result
|
|
|
|
def _get(self, item_id):
|
|
def _decode(obj):
|
|
if PY2:
|
|
obj = str(obj)
|
|
return pickle.loads(obj)
|
|
|
|
self._open()
|
|
query = 'SELECT time, value FROM %s WHERE key=?' % self._table_name
|
|
result = self._execute(False, query, [item_id])
|
|
if result is None:
|
|
self._close()
|
|
return None
|
|
|
|
item = result.fetchone()
|
|
if item is None:
|
|
self._close()
|
|
return None
|
|
|
|
self._close()
|
|
return _decode(item[1]), item[0]
|
|
|
|
def _remove(self, item_id):
|
|
self._open()
|
|
query = 'DELETE FROM %s WHERE key = ?' % self._table_name
|
|
self._execute(True, query, [item_id])
|
|
|
|
@staticmethod
|
|
def strptime(stamp, stamp_fmt):
|
|
# noinspection PyUnresolvedReferences
|
|
import _strptime
|
|
try:
|
|
time.strptime('01 01 2012', '%d %m %Y') # dummy call
|
|
except:
|
|
pass
|
|
return time.strptime(stamp, stamp_fmt)
|
|
|
|
def get_seconds_diff(self, current_stamp):
|
|
stamp_format = '%Y-%m-%d %H:%M:%S.%f'
|
|
current_datetime = datetime.datetime.now()
|
|
if not current_stamp:
|
|
return 86400 # 24 hrs
|
|
try:
|
|
stamp_datetime = datetime.datetime(*(self.strptime(current_stamp, stamp_format)[0:6]))
|
|
except ValueError: # current_stamp has no microseconds
|
|
stamp_format = '%Y-%m-%d %H:%M:%S'
|
|
stamp_datetime = datetime.datetime(*(self.strptime(current_stamp, stamp_format)[0:6]))
|
|
except TypeError:
|
|
logger.log_error('Exception while calculating timestamp difference: '
|
|
'current_stamp |{cs}|{cst}| stamp_format |{sf}|{sft}| \n{tb}'
|
|
.format(cs=current_stamp, cst=type(current_stamp),
|
|
sf=stamp_format, sft=type(stamp_format),
|
|
tb=traceback.print_exc())
|
|
)
|
|
return 604800 # one week
|
|
|
|
time_delta = current_datetime - stamp_datetime
|
|
total_seconds = 0
|
|
if time_delta:
|
|
total_seconds = ((time_delta.seconds + time_delta.days * 24 * 3600) * 10 ** 6) // (10 ** 6)
|
|
return total_seconds
|