You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
338 lines
14 KiB
338 lines
14 KiB
#!/usr/bin/env python |
|
|
|
import argparse |
|
import errno |
|
import hashlib |
|
import logging |
|
import mimetypes |
|
import os |
|
import urlparse |
|
import random |
|
import shutil |
|
import ssl |
|
import string |
|
import sys |
|
import time |
|
import yaml |
|
|
|
from sleekxmpp.componentxmpp import ComponentXMPP |
|
from threading import Event |
|
from threading import Lock |
|
from threading import Thread |
|
|
|
try: |
|
# Python 3 |
|
from http.server import HTTPServer, BaseHTTPRequestHandler |
|
from socketserver import ThreadingMixIn |
|
except ImportError: |
|
# Python 2 |
|
from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler |
|
from SocketServer import ThreadingMixIn |
|
|
|
try: |
|
FileNotFoundError |
|
except NameError: |
|
# Python 2 |
|
class FileNotFoundError(IOError): |
|
def __init__(self, message=None, *args): |
|
super(FileNotFoundError, self).__init__(args) |
|
self.message = message |
|
self.errno = errno.ENOENT |
|
|
|
def __str__(self): |
|
return self.message or os.strerror(self.errno) |
|
|
|
|
|
LOGLEVEL=logging.DEBUG |
|
|
|
global files |
|
global files_lock |
|
global config |
|
global quotas |
|
|
|
def normalize_path(path, sub_url_length): |
|
""" |
|
Normalizes the URL to prevent users from grabbing arbitrary files via `../' |
|
and the like. |
|
""" |
|
return os.path.normcase(os.path.normpath(path))[sub_url_length:] |
|
|
|
def expire(quotaonly=False, kill_event=None): |
|
""" |
|
Expire all files over 'user_quota_soft' and older than 'expire_maxage' |
|
|
|
- quotaonly - If true don't delete anything just calculate the |
|
used space per user and return. Otherwise make an exiry run |
|
every config['expire_interval'] seconds. |
|
- kill_event - threading.Event to listen to. When set, quit to |
|
prevent hanging on KeyboardInterrupt. Only applicable when |
|
quotaonly = False |
|
""" |
|
global config |
|
global quotas |
|
|
|
while True: |
|
if not quotaonly: |
|
# Wait expire_interval secs or return on kill_event |
|
if kill_event.wait(config['expire_interval']): |
|
return |
|
|
|
now = time.time() |
|
# Scan each senders upload directories seperatly |
|
for sender in os.listdir(config['storage_path']): |
|
senderdir = os.path.join(config['storage_path'], sender) |
|
quota = 0 |
|
filelist = [] |
|
# Traverse sender directory, delete anything older expire_maxage and collect file stats. |
|
for dirname, dirs, files in os.walk(senderdir, topdown=False): |
|
removed = [] |
|
for name in files: |
|
fullname = os.path.join(dirname, name) |
|
stats = os.stat(fullname) |
|
if not quotaonly: |
|
if now - stats.st_mtime > config['expire_maxage']: |
|
logging.debug('Expiring %s. Age: %s', fullname, now - stats.st_mtime) |
|
try: |
|
os.unlink(fullname) |
|
removed += [name] |
|
except OSError as e: |
|
logging.warning("Exception '%s' deleting file '%s'.", e, fullname) |
|
quota += stats.st_size |
|
filelist += [(stats.st_mtime, fullname, stats.st_size)] |
|
else: |
|
quota += stats.st_size |
|
filelist += [(stats.st_mtime, fullname, stats.st_size)] |
|
if dirs == [] and removed == files: # Directory is empty, so we can remove it |
|
logging.debug('Removing directory %s.', dirname) |
|
try: |
|
os.rmdir(dirname) |
|
except OSError as e: |
|
logging.warning("Exception '%s' deleting directory '%s'.", e, dirname) |
|
|
|
if not quotaonly and config['user_quota_soft']: |
|
# Delete oldest files of sender until occupied space is <= user_quota_soft |
|
filelist.sort() |
|
while quota > config['user_quota_soft']: |
|
entry = filelist[0] |
|
try: |
|
logging.debug('user_quota_soft exceeded. Removing %s. Age: %s', entry[1], now - entry[0]) |
|
os.unlink(entry[1]) |
|
quota -= entry[2] |
|
except OSError as e: |
|
logging.warning("Exception '%s' deleting file '%s'.", e, entry[1]) |
|
filelist.pop(0) |
|
quotas[sender] = quota |
|
|
|
logging.debug('Expire run finished in %fs', time.time() - now) |
|
|
|
if quotaonly: |
|
return |
|
|
|
|
|
class MissingComponent(ComponentXMPP): |
|
def __init__(self, jid, secret, port): |
|
ComponentXMPP.__init__(self, jid, secret, "localhost", port) |
|
self.register_plugin('xep_0030') |
|
self.register_plugin('upload',module='plugins.upload') |
|
self.add_event_handler('request_upload_slot',self.request_upload_slot) |
|
|
|
def request_upload_slot(self, iq): |
|
global config |
|
global files |
|
global files_lock |
|
request = iq['request'] |
|
maxfilesize = int(config['max_file_size']) |
|
if not request['filename'] or not request['size']: |
|
self._sendError(iq,'modify','bad-request','please specify filename and size') |
|
elif maxfilesize < int(request['size']): |
|
self._sendError(iq,'modify','not-acceptable','file too large. max file size is '+str(maxfilesize)) |
|
elif 'whitelist' not in config or iq['from'].domain in config['whitelist'] or iq['from'].bare in config['whitelist']: |
|
sender = iq['from'].bare |
|
sender_hash = hashlib.sha1(sender.encode()).hexdigest() |
|
if config['user_quota_hard'] and quotas.setdefault(sender_hash, 0) + int(request['size']) > config['user_quota_hard']: |
|
msg = 'quota would be exceeded. max file size is %d' % (config['user_quota_hard'] - quotas[sender_hash]) |
|
logging.debug(msg) |
|
self._sendError(iq, 'modify', 'not-acceptable', msg) |
|
return |
|
filename = request['filename'] |
|
folder = ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(len(sender_hash))) |
|
sane_filename = "".join([c for c in filename if c.isalpha() or c.isdigit() or c=="."]).rstrip() |
|
path = os.path.join(sender_hash, folder) |
|
if sane_filename: |
|
path = os.path.join(path, sane_filename) |
|
with files_lock: |
|
files.add(path) |
|
print(path) |
|
reply = iq.reply() |
|
reply['slot']['get'] = urlparse.urljoin(config['get_url'], path) |
|
reply['slot']['put'] = urlparse.urljoin(config['put_url'], path) |
|
reply.send() |
|
else: |
|
self._sendError(iq,'cancel','not-allowed','not allowed to request upload slots') |
|
|
|
def _sendError(self, iq, error_type, condition, text): |
|
reply = iq.reply() |
|
iq.error() |
|
iq['error']['type'] = error_type |
|
iq['error']['condition'] = condition |
|
iq['error']['text'] = text |
|
iq.send() |
|
|
|
class HttpHandler(BaseHTTPRequestHandler): |
|
def do_PUT(self): |
|
print('do put') |
|
global files |
|
global files_lock |
|
global config |
|
path = normalize_path(self.path, config['put_sub_url_len']) |
|
length = int(self.headers['Content-Length']) |
|
maxfilesize = int(config['max_file_size']) |
|
if config['user_quota_hard']: |
|
sender_hash = path.split('/')[0] |
|
maxfilesize = min(maxfilesize, config['user_quota_hard'] - quotas.setdefault(sender_hash, 0)) |
|
if maxfilesize < length: |
|
self.send_response(400,'file too large') |
|
self.end_headers() |
|
else: |
|
print('path: '+path) |
|
files_lock.acquire() |
|
if path in files: |
|
files.remove(path) |
|
files_lock.release() |
|
filename = os.path.join(config['storage_path'], path) |
|
os.makedirs(os.path.dirname(filename)) |
|
remaining = length |
|
with open(filename,'wb') as f: |
|
data = self.rfile.read(min(4096,remaining)) |
|
while data and remaining >= 0: |
|
databytes = len(data) |
|
remaining -= databytes |
|
if config['user_quota_hard']: |
|
quotas[sender_hash] += databytes |
|
f.write(data) |
|
data = self.rfile.read(min(4096,remaining)) |
|
self.send_response(200,'ok') |
|
self.end_headers() |
|
else: |
|
files_lock.release() |
|
self.send_response(403,'invalid slot') |
|
self.end_headers() |
|
|
|
def do_GET(self, body=True): |
|
global config |
|
path = normalize_path(self.path, config['get_sub_url_len']) |
|
slashcount = path.count('/') |
|
if path[0] in ('/', '\\') or slashcount < 1 or slashcount > 2: |
|
self.send_response(404,'file not found') |
|
self.end_headers() |
|
else: |
|
filename = os.path.join(config['storage_path'], path) |
|
print('requesting file: '+filename) |
|
try: |
|
with open(filename,'rb') as f: |
|
self.send_response(200) |
|
mime, _ = mimetypes.guess_type(filename) |
|
if mime is None: |
|
mime = 'application/octet-stream' |
|
self.send_header("Content-Type", mime) |
|
if mime[:6] != 'image/': |
|
self.send_header("Content-Disposition", 'attachment; filename="{}"'.format(os.path.basename(filename))) |
|
fs = os.fstat(f.fileno()) |
|
self.send_header("Content-Length", str(fs.st_size)) |
|
self.end_headers() |
|
if body: |
|
shutil.copyfileobj(f, self.wfile) |
|
except FileNotFoundError: |
|
self.send_response(404,'file not found') |
|
self.end_headers() |
|
|
|
def do_HEAD(self): |
|
self.do_GET(body=False) |
|
|
|
|
|
class ThreadedHTTPServer(ThreadingMixIn, HTTPServer): |
|
"""Handle requests in a separate thread.""" |
|
|
|
if __name__ == "__main__": |
|
parser = argparse.ArgumentParser() |
|
parser.add_argument("-c", "--config", default='config.yml', help='Specify alternate config file.') |
|
parser.add_argument("-l", "--logfile", default=None, help='File where the server log will be stored. If not specified log to stdout.') |
|
args = parser.parse_args() |
|
|
|
with open(args.config,'r') as ymlfile: |
|
config = yaml.load(ymlfile) |
|
|
|
files = set() |
|
files_lock = Lock() |
|
kill_event = Event() |
|
logging.basicConfig(level=LOGLEVEL, |
|
format='%(asctime)-24s %(levelname)-8s %(message)s', |
|
filename=args.logfile) |
|
|
|
if not config['get_url'].endswith('/'): |
|
config['get_url'] = config['get_url'] + '/' |
|
if not config['put_url'].endswith('/'): |
|
config['put_url'] = config['put_url'] + '/' |
|
|
|
try: |
|
config['get_sub_url_len'] = len(urlparse.urlparse(config['get_url']).path) |
|
config['put_sub_url_len'] = len(urlparse.urlparse(config['put_url']).path) |
|
except ValueError: |
|
logging.warning("Invalid get_sub_url ('%s') or put_sub_url ('%s'). sub_url's disabled.", config['get_sub_url'], config['put_sub_url']) |
|
config['get_sub_url_int'] = 1 |
|
config['put_sub_url_int'] = 1 |
|
|
|
# Sanitize config['user_quota_*'] and calculate initial quotas |
|
quotas = {} |
|
try: |
|
config['user_quota_hard'] = int(config.get('user_quota_hard', 0)) |
|
config['user_quota_soft'] = int(config.get('user_quota_soft', 0)) |
|
if config['user_quota_soft'] or config['user_quota_hard']: |
|
expire(quotaonly=True) |
|
except ValueError: |
|
logging.warning("Invalid user_quota_hard ('%s') or user_quota_soft ('%s'). Quotas disabled.", config['user_quota_soft'], config['user_quota_soft']) |
|
config['user_quota_soft'] = 0 |
|
config['user_quota_hard'] = 0 |
|
|
|
# Sanitize config['expire_*'] and start expiry thread |
|
try: |
|
config['expire_interval'] = float(config.get('expire_interval', 0)) |
|
config['expire_maxage'] = float(config.get('expire_maxage', 0)) |
|
if config['expire_interval'] > 0 and (config['user_quota_soft'] or config['expire_maxage']): |
|
t = Thread(target=expire, kwargs={'kill_event': kill_event}) |
|
t.start() |
|
else: |
|
logging.info('Expiring disabled.') |
|
except ValueError: |
|
logging.warning("Invalid expire_interval ('%s') or expire_maxage ('%s') set in config file. Expiring disabled.", |
|
config['expire_interval'], config['expire_maxage']) |
|
|
|
try: |
|
server = ThreadedHTTPServer((config['http_address'], config['http_port']), HttpHandler) |
|
except Exception as e: |
|
import traceback |
|
logging.debug(traceback.format_exc()) |
|
kill_event.set() |
|
sys.exit(1) |
|
|
|
if 'http_keyfile' in config and 'http_certfile' in config: |
|
server.socket = ssl.wrap_socket(server.socket, keyfile=config['http_keyfile'], certfile=config['http_certfile']) |
|
jid = config['component_jid'] |
|
secret = config['component_secret'] |
|
port = int(config.get('component_port',5347)) |
|
xmpp = MissingComponent(jid,secret,port) |
|
if xmpp.connect(): |
|
xmpp.process() |
|
print("connected") |
|
try: |
|
server.serve_forever() |
|
except (KeyboardInterrupt, Exception) as e: |
|
if e == KeyboardInterrupt: |
|
logging.debug('Ctrl+C pressed') |
|
else: |
|
import traceback |
|
logging.debug(traceback.format_exc()) |
|
kill_event.set() |
|
else: |
|
print("unable to connect") |
|
kill_event.set()
|
|
|