From 990398bdcc6570ed152811cc86f7334126c7247b Mon Sep 17 00:00:00 2001 From: James Ravenscroft Date: Sat, 22 Oct 2022 11:56:50 +0100 Subject: [PATCH] separate out functionality into modules --- poetry.lock | 49 +++++++-- pyproject.toml | 1 + src/microcosm/__init__.py | 165 +++++++---------------------- src/microcosm/__main__.py | 4 +- src/microcosm/authroute.py | 0 src/microcosm/backends/__init__.py | 0 src/microcosm/exception.py | 0 src/microcosm/micropub.py | 87 +++++++++++++++ src/microcosm/webmentions.py | 122 +++++++++++++++++++++ 9 files changed, 295 insertions(+), 133 deletions(-) create mode 100644 src/microcosm/authroute.py create mode 100644 src/microcosm/backends/__init__.py create mode 100644 src/microcosm/exception.py create mode 100644 src/microcosm/micropub.py create mode 100644 src/microcosm/webmentions.py diff --git a/poetry.lock b/poetry.lock index 91ba0d0..6b584ce 100644 --- a/poetry.lock +++ b/poetry.lock @@ -32,6 +32,21 @@ python-versions = "*" pycodestyle = ">=2.8.0" toml = "*" +[[package]] +name = "beautifulsoup4" +version = "4.11.1" +description = "Screen-scraping library" +category = "main" +optional = false +python-versions = ">=3.6.0" + +[package.dependencies] +soupsieve = ">1.2" + +[package.extras] +html5lib = ["html5lib"] +lxml = ["lxml"] + [[package]] name = "certifi" version = "2021.10.8" @@ -89,6 +104,19 @@ Werkzeug = ">=2.0" async = ["asgiref (>=3.2)"] dotenv = ["python-dotenv"] +[[package]] +name = "flask-micropub" +version = "0.2.8" +description = "Adds support for Micropub clients." +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +BeautifulSoup4 = "*" +Flask = "*" +requests = "*" + [[package]] name = "giteapy" version = "1.0.8" @@ -96,7 +124,6 @@ description = "" category = "main" optional = false python-versions = "*" -develop = false [package.dependencies] certifi = ">=2017.4.17" @@ -105,11 +132,8 @@ six = ">=1.10" urllib3 = ">=1.23" [package.source] -type = "git" -url = "https://github.com/dblueai/giteapy.git" -reference = "master" -resolved_reference = "001e9b66795a6d34146c8532e9d8e648d5b93e59" - +type = "url" +url = "https://github.com/dblueai/giteapy/archive/master.zip" [[package]] name = "idna" version = "3.3" @@ -348,6 +372,14 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +[[package]] +name = "soupsieve" +version = "2.3.2.post1" +description = "A modern CSS selector implementation for Beautiful Soup." +category = "main" +optional = false +python-versions = ">=3.6" + [[package]] name = "text-unidecode" version = "1.3" @@ -411,7 +443,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest- [metadata] lock-version = "1.1" python-versions = "^3.7.1" -content-hash = "ea26a2a723bb4541c447d3cdab6d4c16dd941875e55d770300a134f39a21adc4" +content-hash = "eeb044e8389c93739d7e81f1ef28a7c25779bef280d6309c7996a8d6c590bd4c" [metadata.files] atomicwrites = [ @@ -426,6 +458,7 @@ autopep8 = [ {file = "autopep8-1.6.0-py2.py3-none-any.whl", hash = "sha256:ed77137193bbac52d029a52c59bec1b0629b5a186c495f1eb21b126ac466083f"}, {file = "autopep8-1.6.0.tar.gz", hash = "sha256:44f0932855039d2c15c4510d6df665e4730f2b8582704fa48f9c55bd3e17d979"}, ] +beautifulsoup4 = [] certifi = [ {file = "certifi-2021.10.8-py2.py3-none-any.whl", hash = "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"}, {file = "certifi-2021.10.8.tar.gz", hash = "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872"}, @@ -446,6 +479,7 @@ flask = [ {file = "Flask-2.0.2-py3-none-any.whl", hash = "sha256:cb90f62f1d8e4dc4621f52106613488b5ba826b2e1e10a33eac92f723093ab6a"}, {file = "Flask-2.0.2.tar.gz", hash = "sha256:7b2fb8e934ddd50731893bdcdb00fc8c0315916f9fcd50d22c7cc1a95ab634e2"}, ] +flask-micropub = [] giteapy = [] idna = [ {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"}, @@ -625,6 +659,7 @@ six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] +soupsieve = [] text-unidecode = [ {file = "text-unidecode-1.3.tar.gz", hash = "sha256:bad6603bb14d279193107714b288be206cac565dfa49aa5b105294dd5c4aab93"}, {file = "text_unidecode-1.3-py2.py3-none-any.whl", hash = "sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8"}, diff --git a/pyproject.toml b/pyproject.toml index b0dbcf6..16234a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ requests = "^2.27.1" python-dotenv = "^0.19.2" python-slugify = "^5.0.2" PyYAML = "^6.0" +Flask-Micropub = "^0.2.8" [tool.poetry.dev-dependencies] pytest = "^6.2.5" diff --git a/src/microcosm/__init__.py b/src/microcosm/__init__.py index a38cd1a..b95240a 100644 --- a/src/microcosm/__init__.py +++ b/src/microcosm/__init__.py @@ -6,11 +6,9 @@ import dotenv import giteapy import giteapy.rest import time -import json import base64 from werkzeug.datastructures import FileStorage import yaml -import hashlib from urllib.parse import urlparse @@ -18,16 +16,13 @@ from slugify import slugify from datetime import date, datetime from xml.etree import ElementTree -from flask import Flask, jsonify, request, url_for, Response -from requests import api +from flask import Flask, jsonify, request, Response, Blueprint dotenv.load_dotenv() PERMITTED_DOMAIN = os.environ.get( 'PERMITTED_DOMAINS', 'https://brainsteam.co.uk/').split(';') -app = Flask(__name__) -app.config['SECRET_KEY'] = 'my super secret key' ENTITY_TYPE_PLURAL_MAP = { @@ -35,6 +30,28 @@ ENTITY_TYPE_PLURAL_MAP = { "watch":"watches" } +core_bp = Blueprint("core", __name__) + + +def create_app(): + app = Flask(__name__) + app.config['SECRET_KEY'] = 'my super secret key' + + app.config.from_file(os.path.join(os.getcwd(), "config.yaml"), yaml.safe_load) + + from .micropub import micropub, auth_bp + from .webmentions import webhook_bp + + print(app.config) + + micropub.init_app(app, app.config['INDIEAUTH']['client_id']) + + app.register_blueprint(auth_bp) + app.register_blueprint(core_bp) + app.register_blueprint(webhook_bp) + + return app + def authed_endpoint(f): @functools.wraps(f) @@ -62,16 +79,7 @@ def authed_endpoint(f): _api_client = None -def get_api_client() -> giteapy.RepositoryApi: - global _api_client - if _api_client is None: - config = giteapy.Configuration() - config.host = os.environ.get('GITEA_URL') - config.api_key['access_token'] = os.environ.get('GITEA_API_KEY') - _api_client = giteapy.RepositoryApi(giteapy.ApiClient(config)) - - return _api_client def process_photo_url(now: datetime, doc: Dict[str, List[str]], suffix: str = ""): """Process photo submitted via URL""" @@ -173,112 +181,6 @@ def init_frontmatter(now: datetime, post_type: str, name: Optional[str]=None): return frontmatter, file_path -@app.route("/webmentions", methods=['POST']) -def webmention_hook(): - """Accept web mention webhook request""" - - - body = request.get_json() - - print(f"Incoming webmention {body}") - - # webmention should always have a json body - if body is None: - return {"error":"invalid_request"}, 400 - - # assert that secret matches - if body.get('secret') != os.environ.get("WEBMENTION_SECRET", "changeme"): - return {"error":"invalid_secret"}, 401 - - # get existing mentions from gitea - - api = get_api_client() - - old_sha = None - - - try: - mentions_meta = api.repo_get_contents(os.environ.get('GITEA_REPO_OWNER'), - os.environ.get('GITEA_REPO_NAME'), os.environ.get('WEBMENTIONS_JSON_FILE')) - - old_sha = mentions_meta.sha - - mentions = json.loads(base64.decodebytes(mentions_meta.content.encode('utf8'))) - - except giteapy.rest.ApiException as e: - if e.status == 404: - print("no mentions yet, create new mentions file") - mentions = {} - - # parse target url and get path on server - url_path = urlparse(body.get('target')).path - - # create mention entry if needed - if url_path not in mentions: - mentions[url_path] = [] - - # if the mention is already processed, do nothing. - for entry in mentions[url_path]: - if entry['source'] == body['source']: - return {"message": "mention already processed, no action taken."} - - activity_types = { - "in-reply-to": "reply", - "like-of": "like", - "repost-of": "repost", - "bookmark-of": "bookmark", - "mention-of": "mention", - "rsvp":"rsvp" - } - - # format the mention like the data from webmention.io api - new_mention = { - "id": body['post']['wm-id'], - "source": body['source'], - "target": body['target'], - "activity":{ - "type": activity_types[body['post']['wm-property']] - }, - "verified_date": body.get('wm-received', datetime.now().isoformat()), - "data":{ - "author": body['post']['author'], - "content": body['post'].get('content',{}).get('html', None), - "published": body['post'].get('published') - } - } - - # append the new mention - mentions[url_path].append(new_mention) - - content = base64.encodestring(json.dumps(mentions, indent=2).encode("utf8")).decode("utf8") - - # store to repo - if old_sha: - print(f"Update {os.environ.get('WEBMENTIONS_JSON_FILE')}") - body = giteapy.UpdateFileOptions(content=content, sha=old_sha) - - - try: - r = api.repo_update_file(os.environ.get( - 'GITEA_REPO_OWNER'), os.environ.get('GITEA_REPO_NAME'), os.environ.get('WEBMENTIONS_JSON_FILE'), body) - - return {"message":"ok"} - - except Exception as e: - return {"error": str(e)}, 500 - - else: - print(f"Create {os.environ.get('WEBMENTIONS_JSON_FILE')}") - body = giteapy.CreateFileOptions(content=content) - - try: - r = api.repo_create_file(os.environ.get( - 'GITEA_REPO_OWNER'), os.environ.get('GITEA_REPO_NAME'), os.environ.get('WEBMENTIONS_JSON_FILE'), body) - - return {"message":"ok"} - - except Exception as e: - return {"error": str(e)}, 500 def detect_entry_type(doc: dict) -> str: """Given a dictionary object from either form or json, detect type of post""" @@ -450,10 +352,19 @@ def process_json_post(): return docstr, frontmatter, file_path +def get_api_client() -> giteapy.RepositoryApi: + global _api_client + + if _api_client is None: + config = giteapy.Configuration() + config.host = os.environ.get('GITEA_URL') + config.api_key['access_token'] = os.environ.get('GITEA_API_KEY') + _api_client = giteapy.RepositoryApi(giteapy.ApiClient(config)) + + return _api_client - -@app.route('/', methods=['POST']) +@core_bp.route('/', methods=['POST']) @authed_endpoint def req(): @@ -513,7 +424,7 @@ def get_syndication_targets(): return defs -@app.route("/media", methods=["POST"]) +@core_bp.route("/media", methods=["POST"]) @authed_endpoint def media_endpoint(): @@ -548,12 +459,16 @@ def generate_config_json(): { "type": "bookmark", "name": "Bookmark" + }, + { + "type": "like", + "name":"Like" } ] } -@app.route("/", methods=['GET']) +@core_bp.route("/", methods=['GET']) @authed_endpoint def index(): diff --git a/src/microcosm/__main__.py b/src/microcosm/__main__.py index 95886e4..f5bceb8 100644 --- a/src/microcosm/__main__.py +++ b/src/microcosm/__main__.py @@ -1,4 +1,6 @@ -from . import app +from venv import create +from . import create_app +app = create_app() app.run(debug=True) \ No newline at end of file diff --git a/src/microcosm/authroute.py b/src/microcosm/authroute.py new file mode 100644 index 0000000..e69de29 diff --git a/src/microcosm/backends/__init__.py b/src/microcosm/backends/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/microcosm/exception.py b/src/microcosm/exception.py new file mode 100644 index 0000000..e69de29 diff --git a/src/microcosm/micropub.py b/src/microcosm/micropub.py new file mode 100644 index 0000000..934489e --- /dev/null +++ b/src/microcosm/micropub.py @@ -0,0 +1,87 @@ + +from flask_micropub import MicropubClient + +from flask import current_app, request, url_for, Blueprint + +micropub = MicropubClient() + +auth_bp = Blueprint("token", __name__) + + + +@auth_bp.route('/form', methods=['GET']) +def authform(): + return """ + + + +
+ + +
+
+ + + +
+ + + """ + + +@auth_bp.route('/authenticate') +def authenticate(): + return micropub.authenticate( + request.args.get('me'), next_url=url_for('index')) + + +@auth_bp.route('/authorize') +def authorize(): + return micropub.authorize( + request.args.get('me'), next_url=url_for('index'), + scope=request.args.get('scope')) + + +@auth_bp.route('/indieauth-callback') +@micropub.authenticated_handler +def indieauth_callback(resp): + return """ + + + + Authenticated: + + + + """.format(resp.me, resp.next_url, resp.error) + + +@auth_bp.route('/micropub-callback') +@micropub.authorized_handler +def micropub_callback(resp): + + return """ + + + + Authorized: + + + + """.format(resp.me, resp.micropub_endpoint, resp.access_token, + resp.next_url, resp.error) diff --git a/src/microcosm/webmentions.py b/src/microcosm/webmentions.py new file mode 100644 index 0000000..ad4726b --- /dev/null +++ b/src/microcosm/webmentions.py @@ -0,0 +1,122 @@ +import json +import base64 +import giteapy +import os + +from flask import request, Blueprint + +from urllib.parse import urlparse +from datetime import datetime + +webhook_bp = Blueprint("webmention", __name__) + + + + +@webhook_bp.route("/webmentions", methods=['POST']) +def webmention_hook(): + """Accept web mention webhook request""" + + + body = request.get_json() + + print(f"Incoming webmention {body}") + + # webmention should always have a json body + if body is None: + return {"error":"invalid_request"}, 400 + + # assert that secret matches + if body.get('secret') != os.environ.get("WEBMENTION_SECRET", "changeme"): + return {"error":"invalid_secret"}, 401 + + # get existing mentions from gitea + from microcosm import get_api_client + + api = get_api_client() + + old_sha = None + + + try: + mentions_meta = api.repo_get_contents(os.environ.get('GITEA_REPO_OWNER'), + os.environ.get('GITEA_REPO_NAME'), os.environ.get('WEBMENTIONS_JSON_FILE')) + + old_sha = mentions_meta.sha + + mentions = json.loads(base64.decodebytes(mentions_meta.content.encode('utf8'))) + + except giteapy.rest.ApiException as e: + if e.status == 404: + print("no mentions yet, create new mentions file") + mentions = {} + + # parse target url and get path on server + url_path = urlparse(body.get('target')).path + + # create mention entry if needed + if url_path not in mentions: + mentions[url_path] = [] + + # if the mention is already processed, do nothing. + for entry in mentions[url_path]: + if entry['source'] == body['source']: + return {"message": "mention already processed, no action taken."} + + activity_types = { + "in-reply-to": "reply", + "like-of": "like", + "repost-of": "repost", + "bookmark-of": "bookmark", + "mention-of": "mention", + "rsvp":"rsvp" + } + + # format the mention like the data from webmention.io api + new_mention = { + "id": body['post']['wm-id'], + "source": body['source'], + "target": body['target'], + "activity":{ + "type": activity_types[body['post']['wm-property']] + }, + "verified_date": body.get('wm-received', datetime.now().isoformat()), + "data":{ + "author": body['post']['author'], + "content": body['post'].get('content',{}).get('html', None), + "published": body['post'].get('published') + } + } + + # append the new mention + mentions[url_path].append(new_mention) + + content = base64.encodestring(json.dumps(mentions, indent=2).encode("utf8")).decode("utf8") + + # store to repo + if old_sha: + print(f"Update {os.environ.get('WEBMENTIONS_JSON_FILE')}") + body = giteapy.UpdateFileOptions(content=content, sha=old_sha) + + + try: + r = api.repo_update_file(os.environ.get( + 'GITEA_REPO_OWNER'), os.environ.get('GITEA_REPO_NAME'), os.environ.get('WEBMENTIONS_JSON_FILE'), body) + + return {"message":"ok"} + + except Exception as e: + return {"error": str(e)}, 500 + + else: + print(f"Create {os.environ.get('WEBMENTIONS_JSON_FILE')}") + body = giteapy.CreateFileOptions(content=content) + + try: + r = api.repo_create_file(os.environ.get( + 'GITEA_REPO_OWNER'), os.environ.get('GITEA_REPO_NAME'), os.environ.get('WEBMENTIONS_JSON_FILE'), body) + + return {"message":"ok"} + + except Exception as e: + return {"error": str(e)}, 500 \ No newline at end of file