From 361bde9ce9f320f1dfee986a95ee59fb3cbe2dbf Mon Sep 17 00:00:00 2001 From: gaze <397334393@qq.com> Date: Fri, 8 Sep 2023 09:46:29 +0800 Subject: [PATCH] first commit --- .drone.yml | 39 +++++ .gitignore | 3 + Dockerfile | 7 + README.md | 95 +++++++++++ README_CN.md | 98 ++++++++++++ config.dev.yaml | 13 ++ config.yaml | 12 ++ main.py | 407 +++++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 17 ++ surl.sql | 16 ++ 10 files changed, 707 insertions(+) create mode 100644 .drone.yml create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 README_CN.md create mode 100644 config.dev.yaml create mode 100644 config.yaml create mode 100644 main.py create mode 100644 requirements.txt create mode 100644 surl.sql diff --git a/.drone.yml b/.drone.yml new file mode 100644 index 0000000..4747f79 --- /dev/null +++ b/.drone.yml @@ -0,0 +1,39 @@ +kind: pipeline +type: docker +name: short-url-fastapi + +steps: + # 部署 + - name: deploy-clean + image: appleboy/drone-ssh + when: + branch: master + event: push + settings: + host: + from_secret: ssh_host + username: + from_secret: ssh_username + password: + from_secret: ssh_password + port: 22 + command_timeout: 2m + script: + - cd /home/ubuntu/Projects/short_url_fastapi + - | + if sudo docker ps -f name=surl | grep -q surl; then + sudo docker stop surl + else + echo "Container surl is not running, nothing to stop" + fi + - | + if sudo docker ps -af name=surl | grep -q surl; then + sudo docker rm surl + else + echo "Container surl does not exist, nothing to remove." + fi + - sudo docker rmi surl --force + - sudo docker image prune -f + - git pull --rebase + - sudo docker build -t surl . + - sudo docker run -d --name surl -p 8000:8000 -e docs=true surl \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2e62100 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +venv +.idea +__pycache__ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..7a92d4e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,7 @@ +FROM python:3.10.11-slim-buster +WORKDIR /app +COPY . . +RUN pip3 config set global.index-url https://mirrors.aliyun.com/pypi/simple/ +RUN pip3 install -r requirements.txt +EXPOSE 8000 +CMD ["python3", "main.py"] \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..22c5c98 --- /dev/null +++ b/README.md @@ -0,0 +1,95 @@ +# short_url +![maven](https://img.shields.io/badge/python-3.8%2B-blue) +![maven](https://img.shields.io/badge/tornado-6.2-green) +![maven](https://img.shields.io/badge/aiomysql-0.1.1-orange) + +An asynchronous short link backend written in Python based on Tornado and aiomysql + +[简体中文](./README_CN.md) | English + +## Run + +### Clone code and enter the dir +```shell +git clone https://github.com/gazedreamily/short_url.git +cd short_url +``` + +### Connect to your mysql database +```shell +mysql -u [your mysql username] -p +``` + +### Create table +```sql +use [database name] +source [project dir]/surl.sql; +``` + +### Change configuration file +default content of configuration file +```yaml +database: # configuration of database + host: # servername of database server + port: # port of database server + user: # username of database server + password: # password of database server + database: # name of database on database server + +sign: # configuration of authentication + secret: # secret when adding new url + +server: # configuration of web server + host: # servername of web server + port: # port of web server +``` +fill in the configuration file according to your own situation before running + +### Install the operating environment +Linux +```shell +pip3 install -r requirements.txt +``` + +Windows +```shell +pip install -r requirements.txt +``` + +### just run it! +Linux +```shell +python3 main.py +``` + +Windows +```shell +python main.py +``` + +## Usage +### Visit +Use this server configuration like this +```yaml +server: + host: a.com + port: 80 + protocol: http +``` +The domain name in the database is recorded as + +| id | source | target | createTime | expireTime | +|-----|---------|---------------------|--------------| -----------| +| 1 | AbcdEfg | https://google.com/ | | | + +When accessing `http://a.com/AbcdEfg` , the target will be redirected to `https://google.com/`. + +The server will return `404` if there is no source record in the database. + +### ExpireTime +When accessing a link, the backend will judge whether the current link has expired. If the link expires, it will delete the link from the database and return `404` + +### Insert a link +You can use `client.py` to do it. + +When inserting a link, the post carries target_url, current timestamp, signature based on timestamp and secret key, and expiration time (optional). \ No newline at end of file diff --git a/README_CN.md b/README_CN.md new file mode 100644 index 0000000..790d315 --- /dev/null +++ b/README_CN.md @@ -0,0 +1,98 @@ +# short_url +![maven](https://img.shields.io/badge/python-3.8%2B-blue) +![maven](https://img.shields.io/badge/tornado-6.2-green) +![maven](https://img.shields.io/badge/aiomysql-0.1.1-orange) + +基于Tornado与aiomysql使用Python语言编写的异步短链接后端 + +简体中文 | [English](./README.md) + +## 运行 + +### 拉取代码进入目录 +```shell +git clone https://github.com/gazedreamily/short_url.git +cd short_url +``` + +### 连入数据库 +```shell +mysql -u 你的mysql用户 -p +``` + +### 创建数据表 +```sql +use 数据库名 +source 项目路径/surl.sql; +``` + +### 更改配置文件 +**配置优先级为 环境变量 -> 配置文件 -> 默认配置** + +**默认监听0.0.0.0的8000端口** + +默认配置文件内容 +```yaml +database: # 此部分为数据库相关配置 + host: # 数据库主机名（域名） + port: # 数据库端口号 + user: # 数据库用户名 + password: # 数据库密码 + database: # 数据库名 + +sign: # 验证相关 + secret: # 新增短链接时的验证秘钥 + +server: # 服务器相关 + host: # 服务器主机名（域名） + port: # 服务端口号 +``` +运行前请将以上信息根据个人情况填写 + +### 安装相关库 +Linux +```shell +pip3 install -r requirements.txt +``` + +Windows +```shell +pip install -r requirements.txt +``` + +### 运行项目 +Linux +```shell +python3 main.py +``` + +Windows +```shell +python main.py +``` + +## 使用 +### 访问 +以此server配置为模板 +```yaml +server: + host: a.com + port: 80 + protocol: http +``` +数据库中域名记录为 + +| id | source | target | createTime | expireTime | +|-----|---------|---------------------|--------------| -----------| +| 1 | AbcdEfg | https://google.com/ | | | + +访问 `http://a.com/AbcdEfg` 时，浏览器会被重定向到 `https://google.com/` + +若访问的路径没有对应的目标网址，则会返回404 + +### 有效时间 +当访问链接时，后端会判断当前链接是否过期，若链接过期，则会从数据库中删除该链接，并返回404 + +### 插入链接 +可以使用 `client.py` 插入链接 +在插入链接时，post中携带target_url，当前时间戳，基于时间戳和秘钥的签名以及过期时间（可选） \ No newline at end of file diff --git a/config.dev.yaml b/config.dev.yaml new file mode 100644 index 0000000..c9c2892 --- /dev/null +++ b/config.dev.yaml @@ -0,0 +1,13 @@ +# docker run -d --name mysql -e MYSQL_ROOT_PASSWORD=123456 -p 3306:3306 mysql:8.0 +database: + host: localhost + port: 3306 + user: root + password: 123456 + database: surl + +sign: + secret: test + +host: + port: 7777 \ No newline at end of file diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..a3c3e57 --- /dev/null +++ b/config.yaml @@ -0,0 +1,12 @@ +database: + host: localhost + port: 3306 + user: root + password: 123456 + database: surl + +sign: + secret: test + +#host: +# port: 7777 diff --git a/main.py b/main.py new file mode 100644 index 0000000..eab13fa --- /dev/null +++ b/main.py @@ -0,0 +1,407 @@ +import datetime +import time +from typing import Any, Union +import sys +import aiomysql as aiomysql +from fastapi import FastAPI, Request # 导入FastAPI +from fastapi.middleware.cors import CORSMiddleware +import uvicorn # uvicorn:主要用于加载和提供应用程序的服务器 +from fastapi.responses import RedirectResponse +import os +import yaml +import hashlib +import base64 +import hmac +from pydantic import BaseModel +import random +import string + + +config_file = "config.yaml" +default_doc = False +if len(sys.argv) > 1: + env_name = sys.argv[1] + config_file = f"config.{env_name}.yaml" + if env_name == "dev": + default_doc = True +with open(config_file) as f: + config = yaml.safe_load(f) +sql_host = config['database']['host'] +sql_port = int(config['database']['port']) +sql_user = config['database']['user'] +sql_password = str(config['database']['password']) +sql_database = config['database']['database'] +sign_secret = config['sign']['secret'] +host = config['host']['host'] if (config.get('host') and config['host'].get('host')) is not None else None +port = int(config['host']['port']) if (config.get('host') and config['host'].get('port')) is not None else None + +env = os.environ +# 创建一个app实例 +app = FastAPI() if default_doc or (env.get("docs") is not None and env.get("docs").lower() == "true")\ + else FastAPI(openapi_url=None) + + +# 配置 CORS 中间件 +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], # 允许所有来源，可以根据需求进行配置 + allow_credentials=True, + allow_methods=["*"], # 允许所有请求方法 + allow_headers=["*"], # 允许所有请求头 +) + + +async def get_connect(): + conn = await aiomysql.connect( + host=sql_host, + port=sql_port, + user=sql_user, + password=sql_password, + db=sql_database + ) + return conn + + +async def get_sources() -> tuple: + conn = await get_connect() + async with conn.cursor() as cursor: + await cursor.execute('SELECT source FROM surl') + ret = await cursor.fetchall() + conn.close() + return tuple([i[0] for i in ret]) + + +async def get_redirect_url(code: str) -> dict: + conn = await get_connect() + sql = f"SELECT id, source, target, createTime, expireTime FROM surl where source = '{code}';" + async with conn.cursor() as cursor: + await cursor.execute(sql) + ret = await cursor.fetchall() + conn.close() + if ret == (): + return {} + url_info = { + "id": ret[0][0], + "source": ret[0][1], + "target": ret[0][2], + "createTime": ret[0][3], + "expireTime": ret[0][4] + } + return url_info + + +async def get_is_expired(url_info: dict) -> bool: + if url_info == {}: + return True + expire_time = url_info.get("expireTime") + if expire_time is None or expire_time >= datetime.datetime.now(): + return False + sql = f"DELETE FROM surl WHERE id = '{url_info.get('id')}';" + conn = await get_connect() + async with conn.cursor() as cursor: + await cursor.execute(sql) + await conn.commit() + conn.close() + return True + + +async def get_is_out_of_date(ts: int) -> bool: + return abs(time.time() - ts) > 300 + + +async def insert_surl(source: str, target: str, expire: Union[int, None] = None) -> None: + sql = f"INSERT INTO surl (`source`, `target`) value('{source}', '{target}')" + if expire: + expire_time = datetime.datetime.fromtimestamp(expire) + sql = f"INSERT INTO surl (`source`, `target`, `expireTime`) value('{source}', '{target}', '{expire_time}')" + conn = await get_connect() + async with conn.cursor() as cursor: + await cursor.execute(sql) + await conn.commit() + conn.close() + + +async def update_target(source: str, target: str, expire: Union[int, None]) -> None: + sql = f"UPDATE surl SET `target` = '{target}' WHERE `source` = '{source}'" + if expire: + expire_time = datetime.datetime.fromtimestamp(expire) + sql = f"UPDATE surl SET `target` = '{target}', `expireTIme` = '{expire_time}' WHERE `source` = '{source}'" + conn = await get_connect() + print(sql) + async with conn.cursor() as cursor: + await cursor.execute(sql) + await conn.commit() + conn.close() + + +async def delete_surl(source: str) -> None: + sql = f"DELETE FROM surl WHERE `source` = '{source}'" + conn = await get_connect() + async with conn.cursor() as cursor: + await cursor.execute(sql) + await conn.commit() + conn.close() + + +async def get_all_surl_by_offset(page: Union[int, None], size: Union[int, None], base_url: Any) -> list: + if not page: + page = 0 + if not size: + size = 20 + sql = (f"SELECT `source`, `target`, `createTime`, `expireTime` FROM surl " + f"ORDER BY `createTime` LIMIT {size} OFFSET {page * size}") + conn = await get_connect() + async with conn.cursor() as cursor: + await cursor.execute(sql) + ret = await cursor.fetchall() + conn.close() + return [{ + "source": i[0], + "target": i[1], + "url": f"{base_url}s/{i[0]}", + "created_time": i[2], + "expire_time": i[3], + } for i in ret] + + +async def gen_sign(timestamp: Union[str, int]) -> str: + string_to_sign = '{}\n{}'.format(timestamp, sign_secret) + hmac_code = hmac.new(string_to_sign.encode("utf-8"), digestmod=hashlib.sha256).digest() + sign = base64.b64encode(hmac_code).decode('utf-8') + return sign + + +async def gen_new_source(length: int) -> str: + letters = string.ascii_letters # 包含所有字母的字符串 + return ''.join(random.choice(letters) for _ in range(length)) + + +async def get_is_valid(timestamp: Union[str, int], sign: str) -> bool: + return not await get_is_out_of_date(timestamp) and sign == await gen_sign(timestamp) + + +@app.get("/s/{source}") +async def redirect_target(source: str) -> Any: + """ + 短链接重定向 + + 当访问/s/xxxx时，重定向到xxxx对应的链接 + + 无返回值，直接重定向到目标链接 + """ + url_info = await get_redirect_url(source) + if not await get_is_expired(url_info): + return RedirectResponse(url_info["target"]) + return {"code": 404, "msg": "Not Found"} + + +@app.get("/surl/{source}") +async def redirect_target(source: str, request: Request) -> dict: + """ + 短链接重定向地址查询 + + 当访问/surl/xxxx时，返回xxxx对应的链接 + + 链接有效，正常返回 + + { + "code": 200, + "msg": "success", + "data": { + "source": 短链接后缀, + "target": 目标链接, + "url": 短链接 + } + } + + 链接无效，错误返回 + + { + "code": 404, + "msg" "Not Found" + } + """ + url_info = await get_redirect_url(source) + if not await get_is_expired(url_info): + url = f"{request.base_url}s/{source}" + return {"code": 200, "msg": "success", "data": {"source": source, "target": url_info["target"], "url": url}} + return {"code": 404, "msg": "Not Found"} + + +class CreateShortURLRequest(BaseModel): + sign: str + url: str + ts: int + source: Union[int, None] = None + expire_time: Union[int, None] = None + + +@app.post("/create_short_url") +async def create_short_url(params: CreateShortURLRequest, request: Request) -> dict: + """ + 创建短链接接口 + + Params: + + { + "sign": str # 签名，用于验证请求有效性 + "url": str # 目标url + "ts": int # 发出请求时的时间戳 + "source": str # 自定义后缀（可选） + "expire_time": int # 过期时间戳（可选） + } + + Return: + + { + "code": 200, + "msg": success, + "data": { + "source": 随机生成的短链接后缀, + "target": 目标url, + "url": 短链接, + } + } + """ + source = params.source + if not source: + source = await gen_new_source(5) + elif source in await get_sources(): + return {"code": -1, "msg": "source exists"} + if not await get_is_valid(params.ts, params.sign): + return {"code": 400, "msg": "bad request"} + await insert_surl(source, params.url, params.expire_time) + url = f"{request.base_url}s/{source}" + return {"code": 200, "msg": "success", "data": {"source": source, "target": params.url, "url": url}} + + +class EditShortURLRequest(BaseModel): + sign: str + url: str + ts: int + source: str + expire_at: Union[int, None] = None + + +@app.post("/update_short_url") +async def update_short_url(params: EditShortURLRequest, request: Request) -> dict: + """ + 修改短链接接口 + + Params: + + { + "sign": str # 签名，用于验证请求有效性 + "url": str # 目标url + "ts": int # 发出请求时的时间戳 + "source": str # 短链接后缀 + "expire_at": int # 过期时间戳（可选） + } + + Return: + + { + "code": 200, + "msg": success, + "data": { + "source": 短链接后缀, + "target": 目标url, + "url": 短链接, + "expire_time": 过期时间,（如果有expire_time） + "expire_at": 过期时间戳（如果有expire_time） + } + } + """ + if not await get_is_valid(params.ts, params.sign) or params.source not in await get_sources(): + return {"code": 400, "msg": "bad request"} + await update_target(params.source, params.url, params.expire_at) + url = f"{request.base_url}s/{params.source}" + if params.expire_at: + return {"code": 200, "msg": "success", "data": { + "source": params.source, + "target": params.url, + "url": url, + "expire_time": datetime.datetime.fromtimestamp(params.expire_at), + "expire_at": params.expire_at + } + } + return {"code": 200, "msg": "success", "data": {"source": params.source, "target": params.url, "url": url}} + + +class DeleteShortURLRequest(BaseModel): + sign: str + ts: int + source: str + + +@app.post("/delete_short_url") +async def delete_short_url(params: DeleteShortURLRequest) -> dict: + """ + 删除短链接接口 + + Params: + + { + "sign": str # 签名，用于验证请求有效性 + "ts": int # 发出请求时的时间戳 + "source": str # 短链接后缀 + } + + Return: + + { + "code": 200, + "msg": success, + } + """ + if not await get_is_valid(params.ts, params.sign) or params.source not in await get_sources(): + return {"code": 400, "msg": "bad request"} + await delete_surl(params.source) + return {"code": 200, "msg": "success"} + + +class ListShortURLRequest(BaseModel): + sign: str + ts: int + page: Union[int, None] = None + size: Union[int, None] = None + + +@app.post("/list_short_url") +async def list_short_url(params: ListShortURLRequest, request: Request) -> dict: + """ + 查询所有短链接接口 + + Params: + + { + "sign": str # 签名，用于验证请求有效性 + "ts": int # 发出请求时的时间戳 + "page": int # 页数（可选），默认第一页 + "size": int # 每一页的数量（可选），默认20个 + } + + Return: + + { + "code": 200, + "msg": success, + "data": { + "source": 短链接后缀, + "target": 目标地址, + "url": 短链接, + "created_time": 创建时间, + "expire_time": 过期时间, + } + } + """ + if not await get_is_valid(params.ts, params.sign): + return {"code": 400, "msg": "bad request"} + surl_list = await get_all_surl_by_offset(params.page, params.size, request.base_url) + return {"code": 200, "msg": "success", "data": surl_list} + + +if __name__ == '__main__': + host = (env.get("HOST") if env.get("HOST") is not None else host) or "0.0.0.0" + port = (int(env.get("PORT")) if env.get("PORT") is not None else port) or 8000 + uvicorn.run(app='main:app', host=host, port=port, reload=True) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..00c1551 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,17 @@ +aiomysql==0.2.0 +annotated-types==0.5.0 +anyio==3.7.1 +click==8.1.6 +colorama==0.4.6 +exceptiongroup==1.1.2 +fastapi==0.103.1 +h11==0.14.0 +idna==3.4 +pydantic==2.1.1 +pydantic_core==2.4.0 +PyMySQL==1.1.0 +PyYAML==6.0.1 +sniffio==1.3.0 +starlette==0.27.0 +typing_extensions==4.7.1 +uvicorn==0.23.2 diff --git a/surl.sql b/surl.sql new file mode 100644 index 0000000..18f5d10 --- /dev/null +++ b/surl.sql @@ -0,0 +1,16 @@ +SET NAMES utf8mb4; +SET FOREIGN_KEY_CHECKS = 0; + +-- ---------------------------- +-- Table structure for surl +-- ---------------------------- +DROP TABLE IF EXISTS `surl`; +CREATE TABLE `surl` ( + `id` CHAR(36) NOT NULL PRIMARY KEY DEFAULT (UUID()), + `source` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL, + `target` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL, + `createTime` datetime DEFAULT NOW(), + `expireTime` datetime NULL DEFAULT NULL +) ENGINE = InnoDB CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic; + +SET FOREIGN_KEY_CHECKS = 1; \ No newline at end of file