Some content is hidden

Large Commits have some content hidden by default. Use the searcx below for content that may be hidden.

99 files changed

+19348
-0
lines changed
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
# python-web-api-flask-json-to-elasticsearch-
2+
3+
## Description
4+
Creates an api of `git-repo` for a flask project by loading json into elasticsearch.
5+
6+
Elasticsearch DSL is used to build a proxy connection to the `elasticsearch` service.
7+
8+
Has the ability to query by parameters and pagination.
9+
10+
Uses redis as a cache layer for queries.
11+
12+
## Security
13+
Self-signed ssl certificate for both web and api requests.
14+
Requires basic authentication for endpoints.
15+
16+
| username | password |
17+
| -------- | -------- |
18+
| *user* | *pass* |
19+
20+
## Testing
21+
Unit tests ran when program begins.
22+
Remotely tested with *testify*, the ssl is not verified.
23+
24+
## Tech stack
25+
- python
26+
- flask
27+
- flask-httpauth
28+
- pyopenssl
29+
- elasticsearch
30+
- redis
31+
- unittest
32+
- testify
33+
34+
## Docker stack
35+
- python:latest
36+
- elasticsearch
37+
- kibana
38+
- redis
39+
40+
## To run
41+
`sudo ./install.sh -u`
42+
- [Web page availble here](https://localhost)
43+
- [Endpoints](endpoints.md)
44+
45+
## To stop
46+
`sudo ./install.sh -d`
47+
48+
## For help
49+
`sudo ./install.sh -h`
50+
51+
## Credits
52+
- [Elasticsearch pagination](https://logfetch.com/elasticsearch-scroll-python/)
53+
- [Elasticsearch terms aggregation](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-multi-terms-aggregation.html)
54+
- [Flask blueprint tutorial](https://realpython.com/flask-blueprint/)
55+
- [Flask blueprint pass args](https://stackoverflow.com/questions/28640081/how-to-pass-arbitrary-arguments-to-a-flask-blueprint)
56+
- [Flask redis elasticsearch example](https://.com/lakshaysinghal/indexingapi/blob/master/daemon.py)
57+
- [Selenium in Docker](https://stackoverflow.com/questions/66597600/how-to-connect-to-standalone-selenium-firefox-container-from-another-container)
58+
- [Selenium for python](https://selenium-python.readthedocs.io/)
59+
- [All endpoints for flask app](https://.com/pallets/flask/blob/2.1.2/src/flask/cli.py#L931)
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
version: "3.4"
2+
3+
services:
4+
py-api-srv:
5+
build: py-srv/api
6+
healthcheck:
7+
test: "exit 0"
8+
command: sh -c "/wait && python -m unittest && python app.py"
9+
ports:
10+
- 443
11+
environment:
12+
- WAIT_HOSTS=elasticsearch:9200
13+
- WAIT_HOSTS_TIMEOUT=300
14+
- WAIT_SLEEP_INTERVAL=30
15+
- WAIT_HOST_CONNECT_TIMEOUT=30
16+
depends_on:
17+
- elasticsearch
18+
- kibana
19+
- redis
20+
links:
21+
- "elasticsearch:elasticsearch"
22+
- "redis:redis"
23+
24+
py-web-srv:
25+
build: py-srv/web
26+
healthcheck:
27+
test: "exit 0"
28+
command: sh -c "/wait && python -m unittest && python app.py"
29+
ports:
30+
- 443:443
31+
environment:
32+
- WAIT_HOSTS=py-api-srv:443
33+
- WAIT_HOSTS_TIMEOUT=300
34+
- WAIT_SLEEP_INTERVAL=30
35+
- WAIT_HOST_CONNECT_TIMEOUT=30
36+
depends_on:
37+
- py-api-srv
38+
links:
39+
- "py-api-srv:py-api-srv"
40+
41+
redis:
42+
image: redis:latest
43+
44+
elasticsearch:
45+
image: elasticsearch:7.12.0
46+
healthcheck:
47+
test: "exit 0"
48+
interval: 1m30s
49+
timeout: 30s
50+
retries: 5
51+
start_period: 30s
52+
environment:
53+
- discovery.type=single-node
54+
- bootstrap.memory_lock=true
55+
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
56+
ulimits:
57+
memlock:
58+
soft: -1
59+
hard: -1
60+
# volumes:
61+
# - ./es/data:/usr/share/elasticsearch/data
62+
ports:
63+
- 9200:9200
64+
65+
kibana:
66+
image: kibana:7.12.0
67+
healthcheck:
68+
test: "exit 0"
69+
interval: 1m30s
70+
timeout: 30s
71+
retries: 5
72+
start_period: 30s
73+
environment:
74+
- "ELASTICSEARCH_URL=http://elasticsearch:9200"
75+
- "SERVER_NAME=127.0.0.1"
76+
ports:
77+
- 5601:5601
78+
depends_on:
79+
- elasticsearch
80+
81+
# py-api-test-srv:
82+
# build:
83+
# context: py-test-srv/api
84+
# command: sh -c "/wait && python app.py"
85+
# environment:
86+
# - WAIT_HOSTS=py-api-srv:443
87+
# - WAIT_HOSTS_TIMEOUT=300
88+
# - WAIT_SLEEP_INTERVAL=30
89+
# - WAIT_HOST_CONNECT_TIMEOUT=30
90+
# depends_on:
91+
# - py-api-srv
92+
# links:
93+
# - "py-api-srv:py-api-srv"
94+
95+
# py-web-test-srv:
96+
# build:
97+
# context: py-test-srv/web
98+
# command: sh -c "/wait && python app.py"
99+
# environment:
100+
# - WAIT_HOSTS=py-web-srv:443, selenium:4444
101+
# - WAIT_HOSTS_TIMEOUT=300
102+
# - WAIT_SLEEP_INTERVAL=30
103+
# - WAIT_HOST_CONNECT_TIMEOUT=30
104+
# depends_on:
105+
# - py-web-srv
106+
# - selenium
107+
# links:
108+
# - "py-web-srv:py-web-srv"
109+
# - "selenium:selenium"
110+
#
111+
# selenium:
112+
# image: selenium/standalone-firefox
113+
# hostname: selenium
114+
# ports:
115+
# - 4444:4444
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
2+
FROM python:latest
3+
4+
COPY bin/ /app
5+
6+
WORKDIR /app
7+
8+
RUN pip install -r requirements.txt
9+
10+
ENTRYPOINT [ "python" ]
11+
12+
CMD [ "app.py"]
Whitespace-only changes.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
import csv, json, logging, math, requests
2+
3+
from model.cls_repo import Repo
4+
5+
logging.basicConfig(level=logging.INFO)
6+
7+
def get_url(url):
8+
headers = {'Content-Type': 'application/json' }
9+
logging.info(url)
10+
return requests.get(url, headers=headers, verify=False)
11+
12+
def to_json(url):
13+
response = get_url(url)
14+
logging.info("to json "+url)
15+
return json.loads(response.text)
16+
17+
def get_count():
18+
url = "https://api..com/users/bearddan2000"
19+
json_object = to_json(url)
20+
logging.info("get count")
21+
return json_object['public_repos']
22+
23+
def get_repo_by_page(page: int):
24+
lst = []
25+
url = f"https://api..com/users/bearddan2000/repos?per_page=100&page={page}"
26+
json_object = to_json(url)
27+
for item in json_object:
28+
name = item['name']
29+
try:
30+
if len(name.split("-")) > 1:
31+
desc=item['description']
32+
topics=item['topics']
33+
repo = Repo(name,desc,topics)
34+
lst.append(repo)
35+
logging.info(repo)
36+
except:
37+
pass
38+
return lst
39+
40+
def repo_to_csv(rows: list):
41+
# field names
42+
fields = ['Name', 'Desc', 'Topics']
43+
logging.info(fields)
44+
with open('/app/Repo.csv', 'w') as f:
45+
46+
# using csv.writer method from CSV package
47+
write = csv.writer(f)
48+
49+
write.writerow(fields)
50+
write.writerows(rows)
51+
52+
def main():
53+
print("hello world")
54+
repo = []
55+
total_by_hundred: int = get_count()/100
56+
total_rounded: int = math.ceil(total_by_hundred)
57+
for page in range(total_rounded):
58+
collection = get_repo_by_page(page+1)
59+
if collection is not None:
60+
repo.append(collection)
61+
repo_to_csv(repo)
62+
63+
main()
64+
65+
def test_get_count_get_url():
66+
url = "https://api..com/users/bearddan2000"
67+
assert get_url(url).status_code == 200
68+
69+
def test_get_count_to_json_public_repos():
70+
url = "https://api..com/users/bearddan2000"
71+
assert to_json(url).has_key('public_repos') == True
72+
73+
def test_get_count():
74+
assert get_count() > 0
75+
76+
def test_get_repo_get_url():
77+
url = "https://api..com/users/bearddan2000/repos?per_page=100&page=1"
78+
assert get_url(url).status_code == 200
79+
80+
def test_get_repo_to_json_name():
81+
url = "https://api..com/users/bearddan2000/repos?per_page=100&page=1"
82+
assert to_json(url).has_key('name') == True
83+
84+
def test_get_repo_to_json_description():
85+
url = "https://api..com/users/bearddan2000/repos?per_page=100&page=1"
86+
assert to_json(url).has_key('description') == True
87+
88+
def test_get_repo_to_json_topics():
89+
url = "https://api..com/users/bearddan2000/repos?per_page=100&page=1"
90+
assert to_json(url).has_key('topics') == True
Whitespace-only changes.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
class Repo():
2+
3+
def __init__(self, name, desc, topics: list) -> None:
4+
self.name = name
5+
self.compose_desc(desc)
6+
self.compose_topics(topics)
7+
8+
def compose_desc(self, desc):
9+
self.desc = '-'
10+
if desc is not None:
11+
self.desc = desc
12+
13+
def compose_topics(self, topics):
14+
tmp_lst = topics
15+
for item in self.name.split("-"):
16+
tmp_lst.append(item)
17+
condensed_lst = list(set(tmp_lst))
18+
self.topics = "::".join(condensed_lst)
19+
20+
def __str__(self) -> str:
21+
return f"{self.name}, {self.desc}, {self.topics}"
22+
23+
def __iter__(self):
24+
return iter([self.name, self.desc, self.topics])
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
requests
2+
3+
# deps for pytest
4+
pytest
5+
allure-pytest
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
[README](README.md)
2+
3+
## Security
4+
| username | password |
5+
| -------- | -------- |
6+
| *user* | *pass* |
7+
8+
## Endpoints
9+
| desc | cli/api | web |
10+
| ---------- | --------------------------------------------- | -------- |
11+
| all | curl -i -k https://localhost/api | https://localhost/ |
12+
| all pg # | curl -i -k https://localhost/api/*int* | https://localhost/*int* |
13+
| search name | curl -i -k https://localhost/api/name/*str* -u 'user:pass' | https://localhost/name/*str* |
14+
| search name pg # | curl -i -k https://localhost/api/name/*int*/*str* -u 'user:pass' | https://localhost/name/*int*/*str* |
15+
| all topic | - | https://localhost/topics/ |
16+
| search topic | curl -i -k https://localhost/api/topic/*int* -u 'user:pass' | https://localhost/topics/*str* |
17+
| search topic pg # | curl -i -k https://localhost/api/topic/*int*/*str* -u 'user:pass' | https://localhost/topics/*int*/*str* |
18+
| all catagory | - | https://localhost/catagory/ |
19+
| search catagory | curl -i -k https://localhost/api/catagory/*str* -u 'user:pass' | https://localhost/catagory/*str* |
20+
| search catagory pg # | curl -i -k https://localhost/api/catagory/*int*/*str* -u 'user:pass' | https://localhost/catagory/*int*/*str* |
21+
| all subcatagory | - | https://localhost/topics/catagory/sub/ |
22+
| search subcatagory | curl -i -k https://localhost/api/catagory/sub/*str* -u 'user:pass' | https://localhost/topics/catagory/sub/*str* |
23+
| search subcatagory pg # | curl -i -k https://localhost/api/catagory/*int*/sub/*str* -u 'user:pass' | https://localhost/catagory/*int*/sub/*str* |
24+
| search build | curl -i -k https://localhost/api/filter/build/*str* -u 'user:pass' | https://localhost/filter/build/*str* |
25+
| search build pg # | curl -i -k https://localhost/api/filter/*int*/build/*str* -u 'user:pass' | https://localhost/filter/*int*/build/*str* |
26+
| search language | curl -i -k https://localhost/api/filter/language/*str* -u 'user:pass' | https://localhost/filter/language/*str* |
27+
| search language pg # | curl -i -k https://localhost/api/filter/*int*/language/*str* -u 'user:pass' | https://localhost/filter/*int*/language/*str* |
28+
| search platform | curl -i -k https://localhost/api/filter/platform/*str* -u 'user:pass' | https://localhost/filter/platform/*str* |
29+
| search platform pg # | curl -i -k https://localhost/api/filter/*int*/platform/*str* -u 'user:pass' | https://localhost/filter/*int*/platform/*str* |
30+
| search tech | curl -i -k https://localhost/api/filter/tech/*str* -u 'user:pass' | https://localhost/filter/tech/*str* |
31+
| search tech pg # | curl -i -k https://localhost/api/filter/*int*/tech/*str* -u 'user:pass' | https://localhost/filter/*int*/tech/*str* |
32+
33+
## Defintions
34+
- *int* any number 0-9
35+
- *str* any word
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
[2023-05-19 10:04:28 INFO]: install::setup-logging ended
2+
================
3+
[2023-05-19 10:04:28 INFO]: install::start-up started
4+
[2023-05-19 10:04:28 INFO]: install::start-up starting services
5+
[2023-05-19 10:06:43 INFO]: install::root-check started
6+
[2023-05-19 10:06:43 INFO]: install::root-check ended
7+
================
8+
[2023-05-19 10:06:43 INFO]: install::docker-check started
9+
[2023-05-19 10:06:43 INFO]: install::docker-check ended
10+
================
11+
[2023-05-19 10:06:43 INFO]: install::docker-compose-check started
12+
[2023-05-19 10:06:43 INFO]: install::docker-compose-check ended
13+
================
14+
[2023-05-19 10:06:43 INFO]: install::tear-down started
15+
[2023-05-19 10:06:43 INFO]: install::tear-down starting services
16+
[2023-05-19 10:06:43 INFO]: install::tear-down ended
17+
================

0 commit comments

Comments
 (0)