Skip to content

Commit bc536d2

Browse files
authored
Merge pull request #20 from MrIbrahem/update
Update
2 parents 37c2f98 + 9bce820 commit bc536d2

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

54 files changed

+658023
-505
lines changed

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,3 +21,4 @@ public_html/web.config
2121
/public_html
2222
python/src/static/js/duplicate_lemmas2.js
2323
python/src/static/js/‏‏duplicate_lemmas - نسخة.js
24+
python/src/logs/errors.log

python/mysql.sql

Lines changed: 15 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ SET NAMES utf8mb4;
1010
DROP TABLE IF EXISTS `lemmas_p11038`;
1111
CREATE TABLE `lemmas_p11038` (
1212
`id` int NOT NULL AUTO_INCREMENT,
13-
`lemma_id` int NOT NULL,
13+
`lemma_id` bigint NOT NULL,
1414
`lemma` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,
1515
`pos` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT '',
1616
`pos_cat` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT '',
@@ -22,11 +22,6 @@ CREATE TABLE `lemmas_p11038` (
2222
KEY `lemma_id` (`lemma_id`)
2323
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
2424

25-
26-
DROP VIEW IF EXISTS `p11038_lemmas_match`;
27-
CREATE TABLE `p11038_lemmas_match` (`id` int);
28-
29-
3025
DROP TABLE IF EXISTS `wd_data`;
3126
CREATE TABLE `wd_data` (
3227
`id` int NOT NULL AUTO_INCREMENT,
@@ -35,25 +30,29 @@ CREATE TABLE `wd_data` (
3530
`lemma` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,
3631
PRIMARY KEY (`id`),
3732
UNIQUE KEY `wd_id2` (`wd_id`),
38-
KEY `wd_id` (`wd_id`)
33+
KEY `wd_id` (`wd_id`),
34+
KEY `lemma` (`lemma`)
3935
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
4036

41-
42-
DROP VIEW IF EXISTS `wd_data_both`;
43-
CREATE TABLE `wd_data_both` (`vi_wd_id` varchar(255), `vi_wd_id_category` varchar(255), `vi_lemma` varchar(255), `vi_value` varchar(255));
44-
45-
4637
DROP TABLE IF EXISTS `wd_data_p11038`;
4738
CREATE TABLE `wd_data_p11038` (
4839
`id` int NOT NULL AUTO_INCREMENT,
4940
`wd_data_id` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,
50-
`value` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,
41+
`value` bigint NOT NULL,
5142
PRIMARY KEY (`id`),
52-
UNIQUE KEY `wd_data_id` (`wd_data_id`,`value`),
53-
KEY `wd_data_id_value` (`wd_data_id`,`value`),
43+
UNIQUE KEY `wd_data_id_value` (`wd_data_id`,`value`),
44+
KEY `wd_data_id` (`wd_data_id`),
45+
KEY `value` (`value`),
5446
CONSTRAINT `fk_wd_data` FOREIGN KEY (`wd_data_id`) REFERENCES `wd_data` (`wd_id`) ON DELETE CASCADE
5547
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
5648

49+
DROP VIEW IF EXISTS `p11038_lemmas_match`;
50+
CREATE TABLE `p11038_lemmas_match` (`id` int);
51+
52+
53+
DROP VIEW IF EXISTS `wd_data_both`;
54+
CREATE TABLE `wd_data_both` (`vi_wd_id` varchar(255), `vi_wd_id_category` varchar(255), `vi_lemma` varchar(255), `vi_value` bigint);
55+
5756

5857
DROP VIEW IF EXISTS `wd_data_view`;
5958
CREATE TABLE `wd_data_view` (`wd_id` varchar(255), `wd_id_category` varchar(255), `lemma` varchar(255), `P11038_values` text);
@@ -68,8 +67,4 @@ CREATE ALGORITHM=UNDEFINED SQL SECURITY DEFINER VIEW `wd_data_both` AS select `d
6867
DROP TABLE IF EXISTS `wd_data_view`;
6968
CREATE ALGORITHM=UNDEFINED SQL SECURITY DEFINER VIEW `wd_data_view` AS select `d`.`wd_id` AS `wd_id`,`d`.`wd_id_category` AS `wd_id_category`,`d`.`lemma` AS `lemma`,group_concat(`p`.`value` separator ', ') AS `P11038_values` from (`wd_data` `d` left join `wd_data_p11038` `p` on((`d`.`wd_id` = `p`.`wd_data_id`))) group by `d`.`wd_id`,`d`.`wd_id_category`,`d`.`lemma`;
7069

71-
-- 2025-08-04 21:05:29 UTC
72-
Cc
73-
W
74-
.*
75-
70+
-- 2025-08-12 02:10:00 UTC

python/src/app.py

Lines changed: 91 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -2,29 +2,67 @@
22
import sys
33
from flask import Flask, render_template, request, Response, session
44
import json
5+
import time
6+
from flask import g
7+
8+
from pyx import logs_bot_new
9+
from pyx.wd_data_bots import wd_data_P11038
10+
from pyx.sparql_bots import sparql_bot
11+
from pyx.sparql_bots.render import render_all_arabic_by_category
12+
from pyx.bots.not_in_db_bot import get_not_in_db
13+
514

615
app = Flask(__name__)
716
# CORS(app) # ← لتفعيل CORS
8-
import logs_bot_new
9-
from bots import sparql_bot
10-
from bots.match_sparql import get_wd_not_in_sql
11-
# from logs_db import wd_data_table # count_all, get_all
12-
from logs_db import wd_data_P11038
1317

1418

15-
def jsonify(data : dict) -> str:
16-
response_json = json.dumps(data, ensure_ascii=False, indent=4)
19+
@app.before_request
20+
def before_request():
21+
g.start_time = time.time()
22+
23+
24+
@app.after_request
25+
def after_request(response):
26+
if hasattr(g, 'start_time'):
27+
g.load_time = time.time() - g.start_time
28+
return response
29+
30+
31+
@app.context_processor
32+
def inject_load_time():
33+
# نحسب الوقت الحالي - وقت البداية
34+
load_time = 0
35+
if hasattr(g, 'start_time'):
36+
load_time = time.time() - g.start_time
37+
return dict(load_time=load_time)
38+
39+
40+
def jsonify(data : dict, **kwargs) -> str:
41+
diff = 0
42+
if hasattr(g, 'start_time'):
43+
diff = time.time() - g.start_time
44+
# ---
45+
result = {
46+
'load_time' : round(diff, 3),
47+
}
48+
# ---
49+
result.update(kwargs)
50+
# ---
51+
result["data"] = data
52+
# ---
53+
response_json = json.dumps(result, ensure_ascii=False, indent=4)
54+
# ---
1755
return Response(response=response_json, content_type="application/json; charset=utf-8")
1856

1957

2058
@app.route("/api/wd_data_count", methods=["GET"])
2159
def wd_data_api_count():
2260
# ---
23-
filter_data = request.args.get("filter_data", "all", type=str)
61+
_filter_data = request.args.get("filter_data", "all", type=str)
2462
# ---
25-
counts = wd_data_P11038.count_all(filter_data)
63+
counts, db_exec_time = wd_data_P11038.count_all_p11038()
2664
# ---
27-
return jsonify(counts)
65+
return jsonify(counts, db_exec_time=db_exec_time)
2866

2967

3068
@app.route("/api/wd_data", methods=["GET"])
@@ -36,25 +74,37 @@ def wd_data_api():
3674
order_by = request.args.get("order_by", "id", type=str)
3775
filter_data = request.args.get("filter_data", "with", type=str)
3876
# ---
39-
all_result = wd_data_P11038.get_lemmas(limit=limit, offset=offset, order=order, order_by=order_by, filter_data=filter_data)
77+
all_result, db_exec_time = wd_data_P11038.get_lemmas(limit=limit, offset=offset, order=order, order_by=order_by, filter_data=filter_data)
78+
# ---
79+
return jsonify(all_result, db_exec_time=db_exec_time)
80+
81+
82+
@app.route("/api/not_in_db", methods=["GET"])
83+
def not_in_db_api():
4084
# ---
41-
return jsonify(all_result)
85+
result, sparql_exec_time, db_exec_time = get_not_in_db()
86+
# ---
87+
return jsonify(result, db_exec_time=db_exec_time, sparql_exec_time=sparql_exec_time)
4288

4389

44-
@app.route("/api/wd_not_in_sql", methods=["GET"])
45-
def api_wd_not_in_sql():
90+
@app.route("/api/logs_new", methods=["GET"])
91+
def logs_new_api():
4692
# ---
47-
result = get_wd_not_in_sql()
93+
result, db_exec_time = logs_bot_new.find_logs(request)
4894
# ---
49-
return jsonify(result)
95+
return jsonify(result, db_exec_time=db_exec_time)
5096

5197

5298
@app.route("/logs_new", methods=["GET"])
5399
def view_logs_new():
54100
# ---
55-
result = logs_bot_new.find_logs(request)
101+
result, db_exec_time = logs_bot_new.find_logs(request)
102+
# ---
103+
time_tab = {
104+
"db_exec_time": db_exec_time
105+
}
56106
# ---
57-
return render_template("logs_new.php", result=result)
107+
return render_template("logs_new.php", result=result, time_tab=time_tab)
58108

59109

60110
@app.route("/autocomplete.php", methods=["GET"])
@@ -79,40 +129,39 @@ def P11038():
79129

80130
@app.route("/P11038_wd", methods=["GET"])
81131
def P11038_wd():
82-
wd_count = sparql_bot.count_arabic_with_P11038()
132+
# ---
83133
limit = request.args.get('limit', 100, type=int)
84-
85-
result = sparql_bot.all_arabic(limit)
86-
split_by_category = {}
87-
for item in result:
88-
category = item['category']
89-
# ---
90-
if category not in split_by_category:
91-
split_by_category[category] = {
92-
'category': category,
93-
'categoryLabel': item['categoryLabel'],
94-
'members': []
95-
}
96-
# ---
97-
split_by_category[category]['members'].append(item)
98-
99-
return render_template("P11038_wd.html", limit=limit, result=split_by_category, wd_count=wd_count)
134+
# ---
135+
wd_count, _ = sparql_bot.count_arabic_with_P11038()
136+
# ---
137+
split_by_category, sparql_exec_time = render_all_arabic_by_category(limit)
138+
# ---
139+
time_tab = {
140+
"sparql_exec_time": sparql_exec_time,
141+
}
142+
# ---
143+
return render_template(
144+
"P11038_wd.html",
145+
limit=limit,
146+
result=split_by_category,
147+
wd_count=wd_count,
148+
time_tab=time_tab,
149+
)
100150

101151

102152
@app.route("/not_in_db", methods=["GET"])
103153
def not_in_db():
104154
# ---
105155
limit = request.args.get('limit', 100, type=int)
106156
# ---
107-
result = get_wd_not_in_sql()
108-
# ---
109-
# sort result by len of P11038_list
110-
result = sorted(result, key=lambda x: len(x['P11038_list']), reverse=True)
157+
result, sparql_exec_time, db_exec_time = get_not_in_db(limit)
111158
# ---
112-
if limit > 0:
113-
result = result[:limit]
159+
time_tab = {
160+
"db_exec_time": db_exec_time,
161+
"sparql_exec_time": sparql_exec_time,
162+
}
114163
# ---
115-
return render_template("not_in_db.html", data=result, limit=limit)
164+
return render_template("not_in_db.html", data=result, limit=limit, time_tab=time_tab)
116165

117166

118167
@app.route("/not_in_db1", methods=["GET"])

python/src/bots/match_sparql.py

Lines changed: 0 additions & 91 deletions
This file was deleted.

0 commit comments

Comments
 (0)