import requests from bs4 import BeautifulSoup from PIL import Image from io import BytesIO import time from datetime import datetime import re from loguru import logger # импорты из моих модулей from handlers.img_save_hand import image_save from handlers.translate_text import translate_text from handlers.create_attachmtd_hand import create_attach_metadata_hand, generate_page_content from handlers.plagin_data_hand import get_plagin_data from db.wp_posts_db import addNewPost, updatePostGuid, updatePostGuidMimetype, checkPlagin from db.wp_postmeta_db import addNewPostmeta from db.wp_term_relationships_db import addNewTermRelationship from db.plagin_text import addNewPlaginText from db.plt_hrefs_text import addNewHrefText, checkHrefText logger.add('logs/logs.log', rotation='100 mb', level='DEBUG') def lets_parse(url='https://gptstore.ai/gpts'): page = 1 while True: logger.debug(f"page - {page}") cookies = { 'ph_phc_NwPasoEFOb5YHKAdjamWq0K4d6sDqTq89tSVWovv4nN_posthog': '^%^7B^%^22distinct_id^%^22^%^3A^%^2201905113-076d-716e-b87c-cd201b325003^%^22^%^2C^%^22^%^24sesid^%^22^%^3A^%^5B1719347119985^%^2C^%^2201905113-0771-7511-9794-61cda198a886^%^22^%^2C1719347119985^%^5D^%^7D', '__Host-next-auth.csrf-token': 'd5b912946563bf0a0d92830dc85284d85db9d7881ba506cbb9fd53a25d94ba0a^%^7C44bacf760340dc5d038e86b8a389002702bb6e9497b21afa24ed6f3ec10b42f7', '__Secure-next-auth.callback-url': 'https^%^3A^%^2F^%^2Fgptstore.ai', 'cf_clearance': 'zIz4FMv3163pvCGGYwVWMIWDDwX34g0WVs05V4fUF_4-1719347121-1.0.1.1-iOUuBHgqUY06Xh8nfWydEaJRfVQ9C1F62pYRKHmYi0rctiVu3.SO8ZLVEu89T40RAw1.7zrzzT9NpQBQzbCHtw', } headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:127.0) Gecko/20100101 Firefox/127.0', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', 'Accept-Language': 'ru-RU,ru;q=0.8,en-US;q=0.5,en;q=0.3', # 'Accept-Encoding': 'gzip, deflate, br, zstd', 'Alt-Used': 'gptstore.ai', 'Connection': 'keep-alive', # 'Cookie': 'ph_phc_NwPasoEFOb5YHKAdjamWq0K4d6sDqTq89tSVWovv4nN_posthog=^%^7B^%^22distinct_id^%^22^%^3A^%^2201905113-076d-716e-b87c-cd201b325003^%^22^%^2C^%^22^%^24sesid^%^22^%^3A^%^5B1719347119985^%^2C^%^2201905113-0771-7511-9794-61cda198a886^%^22^%^2C1719347119985^%^5D^%^7D; __Host-next-auth.csrf-token=d5b912946563bf0a0d92830dc85284d85db9d7881ba506cbb9fd53a25d94ba0a^%^7C44bacf760340dc5d038e86b8a389002702bb6e9497b21afa24ed6f3ec10b42f7; __Secure-next-auth.callback-url=https^%^3A^%^2F^%^2Fgptstore.ai; cf_clearance=zIz4FMv3163pvCGGYwVWMIWDDwX34g0WVs05V4fUF_4-1719347121-1.0.1.1-iOUuBHgqUY06Xh8nfWydEaJRfVQ9C1F62pYRKHmYi0rctiVu3.SO8ZLVEu89T40RAw1.7zrzzT9NpQBQzbCHtw', 'Upgrade-Insecure-Requests': '1', 'Sec-Fetch-Dest': 'document', 'Sec-Fetch-Mode': 'navigate', 'Sec-Fetch-Site': 'none', 'Sec-Fetch-User': '?1', 'Priority': 'u=1', # Requests doesn't support trailers # 'TE': 'trailers', } params = { 'page': f'{page}', } response = requests.get(url=url, params=params, cookies=cookies, headers=headers) # with open('gptstore_ai_gpts.html', 'w', encoding='utf-8') as file: # file.write(response.text) soup = BeautifulSoup(response.text, 'lxml') ul_main_tag = soup.find('ul', class_='grid gap-2 sm:grid-cols-2 md:grid-cols-3 lg:grid-cols-4') li_main_tag = ul_main_tag.find_all('li') if li_main_tag: print("Найден li с указанным классом:") for li in li_main_tag: #### работа с изображением #### img_source = li.find('img').get('src') logger.debug(img_source) image_data = image_save(img_source) image_name = image_data[0] image_mime_type = image_data[1] to_russ_text = li.find('div', class_='overflow-hidden text-xs line-clamp-3').text # russ_text = translate_text(to_russ_text) plagin_name = li.find('a').text plagin_href = li.find('a').get('href') logger.debug(f'plagin name: {plagin_name} || plagin href: {plagin_href}') name_from_href = plagin_href.split('/')[-1] pl_data = get_plagin_data('https://gptstore.ai' + plagin_href, logger) russ_text = translate_text(pl_data[0]) if russ_text is not None: # plagin_text += russ_text plugin_data = { "title": plagin_name, "benefits": russ_text, "user_reviews": [ {"name": "Имя пользователя 1", "review": "Отзыв 1"}, {"name": "Имя пользователя 2", "review": "Отзыв 2"} ], "plugin_url": pl_data[1] } else: # plagin_text += to_russ_text plugin_data = { "title": plagin_name, "benefits": to_russ_text, "user_reviews": [ {"name": "Имя пользователя 1", "review": "Отзыв 1"}, {"name": "Имя пользователя 2", "review": "Отзыв 2"} ], "plugin_url": pl_data[1] } plagin_text = generate_page_content(plugin_data, image_data[0]) print(plagin_text) # Замена пробелов на дефисы и приведение к нижнему регистру # post_name = plagin_name.replace(" ", "-").lower() # post_name = plagin_name.replace(".", "").lower() # post_name = plagin_name.replace(":", "").lower() # Оставляем только буквы и цифры post_name = re.sub(r'[^a-zA-Z0-9\s]', '', plagin_name) # Заменяем пробелы на дефисы post_name = post_name.replace(" ", "-").lower() post_name = post_name.replace(" ", "-").lower() post_name = post_name.replace(" ", "-").lower() # Удаляем дефисы в начале и в конце строки post_name = re.sub(r'^-+|-+$', '', post_name) # Получаем текущие дату и время now = datetime.now() # Форматируем дату и время в нужный формат formatted_date_time = now.strftime("%Y-%m-%d %H:%M:%S") print(formatted_date_time) if checkHrefText(name_from_href): print("есть уже этот плагин в базе") else: # вставка в таблицу wp_posts inserted_post_id = addNewPost(formatted_date_time, plagin_text, plagin_name, '', 'publish', 'open', 'open', post_name, '', '', '', 0, 'post') guid = f'http://gpt-plagin.local/?p={inserted_post_id}' updatePostGuid(guid, inserted_post_id) if image_name is not None: image_name_without = image_name.split('.')[0] inserted_img_id = addNewPost(formatted_date_time, '', image_name_without, '', 'inherit', 'open', 'closed', image_name_without, '', '', '', inserted_post_id, 'attachment') year = now.strftime("%Y") month = now.strftime("%m") image_guid = f'http://gpt-plagin.local/wp-content/uploads/{year}/{month}/{image_name}' updatePostGuidMimetype(image_guid, image_mime_type, inserted_img_id) revision_post_name = f'{inserted_post_id}-revision-v1' inserted_revision_id = addNewPost(formatted_date_time, plagin_text, plagin_name, '', 'inherit', 'closed', 'closed', revision_post_name, '', '', '', inserted_post_id, 'revision') revision_guid = f'http://gpt-plagin.local/?p={inserted_revision_id}' updatePostGuid(revision_guid, inserted_revision_id) # вставка в таблицу wp_postmeta # Получение текущей временной метки в секундах с начала эпохи Unix current_timestamp = int(time.time()) meta_time = f'{current_timestamp}:1' addNewPostmeta(inserted_post_id, '_edit_lock', meta_time) if image_name is not None: meta_image = f'{year}/{month}/{image_name}' addNewPostmeta(inserted_img_id, '_wp_attached_file', meta_image) wp_attachment_metadata = create_attach_metadata_hand(image_name, year, month, image_mime_type) print(wp_attachment_metadata) addNewPostmeta(inserted_img_id, 'wp_attachment_metadata', wp_attachment_metadata) addNewPostmeta(inserted_post_id, '_thumbnail_id', inserted_img_id) addNewTermRelationship(inserted_post_id, 1) if russ_text is not None: addNewPlaginText(inserted_post_id, russ_text) else: addNewPlaginText(inserted_post_id, to_russ_text) addNewHrefText(inserted_post_id, name_from_href) time.sleep(12) else: print("li с указанным классом не найден.") break page += 1