import requests from bs4 import BeautifulSoup from PIL import Image from io import BytesIO import time from datetime import datetime import re from loguru import logger # импорты из моих модулей from handlers.img_save_hand import image_save from handlers.translate_text import translate_text from handlers.create_attachmtd_hand import create_attach_metadata_hand, generate_page_content from db.wp_posts_db import addNewPost, updatePostGuid, updatePostGuidMimetype, checkPlagin from db.wp_postmeta_db import addNewPostmeta from db.wp_term_relationships_db import addNewTermRelationship logger.add('logs/logs.log', rotation='100 mb', level='DEBUG') def get_site(url): page = 38 while True: logger.debug(f"page - {page}") cookies = { 'ph_phc_NwPasoEFOb5YHKAdjamWq0K4d6sDqTq89tSVWovv4nN_posthog': '^%^7B^%^22distinct_id^%^22^%^3A^%^2201905113-076d-716e-b87c-cd201b325003^%^22^%^2C^%^22^%^24sesid^%^22^%^3A^%^5B1719347119985^%^2C^%^2201905113-0771-7511-9794-61cda198a886^%^22^%^2C1719347119985^%^5D^%^7D', '__Host-next-auth.csrf-token': 'd5b912946563bf0a0d92830dc85284d85db9d7881ba506cbb9fd53a25d94ba0a^%^7C44bacf760340dc5d038e86b8a389002702bb6e9497b21afa24ed6f3ec10b42f7', '__Secure-next-auth.callback-url': 'https^%^3A^%^2F^%^2Fgptstore.ai', 'cf_clearance': 'zIz4FMv3163pvCGGYwVWMIWDDwX34g0WVs05V4fUF_4-1719347121-1.0.1.1-iOUuBHgqUY06Xh8nfWydEaJRfVQ9C1F62pYRKHmYi0rctiVu3.SO8ZLVEu89T40RAw1.7zrzzT9NpQBQzbCHtw', } headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:127.0) Gecko/20100101 Firefox/127.0', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', 'Accept-Language': 'ru-RU,ru;q=0.8,en-US;q=0.5,en;q=0.3', # 'Accept-Encoding': 'gzip, deflate, br, zstd', 'Alt-Used': 'gptstore.ai', 'Connection': 'keep-alive', # 'Cookie': 'ph_phc_NwPasoEFOb5YHKAdjamWq0K4d6sDqTq89tSVWovv4nN_posthog=^%^7B^%^22distinct_id^%^22^%^3A^%^2201905113-076d-716e-b87c-cd201b325003^%^22^%^2C^%^22^%^24sesid^%^22^%^3A^%^5B1719347119985^%^2C^%^2201905113-0771-7511-9794-61cda198a886^%^22^%^2C1719347119985^%^5D^%^7D; __Host-next-auth.csrf-token=d5b912946563bf0a0d92830dc85284d85db9d7881ba506cbb9fd53a25d94ba0a^%^7C44bacf760340dc5d038e86b8a389002702bb6e9497b21afa24ed6f3ec10b42f7; __Secure-next-auth.callback-url=https^%^3A^%^2F^%^2Fgptstore.ai; cf_clearance=zIz4FMv3163pvCGGYwVWMIWDDwX34g0WVs05V4fUF_4-1719347121-1.0.1.1-iOUuBHgqUY06Xh8nfWydEaJRfVQ9C1F62pYRKHmYi0rctiVu3.SO8ZLVEu89T40RAw1.7zrzzT9NpQBQzbCHtw', 'Upgrade-Insecure-Requests': '1', 'Sec-Fetch-Dest': 'document', 'Sec-Fetch-Mode': 'navigate', 'Sec-Fetch-Site': 'none', 'Sec-Fetch-User': '?1', 'Priority': 'u=1', # Requests doesn't support trailers # 'TE': 'trailers', } params = { 'page': f'{page}', } response = requests.get(url=url, params=params, cookies=cookies, headers=headers) # with open('gptstore_ai_gpts.html', 'w', encoding='utf-8') as file: # file.write(response.text) soup = BeautifulSoup(response.text, 'lxml') ul_main_tag = soup.find('ul', class_='grid gap-2 sm:grid-cols-2 md:grid-cols-3 lg:grid-cols-4') li_main_tag = ul_main_tag.find_all('li') if li_main_tag: print("Найден li с указанным классом:") for li in li_main_tag: #### работа с изображением #### img_source = li.find('img').get('src') logger.debug(img_source) image_data = image_save(img_source) image_name = image_data[0] image_mime_type = image_data[1] to_russ_text = li.find('div', class_='overflow-hidden text-xs line-clamp-3').text russ_text = translate_text(to_russ_text) plagin_name = li.find('a').text plagin_href = li.find('a').get('href') plagin_text = '' plagin_text += '

' if russ_text is not None: # plagin_text += russ_text plugin_data = { "title": plagin_name, "benefits": russ_text, "user_reviews": [ {"name": "Имя пользователя 1", "review": "Отзыв 1"}, {"name": "Имя пользователя 2", "review": "Отзыв 2"} ] } plagin_text += generate_page_content(plugin_data) else: # plagin_text += to_russ_text plugin_data = { "title": plagin_name, "benefits": to_russ_text, "user_reviews": [ {"name": "Имя пользователя 1", "review": "Отзыв 1"}, {"name": "Имя пользователя 2", "review": "Отзыв 2"} ] } plagin_text += generate_page_content(plugin_data) plagin_text += '

' plagin_text += '' print(plagin_text) logger.debug(plagin_name) # Замена пробелов на дефисы и приведение к нижнему регистру # post_name = plagin_name.replace(" ", "-").lower() # post_name = plagin_name.replace(".", "").lower() # post_name = plagin_name.replace(":", "").lower() # Оставляем только буквы и цифры post_name = re.sub(r'[^a-zA-Z0-9\s]', '', plagin_name) # Заменяем пробелы на дефисы post_name = post_name.replace(" ", "-").lower() post_name = post_name.replace(" ", "-") post_name = post_name.replace(" ", "-") # Удаляем дефисы в начале и в конце строки post_name = re.sub(r'^-+|-+$', '', post_name) # Получаем текущие дату и время now = datetime.now() # Форматируем дату и время в нужный формат formatted_date_time = now.strftime("%Y-%m-%d %H:%M:%S") print(formatted_date_time) if checkPlagin(plagin_name): print("есть уже этот плагин в базе") else: # вставка в таблицу wp_posts inserted_post_id = addNewPost(formatted_date_time, plagin_text, plagin_name, '', 'publish', 'closed', 'open', post_name, '', '', '', 0, 'post') guid = f'http://gpt-plagin.local/?p={inserted_post_id}' updatePostGuid(guid, inserted_post_id) if image_name is not None: image_name_without = image_name.split('.')[0] inserted_img_id = addNewPost(formatted_date_time, '', image_name_without, '', 'inherit', 'open', 'closed', image_name_without, '', '', '', inserted_post_id, 'attachment') year = now.strftime("%Y") month = now.strftime("%m") image_guid = f'http://2.56.91.28/wp-content/uploads/{year}/{month}/{image_name}' updatePostGuidMimetype(image_guid, image_mime_type, inserted_img_id) revision_post_name = f'{inserted_post_id}-revision-v1' inserted_revision_id = addNewPost(formatted_date_time, plagin_text, plagin_name, '', 'inherit', 'closed', 'closed', revision_post_name, '', '', '', inserted_post_id, 'revision') revision_guid = f'http://gpt-plagin.local/?p={inserted_revision_id}' updatePostGuid(revision_guid, inserted_revision_id) # вставка в таблицу wp_postmeta # Получение текущей временной метки в секундах с начала эпохи Unix current_timestamp = int(time.time()) meta_time = f'{current_timestamp}:1' addNewPostmeta(inserted_post_id, '_edit_lock', meta_time) if image_name is not None: meta_image = f'{year}/{month}/{image_name}' addNewPostmeta(inserted_img_id, '_wp_attached_file', meta_image) wp_attachment_metadata = create_attach_metadata_hand(image_name, year, month, image_mime_type) print(wp_attachment_metadata) addNewPostmeta(inserted_img_id, 'wp_attachment_metadata', wp_attachment_metadata) addNewPostmeta(inserted_post_id, '_thumbnail_id', inserted_img_id) addNewTermRelationship(inserted_post_id, 1) time.sleep(12) else: print("li с указанным классом не найден.") break page += 1 def go_to_plagin(href, page): url = 'https://gptstore.ai' + href cookies = { '__Host-next-auth.csrf-token': 'b83d30f36e9dc6027a53cc59c55d12483f200691d95945471b2bb065e1a661ae^%^7Ca446cd34e9758950abae63168e008562d34d96fc40fce26b5cb53eb74d51f0c6', '__Secure-next-auth.callback-url': 'https^%^3A^%^2F^%^2Fgptstore.ai', 'cf_clearance': '.9Pi94asgE2kfAwD1KJEQJg4PJB.cmrTl9jl4AOx.Us-1719400831-1.0.1.1-v5xD.26kqCLYfPU6X12hFujEHRVwXQHsj5QhHWvJktyurRMZZ3IE_TECPt.XXkRmH_XaOll1vjrizw8_A7rQdw', 'ph_phc_NwPasoEFOb5YHKAdjamWq0K4d6sDqTq89tSVWovv4nN_posthog': '^%^7B^%^22distinct_id^%^22^%^3A^%^2201905113-076d-716e-b87c-cd201b325003^%^22^%^2C^%^22^%^24sesid^%^22^%^3A^%^5B1719401628420^%^2C^%^2201905446-93f0-798d-b553-5375bdd1642f^%^22^%^2C1719400829936^%^5D^%^7D', } headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:127.0) Gecko/20100101 Firefox/127.0', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', 'Accept-Language': 'ru-RU,ru;q=0.8,en-US;q=0.5,en;q=0.3', # 'Accept-Encoding': 'gzip, deflate, br, zstd', 'Referer': f'https://gptstore.ai/gpts?page={page}', 'Connection': 'keep-alive', # 'Cookie': '__Host-next-auth.csrf-token=b83d30f36e9dc6027a53cc59c55d12483f200691d95945471b2bb065e1a661ae^%^7Ca446cd34e9758950abae63168e008562d34d96fc40fce26b5cb53eb74d51f0c6; __Secure-next-auth.callback-url=https^%^3A^%^2F^%^2Fgptstore.ai; cf_clearance=.9Pi94asgE2kfAwD1KJEQJg4PJB.cmrTl9jl4AOx.Us-1719400831-1.0.1.1-v5xD.26kqCLYfPU6X12hFujEHRVwXQHsj5QhHWvJktyurRMZZ3IE_TECPt.XXkRmH_XaOll1vjrizw8_A7rQdw; ph_phc_NwPasoEFOb5YHKAdjamWq0K4d6sDqTq89tSVWovv4nN_posthog=^%^7B^%^22distinct_id^%^22^%^3A^%^2201905113-076d-716e-b87c-cd201b325003^%^22^%^2C^%^22^%^24sesid^%^22^%^3A^%^5B1719401628420^%^2C^%^2201905446-93f0-798d-b553-5375bdd1642f^%^22^%^2C1719400829936^%^5D^%^7D', 'Upgrade-Insecure-Requests': '1', 'Sec-Fetch-Dest': 'document', 'Sec-Fetch-Mode': 'navigate', 'Sec-Fetch-Site': 'same-origin', 'Priority': 'u=1', 'Pragma': 'no-cache', 'Cache-Control': 'no-cache', } response = requests.get(url=url, cookies=cookies, headers=headers) def main(url): get_site(url) if __name__ == "__main__": url = 'https://gptstore.ai/gpts' main(url=url)