开发：CES 2024 行业图谱WIP

开发：CES 2024 行业图谱WIP 开发：CES 2024 行业图谱WIP Modified March 27 No access CES 2024 Industry Mapping.xlsx No access CES2024行业图谱.xlsx from openai import OpenAI Replace with actual OpenAI import def read json files(directory): """ Read all JSON files in the specified directory, sorted by time. """ files = [os.path.join(directory, f) for f in os.listdir(directory) if f.endswith('.json')] files.sort(key=lambda x: os.path.getmtime(x)) return files def summarize file(file path, client): """ Summarize file content using GPT model (in Chinese). """ with open(file path, 'r') as file: data = json.load(file) question = data.get("question", "Unknown") 从原始数据提取公司名字 filtered words = [word for word in question.split()[:3] if word.lower() not in ['ces', '2024','and','the','co.,','ltd.','technology','inc.','ltd','co.,ltd','co.,ltd.','electronics','limited','electronic','inc','technologies','llc','corporation','international','tech','group','gmbh','industrial','company','corp.']] print(filtered words) Combine raw content that contains any of the filtered question words 当爬虫数据包含公司名字，才会给到OPENAI raw contents = [item['raw content'] for item in data.get("response", {}).get("results", []) if 'raw content' in item and any(word.lower() in item['raw content'].lower() for word in filtered words)] combined content = ' '.join(raw contents) Truncate combined content to 10k words words = combined content.split() 单次请求限制8000个words，大约10k tokens。 if len(words) 8000: combined content = ' '.join(words[:8000]) GPT model call (replace with actual API call) prompt = f"Based on raw content as below, structurally and briefly summarize {question}. Summary content is strictly limited in 150 words. \n\nIf raw content is empty, just reply no infomation. \nIf there is no information about the brand mentioned in front, just reply no infomation. CES is a exhibition event, not a company name or brand. The key words before the CES is the company name.\n\nOutput reference format:\n\nBrand or Company name\n\nProduct 1\n\n feature 1\n\n feature 2\n\n feature 3\n\n and so on\n\n Raw Content: {combined content}" 这里填Prompt，指定输出格式。 response = client.chat.completions.create( model="gpt 3.5 turbo 1106", 这里填模型 messages=[{"role": "system", "content": "You are a product expert who summarizes the specifications and features of products based on the raw content, ignoring irrelevant content from other brand. The summary should be limited to 150 words."}, {"role": "user", "content": prompt}], 这里定义openai的角色 temperature=0.5 这里让它基于爬虫数据归纳，而不是自己想象。数值1 2就是想象为主。 ) summary = response.choices[0].message.content print(summary) return question, summary def save summary(directory, file name, question, summary): """ Save summary to the specified directory. """ output path = os.path.join(directory, file name) with open(output path, 'w', encoding='utf 8') as file: json.dump({"Question": question, "Summary": summary}, file, indent=2, ensure ascii=False) def main(): crawler directory = 'Crawler' 这里设置爬虫原始数据目录 summary directory = 'Summary' 这里设置保存分析结果目录 openai api key = 'API KEY' 这里设置OPENAI API KEY。这个KEY如何申请看youtube。 client = OpenAI(api key=openai api key) json files = read json files(crawler directory) for file path in json files: question, summary = summarize file(file path, client) file name = os.path.basename(file path) save summary(summary directory, file name, question, summary) time.sleep(20) Run every 10 seconds GPT也有频率限制 if name == " main ": main() 幻觉修复 ChatGPT 3.5回应中有15%的幻觉，更换ChatGPT 4修复。 原因如下 • 数据质量低，没有厂商的信息，Tavily只找到CES overall的信息。 • ChatGPT 3.5无法区别品牌与CES之间关系，会让CES热点信息代替品牌信息。 解决方案 • 更换信息源，换成Azure Bing Search，增加信息覆盖范围。因请求量较大，请使用付费接口，避免免费额度不足导致中断。 • 更换大语言模型，换成ChatGPT 4.0。 Python代码执行 • 在Summary目录检索一些幻觉关键字，如“transparent”、“rabbit r1”等，这些都是CES 2014 热门词汇。 • 当json文件出现这些词汇，python会提取该文件的品牌名称，在Bing再搜索一次让ChatGPT4归纳修复。 import os import json import requests from openai import OpenAI Load your API keys from environment variables or config files azure subscription key = "API KEY" 填写Azure api key，用企业邮箱申请就可以申请免费1000次额度。 openai api key = "API KEY" 填写openai api key Initialize OpenAI client openai client = OpenAI(api key=openai api key) Directory containing JSON files json dir = "Summary" 需要检查的目录 Function to search using Bing def bing search(query): headers = {"Ocp Apim Subscription Key": azure subscription key} params = {"q": query, "textDecorations": True, "textFormat": "HTML"} response = requests.get("https://api.bing.microsoft.com/v7.0/search", headers=headers, params=params) return response.json() Function to summarize using GPT 3.5 def summarize with gpt(filtered results, question): if not filtered results: return "No relevant brand information found." prompt = f"Based on raw content as below, structurally and briefly summarize {question}. Summary content is strictly limited to 150 words.\nIf raw content is empty, just reply no infomation. \nIf there is no information about the brand mentioned in front, just reply no infomation. CES is a exhibition event, not a company name or brand. The key words before the CES is the company name.\n\nOutput reference format:\n\nBrand or Company name\n\nProduct 1\n\n feature 1\n\n feature 2\n\n feature 3\n\n and so on\n\n Raw Content:\n\n" prompt += " ".join(filtered results) print(prompt) response = openai client.chat.completions.create( model="gpt 4 1106 preview", 换了目前最好的模型基座 messages=[{"role": "system", "content": "You are a product expert who summarizes the specifications and features of products based on the raw content, ignoring irrelevant content from other brand. The summary should be limited to 150 words."}, {"role": "user", "content": prompt}], max tokens=150, 指定归纳的字数 temperature=0.5) 让AI遵循文本进行归纳分析 print(response.choices[0].message.content) return response.choices[0].message.content Process each JSON file in the directory for filename in os.listdir(json dir): if filename.endswith(".json"): file path = os.path.join(json dir, filename) with open(file path, 'r') as file: data = json.load(file) summary = data.get("Summary") if summary and ("transparent" in summary or "No information" in summary): 设置多个幻觉关键词 question = data.get("Question", "Unknown") 假如发现有幻觉内容，就提取公司名称 filtered words = [word for word in question.split()[:3] if word.lower() not in ['ces', '2024','and','the','co.,','ltd.','technology','inc.','ltd','co.,ltd','co.,ltd.','electronics','limited','electronic','inc','technologies','llc','corporation','international','tech','group','gmbh','industrial','company','corp.']] 过滤无用关键词后，再送到BING搜索 print(filtered words) search results = bing search(question) filtered results = [snippet["snippet"] for snippet in search results["webPages"]["value"] if any(word.lower() in snippet["snippet"].lower() for word in filtered words)] new summary = summarize with gpt(filtered results, question) GPT4根据BING内容修复 Update the Summary field data["Summary"] = new summary Save the updated JSON file with open(file path, 'w') as file: json.dump(data, file, indent=4) 合并输出 将多个关联文件合并 文章导读 You don't have permission to access this synced block. Please log in and try again. Log In No access CES 2024 Industry Mapping.xlsx No access CES 2024 Industry Mapping.xlsx No access CES2024行业图谱.xlsx No access CES2024行业图谱.xlsx from openai import OpenAI Replace with actual OpenAI import def read json files(directory): """ Read all JSON files in the specified directory, sorted by time. """ files = [os.path.join(directory, f) for f in os.listdir(directory) if f.endswith('.json')] files.sort(key=lambda x: os.path.getmtime(x)) return files def summarize file(file path, client): """ Summarize file content using GPT model (in Chinese). """ with open(file path, 'r') as file: data = json.load(file) question = data.get("question", "Unknown") 从原始数据提取公司名字 filtered words = [word for word in question.split()[:3] if word.lower() not in ['ces', '2024','and','the','co.,','ltd.','technology','inc.','ltd','co.,ltd','co.,ltd.','electronics','limited','electronic','inc','technologies','llc','corporation','international','tech','group','gmbh','industrial','company','corp.']] print(filtered words) Combine raw content that contains any of the filtered question words 当爬虫数据包含公司名字，才会给到OPENAI raw contents = [item['raw content'] for item in data.get("response", {}).get("results", []) if 'raw content' in item and any(word.lower() in item['raw content'].lower() for word in filtered words)] combined content = ' '.join(raw contents) Truncate combined content to 10k words words = combined content.split() 单次请求限制8000个words，大约10k tokens。 if len(words) 8000: combined content = ' '.join(words[:8000]) GPT model call (replace with actual API call) prompt = f"Based on raw content as below, structurally and briefly summarize {question}. Summary content is strictly limited in 150 words. \n\nIf raw content is empty, just reply no infomation. \nIf there is no information about the brand mentioned in front, just reply no infomation. CES is a exhibition event, not a company name or brand. The key words before the CES is the company name.\n\nOutput reference format:\n\nBrand or Company name\n\nProduct 1\n\n feature 1\n\n feature 2\n\n feature 3\n\n and so on\n\n Raw Content: {combined content}" 这里填Prompt，指定输出格式。 response = client.chat.completions.create( model="gpt 3.5 turbo 1106", 这里填模型 messages=[{"role": "system", "content": "You are a product expert who summarizes the specifications and features of products based on the raw content, ignoring irrelevant content from other brand. The summary should be limited to 150 words."}, {"role": "user", "content": prompt}], 这里定义openai的角色 temperature=0.5 这里让它基于爬虫数据归纳，而不是自己想象。数值1 2就是想象为主。 ) summary = response.choices[0].message.content print(summary) return question, summary def save summary(directory, file name, question, summary): """ Save summary to the specified directory. """ output path = os.path.join(directory, file name) with open(output path, 'w', encoding='utf 8') as file: json.dump({"Question": question, "Summary": summary}, file, indent=2, ensure ascii=False) def main(): crawler directory = 'Crawler' 这里设置爬虫原始数据目录 summary directory = 'Summary' 这里设置保存分析结果目录 openai api key = 'API KEY' 这里设置OPENAI API KEY。这个KEY如何申请看youtube。 client = OpenAI(api key=openai api key) json files = read json files(crawler directory) for file path in json files: question, summary = summarize file(file path, client) file name = os.path.basename(file path) save summary(summary directory, file name, question, summary) time.sleep(20) Run every 10 seconds GPT也有频率限制 if name == " main ": main() from openai import OpenAI Replace with actual OpenAI import def read json files(directory): """ Read all JSON files in the specified directory, sorted by time. """ files = [os.path.join(directory, f) for f in os.listdir(directory) if f.endswith('.json')] files.sort(key=lambda x: os.path.getmtime(x)) return files def summarize file(file path, client): """ Summarize file content using GPT model (in Chinese). """ with open(file path, 'r') as file: data = json.load(file) question = data.get("question", "Unknown") 从原始数据提取公司名字 filtered words = [word for word in question.split()[:3] if word.lower() not in ['ces', '2024','and','the','co.,','ltd.','technology','inc.','ltd','co.,ltd','co.,ltd.','electronics','limited','electronic','inc','technologies','llc','corporation','international','tech','group','gmbh','industrial','company','corp.']] print(filtered words) Combine raw content that contains any of the filtered question words 当爬虫数据包含公司名字，才会给到OPENAI raw contents = [item['raw content'] for item in data.get("response", {}).get("results", []) if 'raw content' in item and any(word.lower() in item['raw content'].lower() for word in filtered words)] combined content = ' '.join(raw contents) Truncate combined content to 10k words words = combined content.split() 单次请求限制8000个words，大约10k tokens。 if len(words) 8000: combined content = ' '.join(words[:8000]) GPT model call (replace with actual API call) prompt = f"Based on raw content as below, structurally and briefly summarize {question}. Summary content is strictly limited in 150 words. \n\nIf raw content is empty, just reply no infomation. \nIf there is no information about the brand mentioned in front, just reply no infomation. CES is a exhibition event, not a company name or brand. The key words before the CES is the company name.\n\nOutput reference format:\n\nBrand or Company name\n\nProduct 1\n\n feature 1\n\n feature 2\n\n feature 3\n\n and so on\n\n Raw Content: {combined content}" 这里填Prompt，指定输出格式。 response = client.chat.completions.create( model="gpt 3.5 turbo 1106", 这里填模型 messages=[{"role": "system", "content": "You are a product expert who summarizes the specifications and features of products based on the raw content, ignoring irrelevant content from other brand. The summary should be limited to 150 words."}, {"role": "user", "content": prompt}], 这里定义openai的角色 temperature=0.5 这里让它基于爬虫数据归纳，而不是自己想象。数值1 2就是想象为主。 ) summary = response.choices[0].message.content print(summary) return question, summary def save summary(directory, file name, question, summary): """ Save summary to the specified directory. """ output path = os.path.join(directory, file name) with open(output path, 'w', encoding='utf 8') as file: json.dump({"Question": question, "Summary": summary}, file, indent=2, ensure ascii=False) def main(): crawler directory = 'Crawler' 这里设置爬虫原始数据目录 summary directory = 'Summary' 这里设置保存分析结果目录 openai api key = 'API KEY' 这里设置OPENAI API KEY。这个KEY如何申请看youtube。 client = OpenAI(api key=openai api key) json files = read json files(crawler directory) for file path in json files: question, summary = summarize file(file path, client) file name = os.path.basename(file path) save summary(summary directory, file name, question, summary) time.sleep(20) Run every 10 seconds GPT也有频率限制 if name == " main ": main() from openai import OpenAI Replace with actual OpenAI import def read json files(directory): """ Read all JSON files in the specified directory, sorted by time. """ files = [os.path.join(directory, f) for f in os.listdir(directory) if f.endswith('.json')] files.sort(key=lambda x: os.path.getmtime(x)) return files def summarize file(file path, client): """ Summarize file content using GPT model (in Chinese). """ with open(file path, 'r') as file: data = json.load(file) question = data.get("question", "Unknown") 从原始数据提取公司名字 filtered words = [word for word in question.split()[:3] if word.lower() not in ['ces', '2024','and','the','co.,','ltd.','technology','inc.','ltd','co.,ltd','co.,ltd.','electronics','limited','electronic','inc','technologies','llc','corporation','international','tech','group','gmbh','industrial','company','corp.']] print(filtered words) Combine raw content that contains any of the filtered question words 当爬虫数据包含公司名字，才会给到OPENAI raw contents = [item['raw content'] for item in data.get("response", {}).get("results", []) if 'raw content' in item and any(word.lower() in item['raw content'].lower() for word in filtered words)] combined content = ' '.join(raw contents) Truncate combined content to 10k words words