用GPT4V写AR交互

用GPT4V写AR交互 用GPT4V写AR交互 Modified January 11, 2024 Audio or video is not supported Please download the file and open it with the appropriate software. 未命名.mov · 16.22MB Audio or video is not supported Please download the file and open it with the appropriate software. copy 55188469 5B2D 4C20 8509 1A3D38A83918.MOV · 16.15MB copy 55188469 5B2D 4C20 8509 1A3D38A83918 00:00 选择良好的特征点 good new = p1[st == 1] good old = p0[st == 1] 绘制特征点的轨迹 for i, (new, old) in enumerate(zip(good new, good old)): a, b = new.ravel() a, b = int(a), int(b) 将坐标转换为整数 frame = cv2.putText(frame, 'A', (a, b), font, 1, (255, 0, 0), 2, cv2.LINE AA) cv2.imshow('Frame', frame) 按下'q'键退出循环 if cv2.waitKey(1) & 0xFF == ord('q'): break 更新上一帧的图像和特征点位置 old gray = frame gray.copy() p0 = good new.reshape( 1, 1, 2) 释放资源并关闭窗口 cap.release() cv2.destroyAllWindows() 原始代码 import cv2 import mediapipe as mp import time import base64 import requests import numpy as np 初始化MediaPipe手势识别 mp hands = mp.solutions.hands hands = mp hands.Hands(min detection confidence=0.5, min tracking confidence=0.5) 初始化摄像头 cap = cv2.VideoCapture(0) 假如想调用iPhone摄像头，这里改为1。 设置Lucas Kanade光流法参数，将返回内容固定在现实环境 lk params = dict(winSize=(15, 15), maxLevel=2, criteria=(cv2.TERM CRITERIA EPS | cv2.TERM CRITERIA COUNT, 10, 0.03)) 设置特征点检测的参数 feature params = dict(maxCorners=100, qualityLevel=0.3, minDistance=7, blockSize=7) def is pinch(hand landmarks): """检测是否进行了捏动作""" thumb tip = hand landmarks.landmark[mp hands.HandLandmark.THUMB TIP] index finger tip = hand landmarks.landmark[mp hands.HandLandmark.INDEX FINGER TIP] distance = ((thumb tip.x index finger tip.x) 2 + (thumb tip.y index finger tip.y) 2) 0.5 return distance < 0.02 这里调整pinch敏感度 def send image to gpt4 vision for questions(base64 image): """将图像发送到GPT 4 Vision API""" api key = "API Key" 请替换为您的API密钥 headers = { "Content Type": "application/json", "Authorization": f"Bearer {api key}" } payload = { "model": "gpt 4 vision preview", "messages": [ { "role": "user", "content": [ {"type": "text", "text": "Look at the image in the center and answer the questions in the picture briefly in 100 words."}, {"type": "image url", "image url": {"url": f"data:image/jpeg;base64,{base64 image}"}} ] } ], "max tokens": 150 } response = requests.post("https://api.ai gaochao.cn/v1/chat/completions", headers=headers, json=payload) 修改中转服务器地址 return response.json() def wrap text(text, line length=30): """将文本按指定长度换行""" return [text[i:i+line length] for i in range(0, len(text), line length)] show text = False text start time = None image captured = False captured frame = None gpt reply = None old gray = None p0 = None while cap.isOpened(): ret, frame = cap.read() if not ret: continue frame gray = cv2.cvtColor(frame, cv2.COLOR BGR2GRAY) if p0 is not None: p1, st, err = cv2.calcOpticalFlowPyrLK(old gray, frame gray, p0, None, lk params) good new = p1[st == 1] good old = p0[st == 1] if len(good new) 0: x, y = good new[0].ravel() if gpt reply is not None: wrapped text = wrap text(gpt reply, 30) y offset = y for line in wrapped text: cv2.putText(frame, line, (int(x), int(y offset)), cv2.FONT HERSHEY SIMPLEX, 0.7, (255, 255, 255), 2) y offset += 20 p0 = good new.reshape( 1, 1, 2) rgb frame = cv2.cvtColor(frame, cv2.COLOR BGR2RGB) results = hands.process(rgb frame) if results.multi hand landmarks: for hand landmarks in results.multi hand landmarks: if is pinch(hand landmarks): show text = True text start time = time.time() if not image captured: captured frame = frame.copy() old gray = frame gray.copy() p0 = cv2.goodFeaturesToTrack(old gray, mask=None, feature params) image captured = True if show text: cv2.putText(frame, 'Assistant: what can I help? (Listening...)', (10, 30), cv2.FONT HERSHEY SIMPLEX, 0.7, (0, 255, 0), 2) cv2.putText(frame, 'User said: Give me suggestion', (10, 70), cv2.FONT HERSHEY SIMPLEX, 0.7, (0, 255, 0), 2) if time.time() text start time 2: show text = False if image captured: retval, buffer = cv2.imencode('.jpg', captured frame) jpg as text = base64.b64encode(buffer).decode() gpt response = send image to gpt4 vision for questions(jpg as text) if 'choices' in gpt response and len(gpt response['choices']) 0: gpt reply = gpt response['choices'][0]['message']['content'] image captured = False 重置捕获状态 old gray = frame gray.copy() cv2.imshow('Frame', frame) if cv2.waitKey(5) & 0xFF == ord('q'): break cap.release() cv2.destroyAllWindows() 文章导读 文章导读 交流微信：26513486。 领域 任务 案例实践（含源码） 自然语言处理 语义分析 实体识别NER AI求职助手 文本相似度 AI做用户研究 文本分类 邮箱清洁工 文本总结 CES 2024 行业图谱 文本生成 搜索问答 GPT智能客服部署 机器翻译 / 编程 GPT具身对话机器人 计算机视觉 语义分析 物体检测、肢体跟踪 用GPT4V写AR交互 图像生成 图生图、文本图 用GPT开发相机APP AI求职助手 AI做用户研究 邮箱清洁工 CES 2024 行业图谱 GPT智能客服部署 GPT具身对话机器人 用GPT4V写AR交互 用GPT开发相机APP Audio or video is not supported Please download the file and open it with the appropriate software. 未命名.mov · 16.22MB Audio or video is not supported Please download the file and open it with the appropriate software. 未命名.mov · 16.22MB Audio or video is not supported Please download the file and open it with the appropriate software. copy 55188469 5B2D 4C20 8509 1A3D38A83918.MOV · 16.15MB copy 55188469 5B2D 4C20 8509 1A3D38A83918 00:00 Audio or video is not supported Please download the file and open it with the appropriate software. copy 55188469 5B2D 4C20 8509 1A3D38A83918.MOV · 16.15MB copy 55188469 5B2D 4C20 8509 1A3D38A83918 00:00 选择良好的特征点 good new = p1[st == 1] good old = p0[st == 1] 绘制特征点的轨迹 for i, (new, old) in enumerate(zip(good new, good old)): a, b = new.ravel() a, b = int(a), int(b) 将坐标转换为整数 frame = cv2.putText(frame, 'A', (a, b), font, 1, (255, 0, 0), 2, cv2.LINE AA) cv2.imshow('Frame', frame) 按下'q'键退出循环 if cv2.waitKey(1) & 0xFF == ord('q'): break 更新上一帧的图像和特征点位置 old gray = frame gray.copy() p0 = good new.reshape( 1, 1, 2) 释放资源并关闭窗口 cap.release() cv2.destroyAllWindows() 选择良好的特征点 good new = p1[st == 1] good old = p0[st == 1] 绘制特征点的轨迹 for i, (new, old) in enumerate(zip(good new, good old)): a, b = new.ravel() a, b = int(a), int(b) 将坐标转换为整数 frame = cv2.putText(frame, 'A', (a, b), font, 1, (255, 0, 0), 2, cv2.LINE AA) cv2.imshow('Frame', frame) 按下'q'键退出循环 if cv2.waitKey(1) & 0xFF == ord('q'): break 更新上一帧的图像和特征点位置 old gray = frame gray.copy() p0 = good new.reshape( 1, 1, 2) 释放资源并关闭窗口 cap.release() cv2.destroyAllWindows() 选择良好的特征点 good new = p1[st == 1] good old = p0[st == 1] 绘制特征点的轨迹 for i, (new, old) in enumerate(zip(good new, good old)): a, b = new.ravel() a, b = int(a), int(b) 将坐标转换为整数 frame = cv2.putText(frame, 'A', (a, b), font, 1, (255, 0, 0), 2, cv2.LINE AA) cv2.imshow('Frame', frame) 按下'q'键退出循环 if cv2.waitKey(1) & 0xFF == ord('q'): break 更新上一帧的图像和特征点位置 old gray = frame gray.copy() p0 = good new.reshape( 1, 1, 2) 释放资源并关闭窗口 cap.release() cv2.destroyAllWindows() 原始代码 import cv2 import mediapipe as mp import time import base64 import requests import numpy as np 初始化MediaPipe手势识别 mp hands = mp.solutions.hands hands = mp hands.Hands(min detection confidence=0.5, min tracking confidence=0.5) 初始化摄像头 cap = cv2.VideoCapture(0) 假如想调用iPhone摄像头，这里改为1。 设置Lucas Kanade光流法参数，将返回内容固定在现实环境 lk params = dict(winSize=(15, 15), maxLevel=2, criteria=(cv2.TERM CRITERIA EPS | cv2.TERM CRITERIA COUNT, 10, 0.03)) 设置特征点检测的参数 feature params = dict(maxCorners=100, qualityLevel=0.3, minDistance=7, blockSize=7) def is pinch(hand landmarks): """检测是否进行了捏动作""" thumb tip = hand landmarks.landmark[mp hands.HandLandmark.THUMB TIP] index finger tip = hand landmarks.landmark[mp hands.HandLandmark.INDEX FINGER TIP] distance = ((thumb tip.x index finger tip.x) 2 + (thumb tip.y index finger tip.y) 2) 0.5 return distance < 0.02 这里调整pinch敏感度 def send image to gpt4 vision for questions(base64 image): """将图像发送到GPT 4 Vision API""" api key = "API Key" 请替换为您的API密钥 headers = { "Content Type": "application/json", "Authorization": f"Bearer {api key}" } payload = { "model": "gpt 4 vision preview", "messages": [ { "role": "user", "content": [ {"type": "text", "text": "Look at the image in the center and answer the questions in the picture briefly in 100 words."}, {"type": "image url", "image url": {"url": f"data:image/jpeg;base64,{base64 image}"}} ] } ], "max tokens": 150 } response = requests.post("https://api.ai gaochao.cn/v1/chat/completions", headers=headers, json=payload) 修改中转服务器地址 return response.json() def wrap text(text, line length=30): """将文本按指定长度换行""" return [text[i:i+line length] for i in range(0, len(text), line length)] show text = False text start time = None image captured = False captured frame = None gpt reply = None old gray = None p0 = None while cap.isOpened(): ret, frame = cap.read() if not ret: continue frame gray = cv2.cvtColor(frame, cv2.COLOR BGR2GRAY) if p0 is not None: p1, st, err = cv2.calcOpticalFlowPyrLK(old gray, frame gray, p0, None, lk params) good new = p1[st == 1] good old = p0[st == 1] if len(good new) 0: x, y = good new[0].ravel() if gpt reply is not None: wrapped text = wrap text(gpt reply, 30) y offset = y for line in wrapped text: cv2.putText(frame, line, (int(x), int(y offset)), cv2.FONT HERSHEY SIMPLEX, 0.7, (255, 255, 255), 2) y offset += 20 p0 = good new.reshape( 1, 1, 2) rgb frame = cv2.cvtColor(frame, cv2.COLOR BGR2RGB) results = hands.process(rgb frame) if results.multi hand landmarks: for hand landmarks in results.multi hand landmarks: if is pinch(hand landmarks): show text = True text start time = time.time() if not image captured: captured frame = frame.copy() old gray = frame gray.copy() p0 = cv2.goodFeaturesToTrack(old gray, mask=None, feature params) image captured = True if show text: cv2.putText(frame, 'Assistant: what can I help? (Listening...)', (10, 30), cv2.FONT HERSHEY SIMPLEX, 0.7, (0, 255, 0), 2) cv2.putText(frame, 'User said: Give me suggestion', (10, 70), cv2.FONT HERSHEY SIMPLEX, 0.7, (0, 255, 0), 2) if time.time() text start time 2: show text = False if image captured: retval, buffer = cv2.imencode('.jpg', captured frame) jpg as text = base64.b64encode(buffer).decode() gpt response = send image to gpt4 vision for questions(jpg as text) if 'choices' in gpt response and len(gpt response['choices']) 0: gpt reply = gpt response['choices'][0]['message']['content'] image captured = False 重置捕获状态 old gray = frame gray.copy() cv2.imshow('Frame', frame) if cv2.waitKey(5) & 0xFF == ord('q'): break cap.release() cv2.destroyAllWindows() import cv2 import mediapipe as mp import time import base64 import requests import numpy as np 初始化MediaPipe手势识别 mp hands = mp.solutions.hands hands = mp hands.Hands(min detection confidence=0.5, min tracking confidence=0.5) 初始化摄像头 cap = cv2.VideoCapture(0) 假如想调用iPhone摄像头，这里改为1。 设置Lucas Kanade光流法参数，将返回内容固定在现实环境 lk params = dict(winSize=(15, 15), maxLevel=2, criteria=(cv2.TERM CRITERIA EPS | cv2.TERM CRITERIA COUNT, 10, 0.03)) 设置特征点检测的参数 feature params = dict(maxCorners=100, qualityLevel=0.3, minDistance=7, blockSize=7) def is pinch(hand landmarks): """检测是否进行了捏动作""" thumb tip = hand landmarks.landmark[mp hands.HandLandmark.THUMB TIP] index finger tip = hand landmarks.landmark[mp hands.HandLandmark.INDEX FINGER TIP] distance = ((thumb tip.x index finger tip.x) 2 + (thumb tip.y index finger tip.y) 2) 0.5 return distance < 0.02 这里调整pinch敏感度 def send image to gpt4 vision for questions(base64 image): """将图像发送到GPT 4 Vision API""" api key = "API Key" 请替换为您的API密钥 headers = { "Content Type": "application/json", "Authorization": f"Bearer {api key}" } payload = { "model": "gpt 4 vision preview", "messages": [ { "role": "user", "content": [ {"type": "text", "text": "Look at the image in the center and answer the questions in the picture briefly in 100 words."}, {"type": "image url", "image url": {"url": f"data:image/jpeg;base64,{base64 image}"}} ] } ], "max tokens": 150 } response = requests.post("https://api.ai gaochao.cn/v1/chat/completions", headers=headers, json=payload) 修改中转服务器地址 return response.json() def wrap text(text, line length=30): """将文本按指定长度换行""" return [text[i:i+line length] for i in range(0, len(text), line length)] show text = False text start time = None image captured = False captured frame = None gpt reply = None old gray = None p0 = None while cap.isOpened(): ret, frame = cap.read() if not ret: continue frame gray = cv2.cvtColor(frame, cv2.COLOR BGR2GRAY) if p0 is not None: p1, st, err = cv2.calcOpticalFlowPyrLK(old gray, frame gray, p0, None, lk params) good new = p1[st == 1] good old = p0[st == 1] if len(good new) 0: x, y = good new[0].ravel() if gpt reply is not None: wrapped text = wrap text(gpt reply, 30) y offset = y for line in wrapped text: cv2.putText(frame, line, (int(x), int(y offset)), cv2.FONT HERSHEY SIMPLEX, 0.7, (255, 255, 255), 2) y offset += 20 p0 = good new.reshape( 1, 1, 2) rgb frame = cv2.cvtColor(frame, cv2.COLOR BGR2RGB) results = hands.process(rgb frame) if results.multi hand landmarks: for hand landmarks in results.multi hand landmarks: if is pinch(hand landmarks): show text = True text start time = time.time() if not image captured: captured frame = frame.copy() old gray = frame gray.copy() p0 = cv2.goodFeaturesToTrack(old gray, mask=None, feature params) image captured = True if show text: cv2.putText(frame, 'Assistant: what can I help? (Listening...)', (10, 30), cv2.FONT HERSHEY SIMPLEX, 0.7, (0, 255, 0), 2) cv2.putText(frame, 'User said: Give me suggestion', (10, 70), cv2.FONT HERSHEY SIMPLEX, 0.7, (0, 255, 0), 2) if time.time() text start time 2: show text = False if image captured: retval, buffer = cv2.imencode('.jpg', captured frame) jpg as text = base64.b64encode(buffer).decode() gpt response = send image to gpt4 vision for questions(jpg as text) if 'choices' in gpt response and len(gpt response['choices']) 0: gpt reply = gpt response['choices'][0]['message']['content'] image captured = False 重置捕获状态 old gray = frame gray.copy() cv2.imshow('Frame', frame) if cv2.waitKey(5) & 0xFF == ord('q'): break cap.release() cv2.destroyAllWindows() import cv2 import mediapipe as mp import time import base64 import requests import numpy as np 初始化MediaPipe手势识别 mp hands = mp.solutions.hands hands = mp hands.Hands(min detection confidence=0.5, min tracking confidence=0.5) 初始化摄像头 cap = cv2.VideoCapture(0) 假如想调用iPhone摄像头，这里改为1。 设置Lucas Kanade光流法参数，将返回内容固定在现实环境 lk params = dict(winSize=(15, 15), maxLevel=2, criteria=(cv2.TERM CRITERIA EPS | cv2.TERM CRITERIA COUNT, 10, 0.03)) 设置特征点检测的参数 feature params = dict(maxCorners=100, qualityLevel=0.3, minDistance=7, blockSize=7) def is pinch(hand landmarks): """检测是否进行了捏动作""" thumb tip = hand landmarks.landmark[mp hands.HandLandmark.THUMB TIP] index finger tip = hand landmarks.landmark[mp hands.HandLandmark.INDEX FINGER TIP] distance = ((thumb tip.x index finger tip.x) 2 + (thumb tip.y index finger tip.y) 2) 0.5 return distance < 0.02 这里调整pinch敏感度 def send image to gpt4 vision for questions(base64 image): """将图像发送到GPT 4 Vision API""" api key = "API Key" 请替换为您的API密钥 headers = { "Content Type": "application/json", "Authorization": f"Bearer {api key}" } payload = { "model": "gpt 4 vision preview", "messages": [ { "role": "user", "content": [ {"type": "text", "text": "Look at the image in the center and answer the questions in the picture bri