{"version":2,"relationshipURLBase":"https://codingai-lec03.pages.dev/course/notebook-blocks/8d37281fe37f43618a865966b0297ef6.html","relationshipResolution":"Resolve relative relationship URLs against relationshipURLBase; complete records and absolute URLs are in blocks.","title":"GLIP 把语言接入对象级定位","url":"https://codingai-lec03.pages.dev/course/notebook-blocks/8d37281fe37f43618a865966b0297ef6.html","markdown":"https://codingai-lec03.pages.dev/course/notebook-blocks/8d37281fe37f43618a865966b0297ef6.md","pages":[{"id":"reading-opening","title":"GLIP 把语言接入对象级定位","unit":"GLIP 把语言接入对象级定位","unitId":"notebook-blocks/8d37281fe37f43618a865966b0297ef6","chapter":"course","number":1}],"aliases":[{"oldId":"b-9d55f29cf14110243a0aef60","targetId":"b-9d55f29cf14110243a0aef60","page":"reading-opening","operation":"move","beforeSHA256":"ca7e12f2ca2547e526317fa1c76ed9ef674c5d7a7eee748c10ae0de4d06ceb2a","afterSHA256":"ca7e12f2ca2547e526317fa1c76ed9ef674c5d7a7eee748c10ae0de4d06ceb2a","reason":"Same source body in authored semantic page"},{"oldId":"b-bd9a7766be3a88b4551cf5bc","targetId":"b-bd9a7766be3a88b4551cf5bc","page":"reading-opening","operation":"move","beforeSHA256":"6adf3f73b479296587fcf7d6aa0d73de7369896c7a2ebd64a0f6dd01844581a7","afterSHA256":"6adf3f73b479296587fcf7d6aa0d73de7369896c7a2ebd64a0f6dd01844581a7","reason":"Same source body in authored semantic page"},{"oldId":"b-e76d102d7c1d1cf8a1031519","targetId":"b-e76d102d7c1d1cf8a1031519","page":"reading-opening","operation":"move","beforeSHA256":"e91f24e4e4ca40e2099a0d81d30db6568f4bf2074b499eeee6e3959617bd97ac","afterSHA256":"e91f24e4e4ca40e2099a0d81d30db6568f4bf2074b499eeee6e3959617bd97ac","reason":"Same source body in authored semantic page"},{"oldId":"b-33a6f532ad78c4650df56a66","targetId":"b-33a6f532ad78c4650df56a66","page":"reading-opening","operation":"move","beforeSHA256":"29c03b145408b9ee4ac1eb7996a1fb4e7b0f911ff21fb7cd9f6ac0b8b3b44714","afterSHA256":"29c03b145408b9ee4ac1eb7996a1fb4e7b0f911ff21fb7cd9f6ac0b8b3b44714","reason":"Same source body in authored semantic page"},{"oldId":"b-147806585b1da8795e3a76bf","targetId":"b-147806585b1da8795e3a76bf","page":"reading-opening","operation":"move","beforeSHA256":"65100119afb6475cd7fa42899997aa1fd6b677b73c87e372086b97657e796edb","afterSHA256":"65100119afb6475cd7fa42899997aa1fd6b677b73c87e372086b97657e796edb","reason":"Same source body in authored semantic page"},{"oldId":"b-4bab784aa921f0fb2d0df4b5","targetId":"b-4bab784aa921f0fb2d0df4b5","page":"reading-opening","operation":"move","beforeSHA256":"769e106acd90a694434b7d5cc7214cd9f0ac370b2137f0450f2095ba8e8ecf54","afterSHA256":"769e106acd90a694434b7d5cc7214cd9f0ac370b2137f0450f2095ba8e8ecf54","reason":"Same source body in authored semantic page"},{"oldId":"b-de13bbc6e12da17bcb39a9ff","targetId":"b-de13bbc6e12da17bcb39a9ff","page":"reading-opening","operation":"move","beforeSHA256":"5fccf2697f7bb75f1c5b9f28774ade65ffb5193515ffc679352f0e3fa00803d4","afterSHA256":"5fccf2697f7bb75f1c5b9f28774ade65ffb5193515ffc679352f0e3fa00803d4","reason":"Same source body in authored semantic page"},{"oldId":"b-7ee0ddec5ea66cbd50e3bebd","targetId":"b-7ee0ddec5ea66cbd50e3bebd","page":"reading-opening","operation":"move","beforeSHA256":"7ff543e39aa88bbec9e002f3eceee9cbd26923179c4a1beb9a5025c6fad29bb7","afterSHA256":"7ff543e39aa88bbec9e002f3eceee9cbd26923179c4a1beb9a5025c6fad29bb7","reason":"Same source body in authored semantic page"},{"oldId":"b-d9bb89d63e5dd522e1f15ce5","targetId":"b-d9bb89d63e5dd522e1f15ce5","page":"reading-opening","operation":"move","beforeSHA256":"abf0150f51dfcccae9133778cb1863a0ccd5f7bbc9387b377717fd90bffbd7e1","afterSHA256":"abf0150f51dfcccae9133778cb1863a0ccd5f7bbc9387b377717fd90bffbd7e1","reason":"Same source body in authored semantic page"},{"oldId":"b-78b4c57057a0a88df9953898","targetId":"b-78b4c57057a0a88df9953898","page":"reading-opening","operation":"move","beforeSHA256":"3e8a946035d93cd2627ddc293d4c6f83a2e06f370f81c1adc0bc6050185b8a47","afterSHA256":"3e8a946035d93cd2627ddc293d4c6f83a2e06f370f81c1adc0bc6050185b8a47","reason":"Same source body in authored semantic page"},{"oldId":"b-03b4147cf63a553496edb8dd","targetId":"b-03b4147cf63a553496edb8dd","page":"reading-opening","operation":"move","beforeSHA256":"daaf0b69b1451fbccc45c0ac1a036b0e91a18bbd91e55650a2bba15dff988047","afterSHA256":"daaf0b69b1451fbccc45c0ac1a036b0e91a18bbd91e55650a2bba15dff988047","reason":"Same source body in authored semantic page"},{"oldId":"b-83d4edc2b50f241ebd5d74ba","targetId":"b-83d4edc2b50f241ebd5d74ba","page":"reading-opening","operation":"rewrite","beforeSHA256":"6abf062a0df864edc73aae136aa66836d1092aab255a4e7c0b6bc6c877d04fce","afterSHA256":"58a6267f320edf88f22da6dab0b12c07dd2293ae03712eb2a3c3a906c0d48eb9","reason":"Same source identity with figure semantics or reading context enriched"},{"oldId":"b-792c3c3fe806542dbb5d8ace","targetId":"b-792c3c3fe806542dbb5d8ace","page":"reading-opening","operation":"move","beforeSHA256":"ddbf9516cdb4c41c64652b966f94ecd8e2eb33ed6dbc8617bf610d708dc13fd9","afterSHA256":"ddbf9516cdb4c41c64652b966f94ecd8e2eb33ed6dbc8617bf610d708dc13fd9","reason":"Same source body in authored semantic page"},{"oldId":"b-efb3e202c2bf4320102a5bcf","targetId":"b-efb3e202c2bf4320102a5bcf","page":"reading-opening","operation":"move","beforeSHA256":"c65a370a8f6724513cacb5f23766c25784e0a7edb2f1019a61398dd2676a1e3c","afterSHA256":"c65a370a8f6724513cacb5f23766c25784e0a7edb2f1019a61398dd2676a1e3c","reason":"Same source body in authored semantic page"},{"oldId":"b-eed2da858db49814948ad796","targetId":"b-eed2da858db49814948ad796","page":"reading-opening","operation":"move","beforeSHA256":"9ccbfc05b07ddd4aaccfb97c7aa85e45d4d99601d4c1329e7184eea5fdffb5ca","afterSHA256":"9ccbfc05b07ddd4aaccfb97c7aa85e45d4d99601d4c1329e7184eea5fdffb5ca","reason":"Same source body in authored semantic page"},{"oldId":"b-388e2d7e7201c47c2cd5642e","targetId":"b-388e2d7e7201c47c2cd5642e","page":"reading-opening","operation":"move","beforeSHA256":"45987936f4a58bce97e53965bf722c5eae4b94758b1bcc9fbb2a564ad724edd6","afterSHA256":"45987936f4a58bce97e53965bf722c5eae4b94758b1bcc9fbb2a564ad724edd6","reason":"Same source body in authored semantic page"},{"oldId":"block-8d37281fe37f43618a865966b0297ef6--glip-把语言接入对象级定位","targetId":"b-147806585b1da8795e3a76bf","operation":"retain-source-identity"},{"oldId":"block-8d37281fe37f43618a865966b0297ef6--原始-llava-接入的是视觉向量","targetId":"b-7ee0ddec5ea66cbd50e3bebd","operation":"retain-source-identity"},{"oldId":"block-8d37281fe37f43618a865966b0297ef6--指令中的事实仍需追溯","targetId":"b-efb3e202c2bf4320102a5bcf","operation":"retain-source-identity"}],"blocks":[{"id":"b-9d55f29cf14110243a0aef60","type":"p","scope":"document","url":"https://codingai-lec03.pages.dev/course/notebook-blocks/8d37281fe37f43618a865966b0297ef6.html#b-9d55f29cf14110243a0aef60","content":"[CLS 11 · 语言定位与视觉模型接口](/course/notebooks/95169c3e298c4c819373bb4534f3dff9.html)","sha256":"ca7e12f2ca2547e526317fa1c76ed9ef674c5d7a7eee748c10ae0de4d06ceb2a","chapter":{"id":"course","title":"GLIP 把语言接入对象级定位"},"unit":{"id":"notebook-blocks/8d37281fe37f43618a865966b0297ef6","title":"GLIP 把语言接入对象级定位"},"page":{"id":"reading-opening","title":"GLIP 把语言接入对象级定位","number":1},"sectionTitle":"GLIP 把语言接入对象级定位","currentQuestion":"GLIP 把语言接入对象级定位","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":null,"nextBlock":"b-bd9a7766be3a88b4551cf5bc","prerequisites":[],"relatedBlocks":[]},{"id":"b-bd9a7766be3a88b4551cf5bc","type":"h1","scope":"document","url":"https://codingai-lec03.pages.dev/course/notebook-blocks/8d37281fe37f43618a865966b0297ef6.html#b-bd9a7766be3a88b4551cf5bc","content":"# GLIP 把语言接入对象级定位","sha256":"6adf3f73b479296587fcf7d6aa0d73de7369896c7a2ebd64a0f6dd01844581a7","chapter":{"id":"course","title":"GLIP 把语言接入对象级定位"},"unit":{"id":"notebook-blocks/8d37281fe37f43618a865966b0297ef6","title":"GLIP 把语言接入对象级定位"},"page":{"id":"reading-opening","title":"GLIP 把语言接入对象级定位","number":1},"sectionTitle":"GLIP 把语言接入对象级定位","currentQuestion":"GLIP 把语言接入对象级定位","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":"b-9d55f29cf14110243a0aef60","nextBlock":"b-e76d102d7c1d1cf8a1031519","prerequisites":[{"id":"b-9d55f29cf14110243a0aef60","url":"#b-9d55f29cf14110243a0aef60","role":"problem-context"}],"relatedBlocks":[]},{"id":"b-e76d102d7c1d1cf8a1031519","type":"p","scope":"document","url":"https://codingai-lec03.pages.dev/course/notebook-blocks/8d37281fe37f43618a865966b0297ef6.html#b-e76d102d7c1d1cf8a1031519","content":"按问题继续","sha256":"e91f24e4e4ca40e2099a0d81d30db6568f4bf2074b499eeee6e3959617bd97ac","chapter":{"id":"course","title":"GLIP 把语言接入对象级定位"},"unit":{"id":"notebook-blocks/8d37281fe37f43618a865966b0297ef6","title":"GLIP 把语言接入对象级定位"},"page":{"id":"reading-opening","title":"GLIP 把语言接入对象级定位","number":1},"sectionTitle":"GLIP 把语言接入对象级定位","currentQuestion":"GLIP 把语言接入对象级定位","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":"b-bd9a7766be3a88b4551cf5bc","nextBlock":"b-33a6f532ad78c4650df56a66","prerequisites":[{"id":"b-bd9a7766be3a88b4551cf5bc","url":"#b-bd9a7766be3a88b4551cf5bc","role":"section-title"},{"id":"b-9d55f29cf14110243a0aef60","url":"#b-9d55f29cf14110243a0aef60","role":"problem-context"}],"relatedBlocks":[]},{"id":"b-33a6f532ad78c4650df56a66","type":"p","scope":"document","url":"https://codingai-lec03.pages.dev/course/notebook-blocks/8d37281fe37f43618a865966b0297ef6.html#b-33a6f532ad78c4650df56a66","content":"目标/参照/关系、受控图像干预、短语区域学习、分割提示及连接器","sha256":"29c03b145408b9ee4ac1eb7996a1fb4e7b0f911ff21fb7cd9f6ac0b8b3b44714","chapter":{"id":"course","title":"GLIP 把语言接入对象级定位"},"unit":{"id":"notebook-blocks/8d37281fe37f43618a865966b0297ef6","title":"GLIP 把语言接入对象级定位"},"page":{"id":"reading-opening","title":"GLIP 把语言接入对象级定位","number":1},"sectionTitle":"GLIP 把语言接入对象级定位","currentQuestion":"GLIP 把语言接入对象级定位","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":"b-e76d102d7c1d1cf8a1031519","nextBlock":"b-147806585b1da8795e3a76bf","prerequisites":[{"id":"b-bd9a7766be3a88b4551cf5bc","url":"#b-bd9a7766be3a88b4551cf5bc","role":"section-title"},{"id":"b-9d55f29cf14110243a0aef60","url":"#b-9d55f29cf14110243a0aef60","role":"problem-context"}],"relatedBlocks":[]},{"id":"b-147806585b1da8795e3a76bf","type":"h2","scope":"document","url":"https://codingai-lec03.pages.dev/course/notebook-blocks/8d37281fe37f43618a865966b0297ef6.html#b-147806585b1da8795e3a76bf","content":"## GLIP 把语言接入对象级定位","sha256":"65100119afb6475cd7fa42899997aa1fd6b677b73c87e372086b97657e796edb","chapter":{"id":"course","title":"GLIP 把语言接入对象级定位"},"unit":{"id":"notebook-blocks/8d37281fe37f43618a865966b0297ef6","title":"GLIP 把语言接入对象级定位"},"page":{"id":"reading-opening","title":"GLIP 把语言接入对象级定位","number":1},"sectionTitle":"GLIP 把语言接入对象级定位","currentQuestion":"GLIP 把语言接入对象级定位","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":"b-33a6f532ad78c4650df56a66","nextBlock":"b-4bab784aa921f0fb2d0df4b5","prerequisites":[{"id":"b-bd9a7766be3a88b4551cf5bc","url":"#b-bd9a7766be3a88b4551cf5bc","role":"section-title"},{"id":"b-9d55f29cf14110243a0aef60","url":"#b-9d55f29cf14110243a0aef60","role":"problem-context"}],"relatedBlocks":[]},{"id":"b-4bab784aa921f0fb2d0df4b5","type":"p","scope":"document","url":"https://codingai-lec03.pages.dev/course/notebook-blocks/8d37281fe37f43618a865966b0297ef6.html#b-4bab784aa921f0fb2d0df4b5","content":"GLIP 统一对象检测与短语定位，用视觉—文字交互和区域对应监督学习位置。它不是把CLIP整图分数画成框。原论文在其协议下直接评估COCO为49.8 AP、LVIS为26.9 AP，并说明预训练未使用COCO图像。这是语言感知区域表示的正面迁移证据，不是本课程图上的推理。[方法、数据与论文来源](/course/blocks/glip-language-objective.html)。","sha256":"769e106acd90a694434b7d5cc7214cd9f0ac370b2137f0450f2095ba8e8ecf54","chapter":{"id":"course","title":"GLIP 把语言接入对象级定位"},"unit":{"id":"notebook-blocks/8d37281fe37f43618a865966b0297ef6","title":"GLIP 把语言接入对象级定位"},"page":{"id":"reading-opening","title":"GLIP 把语言接入对象级定位","number":1},"sectionTitle":"GLIP 把语言接入对象级定位","currentQuestion":"GLIP 把语言接入对象级定位","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":"b-147806585b1da8795e3a76bf","nextBlock":"b-de13bbc6e12da17bcb39a9ff","prerequisites":[{"id":"b-147806585b1da8795e3a76bf","url":"#b-147806585b1da8795e3a76bf","role":"section-title"},{"id":"b-9d55f29cf14110243a0aef60","url":"#b-9d55f29cf14110243a0aef60","role":"problem-context"}],"relatedBlocks":[]},{"id":"b-de13bbc6e12da17bcb39a9ff","type":"p","scope":"document","url":"https://codingai-lec03.pages.dev/course/notebook-blocks/8d37281fe37f43618a865966b0297ef6.html#b-de13bbc6e12da17bcb39a9ff","content":"语言定位先给框，再交分割接口时，要先核对对象身份：上游若选了蓝参照杯，后续精确分割蓝杯仍答错。目标对而边界差，才是另一段接口的问题。","sha256":"5fccf2697f7bb75f1c5b9f28774ade65ffb5193515ffc679352f0e3fa00803d4","chapter":{"id":"course","title":"GLIP 把语言接入对象级定位"},"unit":{"id":"notebook-blocks/8d37281fe37f43618a865966b0297ef6","title":"GLIP 把语言接入对象级定位"},"page":{"id":"reading-opening","title":"GLIP 把语言接入对象级定位","number":1},"sectionTitle":"GLIP 把语言接入对象级定位","currentQuestion":"GLIP 把语言接入对象级定位","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":"b-4bab784aa921f0fb2d0df4b5","nextBlock":"b-7ee0ddec5ea66cbd50e3bebd","prerequisites":[{"id":"b-147806585b1da8795e3a76bf","url":"#b-147806585b1da8795e3a76bf","role":"section-title"},{"id":"b-9d55f29cf14110243a0aef60","url":"#b-9d55f29cf14110243a0aef60","role":"problem-context"}],"relatedBlocks":[]},{"id":"b-7ee0ddec5ea66cbd50e3bebd","type":"h2","scope":"document","url":"https://codingai-lec03.pages.dev/course/notebook-blocks/8d37281fe37f43618a865966b0297ef6.html#b-7ee0ddec5ea66cbd50e3bebd","content":"## 原始 LLaVA 接入的是视觉向量","sha256":"7ff543e39aa88bbec9e002f3eceee9cbd26923179c4a1beb9a5025c6fad29bb7","chapter":{"id":"course","title":"GLIP 把语言接入对象级定位"},"unit":{"id":"notebook-blocks/8d37281fe37f43618a865966b0297ef6","title":"GLIP 把语言接入对象级定位"},"page":{"id":"reading-opening","title":"GLIP 把语言接入对象级定位","number":1},"sectionTitle":"GLIP 把语言接入对象级定位","currentQuestion":"GLIP 把语言接入对象级定位","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":"b-de13bbc6e12da17bcb39a9ff","nextBlock":"b-d9bb89d63e5dd522e1f15ce5","prerequisites":[{"id":"b-147806585b1da8795e3a76bf","url":"#b-147806585b1da8795e3a76bf","role":"section-title"},{"id":"b-9d55f29cf14110243a0aef60","url":"#b-9d55f29cf14110243a0aef60","role":"problem-context"}],"relatedBlocks":[]},{"id":"b-d9bb89d63e5dd522e1f15ce5","type":"p","scope":"document","url":"https://codingai-lec03.pages.dev/course/notebook-blocks/8d37281fe37f43618a865966b0297ef6.html#b-d9bb89d63e5dd522e1f15ce5","content":"CLIP ViT-L/14 的网格特征经可训练线性投影接入 Vicuna，与文字指令一起预测回答。不是先压成一个对象标签，也不是将一项整图相似度输入语言模型。按行写，$Z_v\\in\\mathbb R^{n\\times d_v}$、$W\\in\\mathbb R^{d_v\\times d_l}$，因此 $H_v=Z_vW\\in\\mathbb R^{n\\times d_l}$。","sha256":"abf0150f51dfcccae9133778cb1863a0ccd5f7bbc9387b377717fd90bffbd7e1","chapter":{"id":"course","title":"GLIP 把语言接入对象级定位"},"unit":{"id":"notebook-blocks/8d37281fe37f43618a865966b0297ef6","title":"GLIP 把语言接入对象级定位"},"page":{"id":"reading-opening","title":"GLIP 把语言接入对象级定位","number":1},"sectionTitle":"GLIP 把语言接入对象级定位","currentQuestion":"GLIP 把语言接入对象级定位","sourceSlides":[],"knowledgeBlock":null,"formulas":[{"latex":"Z_v\\in\\mathbb R^{n\\times d_v}","definitionContext":"GLIP 把语言接入对象级定位"},{"latex":"W\\in\\mathbb R^{d_v\\times d_l}","definitionContext":"GLIP 把语言接入对象级定位"},{"latex":"H_v=Z_vW\\in\\mathbb R^{n\\times d_l}","definitionContext":"GLIP 把语言接入对象级定位"}],"previousBlock":"b-7ee0ddec5ea66cbd50e3bebd","nextBlock":"b-78b4c57057a0a88df9953898","prerequisites":[{"id":"b-7ee0ddec5ea66cbd50e3bebd","url":"#b-7ee0ddec5ea66cbd50e3bebd","role":"section-title"},{"id":"b-9d55f29cf14110243a0aef60","url":"#b-9d55f29cf14110243a0aef60","role":"problem-context"}],"relatedBlocks":[]},{"id":"b-78b4c57057a0a88df9953898","type":"p","scope":"document","url":"https://codingai-lec03.pages.dev/course/notebook-blocks/8d37281fe37f43618a865966b0297ef6.html#b-78b4c57057a0a88df9953898","content":"按行堆叠 n 个视觉向量。每个向量有 `d_v` 个分量，因此 `Z_v` 是 `n×d_v` 矩阵；线性投影 `W` 是 `d_v×d_l` 矩阵。乘积 `H_v=Z_vW` 的形状是 `n×d_l`，视觉位置数量 n 保持不变，每个位置转换成语言模型能接收的 `d_l` 维向量。","sha256":"3e8a946035d93cd2627ddc293d4c6f83a2e06f370f81c1adc0bc6050185b8a47","chapter":{"id":"course","title":"GLIP 把语言接入对象级定位"},"unit":{"id":"notebook-blocks/8d37281fe37f43618a865966b0297ef6","title":"GLIP 把语言接入对象级定位"},"page":{"id":"reading-opening","title":"GLIP 把语言接入对象级定位","number":1},"sectionTitle":"GLIP 把语言接入对象级定位","currentQuestion":"GLIP 把语言接入对象级定位","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":"b-d9bb89d63e5dd522e1f15ce5","nextBlock":"b-03b4147cf63a553496edb8dd","prerequisites":[{"id":"b-7ee0ddec5ea66cbd50e3bebd","url":"#b-7ee0ddec5ea66cbd50e3bebd","role":"section-title"},{"id":"b-9d55f29cf14110243a0aef60","url":"#b-9d55f29cf14110243a0aef60","role":"problem-context"}],"relatedBlocks":[]},{"id":"b-03b4147cf63a553496edb8dd","type":"p","scope":"document","url":"https://codingai-lec03.pages.dev/course/notebook-blocks/8d37281fe37f43618a865966b0297ef6.html#b-03b4147cf63a553496edb8dd","content":"CLIP的比较空间不是Vicuna的词向量空间。维度匹配保证计算合法，任务训练建立使用关系；投影不会恢复视觉前处理已丢失的小字或细节。[接口的完整说明](/course/blocks/llava-visual-vector-interface.html)。","sha256":"daaf0b69b1451fbccc45c0ac1a036b0e91a18bbd91e55650a2bba15dff988047","chapter":{"id":"course","title":"GLIP 把语言接入对象级定位"},"unit":{"id":"notebook-blocks/8d37281fe37f43618a865966b0297ef6","title":"GLIP 把语言接入对象级定位"},"page":{"id":"reading-opening","title":"GLIP 把语言接入对象级定位","number":1},"sectionTitle":"GLIP 把语言接入对象级定位","currentQuestion":"GLIP 把语言接入对象级定位","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":"b-78b4c57057a0a88df9953898","nextBlock":"b-83d4edc2b50f241ebd5d74ba","prerequisites":[{"id":"b-7ee0ddec5ea66cbd50e3bebd","url":"#b-7ee0ddec5ea66cbd50e3bebd","role":"section-title"},{"id":"b-9d55f29cf14110243a0aef60","url":"#b-9d55f29cf14110243a0aef60","role":"problem-context"}],"relatedBlocks":[]},{"id":"b-83d4edc2b50f241ebd5d74ba","type":"table","scope":"document","url":"https://codingai-lec03.pages.dev/course/notebook-blocks/8d37281fe37f43618a865966b0297ef6.html#b-83d4edc2b50f241ebd5d74ba","content":"| 原始阶段 | 视觉编码器 | 线性投影 | 语言模型 |\n| --- | --- | --- | --- |\n| 特征对齐 | 冻结 | 更新 | 冻结 |\n| 视觉指令 | 仍冻结 | 更新 | 更新 |","sha256":"58a6267f320edf88f22da6dab0b12c07dd2293ae03712eb2a3c3a906c0d48eb9","chapter":{"id":"course","title":"GLIP 把语言接入对象级定位"},"unit":{"id":"notebook-blocks/8d37281fe37f43618a865966b0297ef6","title":"GLIP 把语言接入对象级定位"},"page":{"id":"reading-opening","title":"GLIP 把语言接入对象级定位","number":1},"sectionTitle":"GLIP 把语言接入对象级定位","currentQuestion":"GLIP 把语言接入对象级定位","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"table":{"headers":["原始阶段","视觉编码器","线性投影","语言模型"],"rows":[["原始阶段","视觉编码器","线性投影","语言模型"],["特征对齐","冻结","更新","冻结"],["视觉指令","仍冻结","更新","更新"]]},"previousBlock":"b-03b4147cf63a553496edb8dd","nextBlock":"b-792c3c3fe806542dbb5d8ace","prerequisites":[{"id":"b-7ee0ddec5ea66cbd50e3bebd","url":"#b-7ee0ddec5ea66cbd50e3bebd","role":"section-title"},{"id":"b-9d55f29cf14110243a0aef60","url":"#b-9d55f29cf14110243a0aef60","role":"problem-context"}],"relatedBlocks":[]},{"id":"b-792c3c3fe806542dbb5d8ace","type":"p","scope":"document","url":"https://codingai-lec03.pages.dev/course/notebook-blocks/8d37281fe37f43618a865966b0297ef6.html#b-792c3c3fe806542dbb5d8ace","content":"第一阶段先建立接口，第二阶段训练按图像与指令使用信息。两个阶段视觉都冻结；不能从论文“End-to-End”的标题推断视觉也更新。目标计在助手回答和停止位置，问题和图像是条件。[训练范围与原论文位置](/course/blocks/llava-two-stage-freezing.html)。","sha256":"ddbf9516cdb4c41c64652b966f94ecd8e2eb33ed6dbc8617bf610d708dc13fd9","chapter":{"id":"course","title":"GLIP 把语言接入对象级定位"},"unit":{"id":"notebook-blocks/8d37281fe37f43618a865966b0297ef6","title":"GLIP 把语言接入对象级定位"},"page":{"id":"reading-opening","title":"GLIP 把语言接入对象级定位","number":1},"sectionTitle":"GLIP 把语言接入对象级定位","currentQuestion":"GLIP 把语言接入对象级定位","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":"b-83d4edc2b50f241ebd5d74ba","nextBlock":"b-efb3e202c2bf4320102a5bcf","prerequisites":[{"id":"b-7ee0ddec5ea66cbd50e3bebd","url":"#b-7ee0ddec5ea66cbd50e3bebd","role":"section-title"},{"id":"b-9d55f29cf14110243a0aef60","url":"#b-9d55f29cf14110243a0aef60","role":"problem-context"}],"relatedBlocks":[]},{"id":"b-efb3e202c2bf4320102a5bcf","type":"h2","scope":"document","url":"https://codingai-lec03.pages.dev/course/notebook-blocks/8d37281fe37f43618a865966b0297ef6.html#b-efb3e202c2bf4320102a5bcf","content":"## 指令中的事实仍需追溯","sha256":"c65a370a8f6724513cacb5f23766c25784e0a7edb2f1019a61398dd2676a1e3c","chapter":{"id":"course","title":"GLIP 把语言接入对象级定位"},"unit":{"id":"notebook-blocks/8d37281fe37f43618a865966b0297ef6","title":"GLIP 把语言接入对象级定位"},"page":{"id":"reading-opening","title":"GLIP 把语言接入对象级定位","number":1},"sectionTitle":"GLIP 把语言接入对象级定位","currentQuestion":"GLIP 把语言接入对象级定位","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":"b-792c3c3fe806542dbb5d8ace","nextBlock":"b-eed2da858db49814948ad796","prerequisites":[{"id":"b-7ee0ddec5ea66cbd50e3bebd","url":"#b-7ee0ddec5ea66cbd50e3bebd","role":"section-title"},{"id":"b-9d55f29cf14110243a0aef60","url":"#b-9d55f29cf14110243a0aef60","role":"problem-context"}],"relatedBlocks":[]},{"id":"b-eed2da858db49814948ad796","type":"p","scope":"document","url":"https://codingai-lec03.pages.dev/course/notebook-blocks/8d37281fe37f43618a865966b0297ef6.html#b-eed2da858db49814948ad796","content":"原始LLaVA的部分指令资料由文本GPT-4依据caption、类别和框坐标制作，该步骤没有直接看原图。随后训练LLaVA才输入原图。若记录遗漏谁接触球、杯把如何遮挡书页，长回答也不能补足监督；应补标注、改问题或承认覆盖范围。","sha256":"9ccbfc05b07ddd4aaccfb97c7aa85e45d4d99601d4c1329e7184eea5fdffb5ca","chapter":{"id":"course","title":"GLIP 把语言接入对象级定位"},"unit":{"id":"notebook-blocks/8d37281fe37f43618a865966b0297ef6","title":"GLIP 把语言接入对象级定位"},"page":{"id":"reading-opening","title":"GLIP 把语言接入对象级定位","number":1},"sectionTitle":"GLIP 把语言接入对象级定位","currentQuestion":"GLIP 把语言接入对象级定位","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":"b-efb3e202c2bf4320102a5bcf","nextBlock":"b-388e2d7e7201c47c2cd5642e","prerequisites":[{"id":"b-efb3e202c2bf4320102a5bcf","url":"#b-efb3e202c2bf4320102a5bcf","role":"section-title"},{"id":"b-9d55f29cf14110243a0aef60","url":"#b-9d55f29cf14110243a0aef60","role":"problem-context"}],"relatedBlocks":[]},{"id":"b-388e2d7e7201c47c2cd5642e","type":"p","scope":"document","url":"https://codingai-lec03.pages.dev/course/notebook-blocks/8d37281fe37f43618a865966b0297ef6.html#b-388e2d7e7201c47c2cd5642e","content":"这种数据制作能把已有图文资料转成多种交互任务，其价值不只是数量，而是定义系统练习怎样使用图像。[数据来源与一个可做的检查](/course/blocks/llava-instruction-source.html)。循环网络与attention内部留第五讲，不作为本册起步前置。","sha256":"45987936f4a58bce97e53965bf722c5eae4b94758b1bcc9fbb2a564ad724edd6","chapter":{"id":"course","title":"GLIP 把语言接入对象级定位"},"unit":{"id":"notebook-blocks/8d37281fe37f43618a865966b0297ef6","title":"GLIP 把语言接入对象级定位"},"page":{"id":"reading-opening","title":"GLIP 把语言接入对象级定位","number":1},"sectionTitle":"GLIP 把语言接入对象级定位","currentQuestion":"GLIP 把语言接入对象级定位","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":"b-eed2da858db49814948ad796","nextBlock":null,"prerequisites":[{"id":"b-efb3e202c2bf4320102a5bcf","url":"#b-efb3e202c2bf4320102a5bcf","role":"section-title"},{"id":"b-9d55f29cf14110243a0aef60","url":"#b-9d55f29cf14110243a0aef60","role":"problem-context"}],"relatedBlocks":[]}]}