{"version":2,"relationshipURLBase":"https://codingai-lec03.pages.dev/course/notebooks/95169c3e298c4c819373bb4534f3dff9.html","relationshipResolution":"Resolve relative relationship URLs against relationshipURLBase; complete records and absolute URLs are in blocks.","title":"CLS 11 · 语言定位与视觉模型接口","url":"https://codingai-lec03.pages.dev/course/notebooks/95169c3e298c4c819373bb4534f3dff9.html","markdown":"https://codingai-lec03.pages.dev/course/notebooks/95169c3e298c4c819373bb4534f3dff9.md","pages":[{"id":"reading-opening","title":"CLS 11 · 语言定位与视觉模型接口","unit":"语言怎样选择当前需要的视觉信息？","unitId":"language-local-and-question","chapter":"III","number":1},{"id":"reading-block-91e584a383d94204a682f2b511a70e7a","title":"蓝杯是参照物，目标是它左边的杯子","unit":"语言怎样选择当前需要的视觉信息？","unitId":"language-local-and-question","chapter":"III","number":2},{"id":"reading-block-8d37281fe37f43618a865966b0297ef6","title":"GLIP 把语言接入对象级定位","unit":"语言怎样选择当前需要的视觉信息？","unitId":"language-local-and-question","chapter":"III","number":3},{"id":"reading-block-30c185dacfcf4fb59e6a8b9c3ae7be20","title":"组合接口与选读边界","unit":"语言怎样选择当前需要的视觉信息？","unitId":"language-local-and-question","chapter":"III","number":4}],"aliases":[{"oldId":"b-7d09558a922134c2da0c70bc","targetId":"b-7d09558a922134c2da0c70bc","page":"reading-opening","operation":"move","beforeSHA256":"1cff69d4abbd446b56955fb5f767a3b547e4abbcfc064b4557454378d30839eb","afterSHA256":"1cff69d4abbd446b56955fb5f767a3b547e4abbcfc064b4557454378d30839eb","reason":"Same source body in authored semantic page"},{"oldId":"b-4c7b78b1177f9e42cd413e6e","targetId":"b-4c7b78b1177f9e42cd413e6e","page":"reading-opening","operation":"move","beforeSHA256":"e91f24e4e4ca40e2099a0d81d30db6568f4bf2074b499eeee6e3959617bd97ac","afterSHA256":"e91f24e4e4ca40e2099a0d81d30db6568f4bf2074b499eeee6e3959617bd97ac","reason":"Same source body in authored semantic page"},{"oldId":"b-bdb9dc635335f67fefa780b1","targetId":"b-bdb9dc635335f67fefa780b1","page":"reading-opening","operation":"move","beforeSHA256":"29c03b145408b9ee4ac1eb7996a1fb4e7b0f911ff21fb7cd9f6ac0b8b3b44714","afterSHA256":"29c03b145408b9ee4ac1eb7996a1fb4e7b0f911ff21fb7cd9f6ac0b8b3b44714","reason":"Same source body in authored semantic page"},{"oldId":"b-7ac21f5fc2502c73d7894945","targetId":"b-7ac21f5fc2502c73d7894945","page":"reading-opening","operation":"move","beforeSHA256":"f22152fd55dcdb0090b5653d37145694fd19388d31e10226549464fe8b483578","afterSHA256":"f22152fd55dcdb0090b5653d37145694fd19388d31e10226549464fe8b483578","reason":"Same source body in authored semantic page"},{"oldId":"b-fe5d080f907d9badb9e038d8","targetId":"b-fe5d080f907d9badb9e038d8","page":"reading-block-91e584a383d94204a682f2b511a70e7a","operation":"move","beforeSHA256":"e18d8768430e774f92dc3916497d9d427a70355e977b362b9314ea60616fb0e1","afterSHA256":"e18d8768430e774f92dc3916497d9d427a70355e977b362b9314ea60616fb0e1","reason":"Same source body in authored semantic page"},{"oldId":"b-e3d416a9ea1af5eddca7db31","targetId":"b-e3d416a9ea1af5eddca7db31","page":"reading-block-91e584a383d94204a682f2b511a70e7a","operation":"move","beforeSHA256":"bf1e55e9a75ef12f9e634addd9f94db3b0f9cd82e8834f84869c886e70e7ef29","afterSHA256":"bf1e55e9a75ef12f9e634addd9f94db3b0f9cd82e8834f84869c886e70e7ef29","reason":"Same source body in authored semantic page"},{"oldId":"b-2607796bf63ef42a8c5ec67f","targetId":"b-2607796bf63ef42a8c5ec67f","page":"reading-block-91e584a383d94204a682f2b511a70e7a","operation":"move","beforeSHA256":"4bfc858ff28e56e227636c7c2bdadd1d2316fb83976df74524f3df2db4570b0b","afterSHA256":"4bfc858ff28e56e227636c7c2bdadd1d2316fb83976df74524f3df2db4570b0b","reason":"Same source body in authored semantic page"},{"oldId":"b-a345ad0d6f90846643faf204","targetId":"b-a345ad0d6f90846643faf204","page":"reading-block-91e584a383d94204a682f2b511a70e7a","operation":"move","beforeSHA256":"aaff1d9b6d74904c4d401c4a88a691294eea0f6b7c617300b925968739e21a32","afterSHA256":"aaff1d9b6d74904c4d401c4a88a691294eea0f6b7c617300b925968739e21a32","reason":"Same source body in authored semantic page"},{"oldId":"b-25daecf78a143b5052e6b870","targetId":"b-25daecf78a143b5052e6b870","page":"reading-block-91e584a383d94204a682f2b511a70e7a","operation":"move","beforeSHA256":"102365b11f61aa181ef8a7259fbe6e2f4be9b44b76b16eb55cb7d1531f901e44","afterSHA256":"102365b11f61aa181ef8a7259fbe6e2f4be9b44b76b16eb55cb7d1531f901e44","reason":"Same source body in authored semantic page"},{"oldId":"b-1ee6c337c1b2d9e320f9a671","targetId":"b-1ee6c337c1b2d9e320f9a671","page":"reading-block-91e584a383d94204a682f2b511a70e7a","operation":"move","beforeSHA256":"31b0ec800c021f4bf78aead857c84965c79eb07ffd3c3ac7c56a9f0d920dfed1","afterSHA256":"31b0ec800c021f4bf78aead857c84965c79eb07ffd3c3ac7c56a9f0d920dfed1","reason":"Same source body in authored semantic page"},{"oldId":"b-8866e181a9bcad8bef65bce9","targetId":"b-8866e181a9bcad8bef65bce9","page":"reading-block-91e584a383d94204a682f2b511a70e7a","operation":"move","beforeSHA256":"0125d825d461dff4940899636ef2d3fe45ee0995675e43a390f6ef0ff3459929","afterSHA256":"0125d825d461dff4940899636ef2d3fe45ee0995675e43a390f6ef0ff3459929","reason":"Same source body in authored semantic page"},{"oldId":"b-bee0a4821c221f238d1671b9","targetId":"b-bee0a4821c221f238d1671b9","page":"reading-block-91e584a383d94204a682f2b511a70e7a","operation":"move","beforeSHA256":"010ebd449316c650880ce1e33e6a791f011cbfdf4378eaeca5eed0e8ff1003ea","afterSHA256":"010ebd449316c650880ce1e33e6a791f011cbfdf4378eaeca5eed0e8ff1003ea","reason":"Same source body in authored semantic page"},{"oldId":"b-604d9638c0c7127d5f501221","targetId":"b-604d9638c0c7127d5f501221","page":"reading-block-91e584a383d94204a682f2b511a70e7a","operation":"move","beforeSHA256":"b862cf21e15ec46a718158ed3a9e968312c4b922fe578f4b7d31f7fb91984d5f","afterSHA256":"b862cf21e15ec46a718158ed3a9e968312c4b922fe578f4b7d31f7fb91984d5f","reason":"Same source body in authored semantic page"},{"oldId":"b-44f6b644072afd768ea3269e","targetId":"b-44f6b644072afd768ea3269e","page":"reading-block-91e584a383d94204a682f2b511a70e7a","operation":"rewrite","beforeSHA256":"b22982855dbc39682d867be61649451b15b257e629a2bbb41ea530f24d60b9f0","afterSHA256":"df7f73ddc106b40fdf4921795bfdc8ca59a90a09e089b8078a3707a00175ce09","reason":"Same source identity with figure semantics or reading context enriched"},{"oldId":"b-6a7f80bc2117ba5e1b4b6d78","targetId":"b-6a7f80bc2117ba5e1b4b6d78","page":"reading-block-91e584a383d94204a682f2b511a70e7a","operation":"move","beforeSHA256":"a94bd4fd75881b7a4e93478d9df96ac10e44d32e7fd749a226a53112bb6849f4","afterSHA256":"de3dccea21f6c1e8c0cb018f1b6c858731e5aa0f6afce8812c286e4f9e87447d","reason":"Same source body in authored semantic page"},{"oldId":"b-c5856840ff545853637b3701","targetId":"b-c5856840ff545853637b3701","page":"reading-block-91e584a383d94204a682f2b511a70e7a","operation":"move","beforeSHA256":"711c38fb9ddf160f916c3b47509108309c3b93e9454902055559cc273c112c0a","afterSHA256":"711c38fb9ddf160f916c3b47509108309c3b93e9454902055559cc273c112c0a","reason":"Same source body in authored semantic page"},{"oldId":"b-2cca6a1ec1e4c8c973a64d86","targetId":"b-2cca6a1ec1e4c8c973a64d86","page":"reading-block-8d37281fe37f43618a865966b0297ef6","operation":"move","beforeSHA256":"65100119afb6475cd7fa42899997aa1fd6b677b73c87e372086b97657e796edb","afterSHA256":"65100119afb6475cd7fa42899997aa1fd6b677b73c87e372086b97657e796edb","reason":"Same source body in authored semantic page"},{"oldId":"b-318e15f5f756bdd11401e267","targetId":"b-318e15f5f756bdd11401e267","page":"reading-block-8d37281fe37f43618a865966b0297ef6","operation":"move","beforeSHA256":"769e106acd90a694434b7d5cc7214cd9f0ac370b2137f0450f2095ba8e8ecf54","afterSHA256":"769e106acd90a694434b7d5cc7214cd9f0ac370b2137f0450f2095ba8e8ecf54","reason":"Same source body in authored semantic page"},{"oldId":"b-4f87760bb0fc291affd89087","targetId":"b-4f87760bb0fc291affd89087","page":"reading-block-8d37281fe37f43618a865966b0297ef6","operation":"move","beforeSHA256":"5fccf2697f7bb75f1c5b9f28774ade65ffb5193515ffc679352f0e3fa00803d4","afterSHA256":"5fccf2697f7bb75f1c5b9f28774ade65ffb5193515ffc679352f0e3fa00803d4","reason":"Same source body in authored semantic page"},{"oldId":"b-4d86a0d3ac0420ae212e1429","targetId":"b-4d86a0d3ac0420ae212e1429","page":"reading-block-8d37281fe37f43618a865966b0297ef6","operation":"move","beforeSHA256":"7ff543e39aa88bbec9e002f3eceee9cbd26923179c4a1beb9a5025c6fad29bb7","afterSHA256":"7ff543e39aa88bbec9e002f3eceee9cbd26923179c4a1beb9a5025c6fad29bb7","reason":"Same source body in authored semantic page"},{"oldId":"b-221400e15aae416b3eab076a","targetId":"b-221400e15aae416b3eab076a","page":"reading-block-8d37281fe37f43618a865966b0297ef6","operation":"move","beforeSHA256":"abf0150f51dfcccae9133778cb1863a0ccd5f7bbc9387b377717fd90bffbd7e1","afterSHA256":"abf0150f51dfcccae9133778cb1863a0ccd5f7bbc9387b377717fd90bffbd7e1","reason":"Same source body in authored semantic page"},{"oldId":"b-f1c1c7613d15a5b7d96684aa","targetId":"b-f1c1c7613d15a5b7d96684aa","page":"reading-block-8d37281fe37f43618a865966b0297ef6","operation":"move","beforeSHA256":"3e8a946035d93cd2627ddc293d4c6f83a2e06f370f81c1adc0bc6050185b8a47","afterSHA256":"3e8a946035d93cd2627ddc293d4c6f83a2e06f370f81c1adc0bc6050185b8a47","reason":"Same source body in authored semantic page"},{"oldId":"b-59af4e040c68e3228114b3e8","targetId":"b-59af4e040c68e3228114b3e8","page":"reading-block-8d37281fe37f43618a865966b0297ef6","operation":"move","beforeSHA256":"daaf0b69b1451fbccc45c0ac1a036b0e91a18bbd91e55650a2bba15dff988047","afterSHA256":"daaf0b69b1451fbccc45c0ac1a036b0e91a18bbd91e55650a2bba15dff988047","reason":"Same source body in authored semantic page"},{"oldId":"b-6c97a641981ee9cb940b33af","targetId":"b-6c97a641981ee9cb940b33af","page":"reading-block-8d37281fe37f43618a865966b0297ef6","operation":"rewrite","beforeSHA256":"6abf062a0df864edc73aae136aa66836d1092aab255a4e7c0b6bc6c877d04fce","afterSHA256":"58a6267f320edf88f22da6dab0b12c07dd2293ae03712eb2a3c3a906c0d48eb9","reason":"Same source identity with figure semantics or reading context enriched"},{"oldId":"b-e2b78d0c0c7937a7078aa855","targetId":"b-e2b78d0c0c7937a7078aa855","page":"reading-block-8d37281fe37f43618a865966b0297ef6","operation":"move","beforeSHA256":"ddbf9516cdb4c41c64652b966f94ecd8e2eb33ed6dbc8617bf610d708dc13fd9","afterSHA256":"ddbf9516cdb4c41c64652b966f94ecd8e2eb33ed6dbc8617bf610d708dc13fd9","reason":"Same source body in authored semantic page"},{"oldId":"b-e552752a67eec1ef8b4db9ad","targetId":"b-e552752a67eec1ef8b4db9ad","page":"reading-block-8d37281fe37f43618a865966b0297ef6","operation":"move","beforeSHA256":"c65a370a8f6724513cacb5f23766c25784e0a7edb2f1019a61398dd2676a1e3c","afterSHA256":"c65a370a8f6724513cacb5f23766c25784e0a7edb2f1019a61398dd2676a1e3c","reason":"Same source body in authored semantic page"},{"oldId":"b-0102b4746bb52e56c9c8ee83","targetId":"b-0102b4746bb52e56c9c8ee83","page":"reading-block-8d37281fe37f43618a865966b0297ef6","operation":"move","beforeSHA256":"9ccbfc05b07ddd4aaccfb97c7aa85e45d4d99601d4c1329e7184eea5fdffb5ca","afterSHA256":"9ccbfc05b07ddd4aaccfb97c7aa85e45d4d99601d4c1329e7184eea5fdffb5ca","reason":"Same source body in authored semantic page"},{"oldId":"b-1b291c61c2b4a0871e9c9035","targetId":"b-1b291c61c2b4a0871e9c9035","page":"reading-block-8d37281fe37f43618a865966b0297ef6","operation":"move","beforeSHA256":"45987936f4a58bce97e53965bf722c5eae4b94758b1bcc9fbb2a564ad724edd6","afterSHA256":"45987936f4a58bce97e53965bf722c5eae4b94758b1bcc9fbb2a564ad724edd6","reason":"Same source body in authored semantic page"},{"oldId":"b-3689f55c577551cb3d752dc4","targetId":"b-3689f55c577551cb3d752dc4","page":"reading-block-30c185dacfcf4fb59e6a8b9c3ae7be20","operation":"move","beforeSHA256":"ea081a32377fa4e7891b015a053f4c782391d173702b678a690a469e35517814","afterSHA256":"ea081a32377fa4e7891b015a053f4c782391d173702b678a690a469e35517814","reason":"Same source body in authored semantic page"},{"oldId":"b-8a022d44bef14d5a50bb2b88","targetId":"b-8a022d44bef14d5a50bb2b88","page":"reading-block-30c185dacfcf4fb59e6a8b9c3ae7be20","operation":"move","beforeSHA256":"5fa92c124d5680ee446ca2019153d88daebd10b674b4494c5556722c5f7076de","afterSHA256":"5fa92c124d5680ee446ca2019153d88daebd10b674b4494c5556722c5f7076de","reason":"Same source body in authored semantic page"},{"oldId":"b-5180a42f4f08d641741569d2","targetId":"b-5180a42f4f08d641741569d2","page":"reading-block-30c185dacfcf4fb59e6a8b9c3ae7be20","operation":"move","beforeSHA256":"4fb0f211cc5ea6b1f697b08c4e3a4ab0675c3aa6420552c28873e1c9b47164ed","afterSHA256":"4fb0f211cc5ea6b1f697b08c4e3a4ab0675c3aa6420552c28873e1c9b47164ed","reason":"Same source body in authored semantic page"},{"oldId":"b-397da040c299a54386043eb7","targetId":"b-397da040c299a54386043eb7","page":"reading-block-30c185dacfcf4fb59e6a8b9c3ae7be20","operation":"move","beforeSHA256":"611142a0bb5d0a7d4cf516bc65f90ef3dc4787c6b456d246bb1b2683bfce1556","afterSHA256":"611142a0bb5d0a7d4cf516bc65f90ef3dc4787c6b456d246bb1b2683bfce1556","reason":"Same source body in authored semantic page"},{"oldId":"b-389c1e99ac7585f649feb66b","targetId":"b-389c1e99ac7585f649feb66b","page":"reading-block-30c185dacfcf4fb59e6a8b9c3ae7be20","operation":"move","beforeSHA256":"6bd6829cbb9417b5e60425f8792e5d6b4dd1d1e3ca7aa225e6e61fe3b920ebfd","afterSHA256":"6bd6829cbb9417b5e60425f8792e5d6b4dd1d1e3ca7aa225e6e61fe3b920ebfd","reason":"Same source body in authored semantic page"},{"oldId":"block-91e584a383d94204a682f2b511a70e7a--蓝杯是参照物目标是它左边的杯子","targetId":"b-25daecf78a143b5052e6b870","operation":"retain-source-identity"},{"oldId":"block-91e584a383d94204a682f2b511a70e7a--输入变化确实经过像素","targetId":"b-bee0a4821c221f238d1671b9","operation":"retain-source-identity"},{"oldId":"block-8d37281fe37f43618a865966b0297ef6--glip-把语言接入对象级定位","targetId":"b-2cca6a1ec1e4c8c973a64d86","operation":"retain-source-identity"},{"oldId":"block-8d37281fe37f43618a865966b0297ef6--原始-llava-接入的是视觉向量","targetId":"b-4d86a0d3ac0420ae212e1429","operation":"retain-source-identity"},{"oldId":"block-8d37281fe37f43618a865966b0297ef6--指令中的事实仍需追溯","targetId":"b-e552752a67eec1ef8b4db9ad","operation":"retain-source-identity"},{"oldId":"block-30c185dacfcf4fb59e6a8b9c3ae7be20--组合接口与选读边界","targetId":"b-3689f55c577551cb3d752dc4","operation":"retain-source-identity"}],"blocks":[{"id":"b-7d09558a922134c2da0c70bc","type":"h1","scope":"document","url":"https://codingai-lec03.pages.dev/course/notebooks/95169c3e298c4c819373bb4534f3dff9.html#b-7d09558a922134c2da0c70bc","content":"# CLS 11 · 语言定位与视觉模型接口","sha256":"1cff69d4abbd446b56955fb5f767a3b547e4abbcfc064b4557454378d30839eb","chapter":{"id":"III","title":"视觉与语言"},"unit":{"id":"language-local-and-question","title":"语言怎样选择当前需要的视觉信息？"},"page":{"id":"reading-opening","title":"CLS 11 · 语言定位与视觉模型接口","number":1},"sectionTitle":"CLS 11 · 语言定位与视觉模型接口","currentQuestion":"CLS 11 · 语言定位与视觉模型接口","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":null,"nextBlock":"b-4c7b78b1177f9e42cd413e6e","prerequisites":[{"id":"b-4c7b78b1177f9e42cd413e6e","url":"#b-4c7b78b1177f9e42cd413e6e","role":"problem-context"}],"relatedBlocks":[]},{"id":"b-4c7b78b1177f9e42cd413e6e","type":"p","scope":"document","url":"https://codingai-lec03.pages.dev/course/notebooks/95169c3e298c4c819373bb4534f3dff9.html#b-4c7b78b1177f9e42cd413e6e","content":"按问题继续","sha256":"e91f24e4e4ca40e2099a0d81d30db6568f4bf2074b499eeee6e3959617bd97ac","chapter":{"id":"III","title":"视觉与语言"},"unit":{"id":"language-local-and-question","title":"语言怎样选择当前需要的视觉信息？"},"page":{"id":"reading-opening","title":"CLS 11 · 语言定位与视觉模型接口","number":1},"sectionTitle":"CLS 11 · 语言定位与视觉模型接口","currentQuestion":"CLS 11 · 语言定位与视觉模型接口","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":"b-7d09558a922134c2da0c70bc","nextBlock":"b-bdb9dc635335f67fefa780b1","prerequisites":[{"id":"b-7d09558a922134c2da0c70bc","url":"#b-7d09558a922134c2da0c70bc","role":"section-title"}],"relatedBlocks":[]},{"id":"b-bdb9dc635335f67fefa780b1","type":"p","scope":"document","url":"https://codingai-lec03.pages.dev/course/notebooks/95169c3e298c4c819373bb4534f3dff9.html#b-bdb9dc635335f67fefa780b1","content":"目标/参照/关系、受控图像干预、短语区域学习、分割提示及连接器","sha256":"29c03b145408b9ee4ac1eb7996a1fb4e7b0f911ff21fb7cd9f6ac0b8b3b44714","chapter":{"id":"III","title":"视觉与语言"},"unit":{"id":"language-local-and-question","title":"语言怎样选择当前需要的视觉信息？"},"page":{"id":"reading-opening","title":"CLS 11 · 语言定位与视觉模型接口","number":1},"sectionTitle":"CLS 11 · 语言定位与视觉模型接口","currentQuestion":"CLS 11 · 语言定位与视觉模型接口","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":"b-4c7b78b1177f9e42cd413e6e","nextBlock":"b-7ac21f5fc2502c73d7894945","prerequisites":[{"id":"b-7d09558a922134c2da0c70bc","url":"#b-7d09558a922134c2da0c70bc","role":"section-title"},{"id":"b-4c7b78b1177f9e42cd413e6e","url":"#b-4c7b78b1177f9e42cd413e6e","role":"problem-context"}],"relatedBlocks":[]},{"id":"b-7ac21f5fc2502c73d7894945","type":"p","scope":"document","url":"https://codingai-lec03.pages.dev/course/notebooks/95169c3e298c4c819373bb4534f3dff9.html#b-7ac21f5fc2502c73d7894945","content":"可先阅读已有结果，再决定是否运行。重算时请核对各节记录的数据、参数、环境与日期。","sha256":"f22152fd55dcdb0090b5653d37145694fd19388d31e10226549464fe8b483578","chapter":{"id":"III","title":"视觉与语言"},"unit":{"id":"language-local-and-question","title":"语言怎样选择当前需要的视觉信息？"},"page":{"id":"reading-opening","title":"CLS 11 · 语言定位与视觉模型接口","number":1},"sectionTitle":"CLS 11 · 语言定位与视觉模型接口","currentQuestion":"CLS 11 · 语言定位与视觉模型接口","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":"b-bdb9dc635335f67fefa780b1","nextBlock":null,"prerequisites":[{"id":"b-7d09558a922134c2da0c70bc","url":"#b-7d09558a922134c2da0c70bc","role":"section-title"},{"id":"b-4c7b78b1177f9e42cd413e6e","url":"#b-4c7b78b1177f9e42cd413e6e","role":"problem-context"}],"relatedBlocks":[]},{"id":"b-fe5d080f907d9badb9e038d8","type":"p","scope":"block-91e584a383d94204a682f2b511a70e7a","url":"https://codingai-lec03.pages.dev/course/notebooks/95169c3e298c4c819373bb4534f3dff9.html#b-fe5d080f907d9badb9e038d8","content":"**学习路径：** [一句话指向哪个对象](/course/study/grounding-interface.html)","sha256":"e18d8768430e774f92dc3916497d9d427a70355e977b362b9314ea60616fb0e1","chapter":{"id":"III","title":"视觉与语言"},"unit":{"id":"language-local-and-question","title":"语言怎样选择当前需要的视觉信息？"},"page":{"id":"reading-block-91e584a383d94204a682f2b511a70e7a","title":"蓝杯是参照物，目标是它左边的杯子","number":2},"sectionTitle":"蓝杯是参照物，目标是它左边的杯子","currentQuestion":"蓝杯是参照物，目标是它左边的杯子","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":null,"nextBlock":"b-e3d416a9ea1af5eddca7db31","prerequisites":[],"relatedBlocks":[]},{"id":"b-e3d416a9ea1af5eddca7db31","type":"p","scope":"block-91e584a383d94204a682f2b511a70e7a","url":"https://codingai-lec03.pages.dev/course/notebooks/95169c3e298c4c819373bb4534f3dff9.html#b-e3d416a9ea1af5eddca7db31","content":"从[P63 语言定位](/course/pages/l3-r2-p63.html)读到[P64 视觉向量接口](/course/pages/l3-r2-p64.html)：语言既可以指定一个区域，也可以要求模型根据图像回答，但两者需要不同输出。","sha256":"bf1e55e9a75ef12f9e634addd9f94db3b0f9cd82e8834f84869c886e70e7ef29","chapter":{"id":"III","title":"视觉与语言"},"unit":{"id":"language-local-and-question","title":"语言怎样选择当前需要的视觉信息？"},"page":{"id":"reading-block-91e584a383d94204a682f2b511a70e7a","title":"蓝杯是参照物，目标是它左边的杯子","number":2},"sectionTitle":"蓝杯是参照物，目标是它左边的杯子","currentQuestion":"蓝杯是参照物，目标是它左边的杯子","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":"b-fe5d080f907d9badb9e038d8","nextBlock":"b-2607796bf63ef42a8c5ec67f","prerequisites":[{"id":"b-fe5d080f907d9badb9e038d8","url":"#b-fe5d080f907d9badb9e038d8","role":"problem-context"}],"relatedBlocks":[]},{"id":"b-2607796bf63ef42a8c5ec67f","type":"p","scope":"block-91e584a383d94204a682f2b511a70e7a","url":"https://codingai-lec03.pages.dev/course/notebooks/95169c3e298c4c819373bb4534f3dff9.html#b-2607796bf63ef42a8c5ec67f","content":"先用“蓝杯左边的杯子”分清目标、参照物和关系，再读GLIP的定位学习与原始LLaVA的两阶段连接。[LAB 15](https://deepnote.com/project/38755e4e-23f6-4f5f-8c68-8a2e4114b1c1/notebook/7dc83786d067465aa76264475db2c62d)是本册附属规则练习，能从像素和坐标逐步核对，不冒充预训练grounding推理。","sha256":"4bfc858ff28e56e227636c7c2bdadd1d2316fb83976df74524f3df2db4570b0b","chapter":{"id":"III","title":"视觉与语言"},"unit":{"id":"language-local-and-question","title":"语言怎样选择当前需要的视觉信息？"},"page":{"id":"reading-block-91e584a383d94204a682f2b511a70e7a","title":"蓝杯是参照物，目标是它左边的杯子","number":2},"sectionTitle":"蓝杯是参照物，目标是它左边的杯子","currentQuestion":"蓝杯是参照物，目标是它左边的杯子","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":"b-e3d416a9ea1af5eddca7db31","nextBlock":"b-a345ad0d6f90846643faf204","prerequisites":[{"id":"b-fe5d080f907d9badb9e038d8","url":"#b-fe5d080f907d9badb9e038d8","role":"problem-context"}],"relatedBlocks":[]},{"id":"b-a345ad0d6f90846643faf204","type":"p","scope":"block-91e584a383d94204a682f2b511a70e7a","url":"https://codingai-lec03.pages.dev/course/notebooks/95169c3e298c4c819373bb4534f3dff9.html#b-a345ad0d6f90846643faf204","content":"读完应能指出错误发生在哪个接口：对象选错、区域读错，还是视觉向量未被语言模型正确使用。SAM与其他连接器按回链深入，attention内部不作为本册前置。","sha256":"aaff1d9b6d74904c4d401c4a88a691294eea0f6b7c617300b925968739e21a32","chapter":{"id":"III","title":"视觉与语言"},"unit":{"id":"language-local-and-question","title":"语言怎样选择当前需要的视觉信息？"},"page":{"id":"reading-block-91e584a383d94204a682f2b511a70e7a","title":"蓝杯是参照物，目标是它左边的杯子","number":2},"sectionTitle":"蓝杯是参照物，目标是它左边的杯子","currentQuestion":"蓝杯是参照物，目标是它左边的杯子","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":"b-2607796bf63ef42a8c5ec67f","nextBlock":"b-25daecf78a143b5052e6b870","prerequisites":[{"id":"b-fe5d080f907d9badb9e038d8","url":"#b-fe5d080f907d9badb9e038d8","role":"problem-context"}],"relatedBlocks":[]},{"id":"b-25daecf78a143b5052e6b870","type":"h2","scope":"block-91e584a383d94204a682f2b511a70e7a","url":"https://codingai-lec03.pages.dev/course/notebooks/95169c3e298c4c819373bb4534f3dff9.html#b-25daecf78a143b5052e6b870","content":"## 蓝杯是参照物，目标是它左边的杯子","sha256":"102365b11f61aa181ef8a7259fbe6e2f4be9b44b76b16eb55cb7d1531f901e44","chapter":{"id":"III","title":"视觉与语言"},"unit":{"id":"language-local-and-question","title":"语言怎样选择当前需要的视觉信息？"},"page":{"id":"reading-block-91e584a383d94204a682f2b511a70e7a","title":"蓝杯是参照物，目标是它左边的杯子","number":2},"sectionTitle":"蓝杯是参照物，目标是它左边的杯子","currentQuestion":"蓝杯是参照物，目标是它左边的杯子","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":"b-a345ad0d6f90846643faf204","nextBlock":"b-1ee6c337c1b2d9e320f9a671","prerequisites":[{"id":"b-fe5d080f907d9badb9e038d8","url":"#b-fe5d080f907d9badb9e038d8","role":"problem-context"}],"relatedBlocks":[]},{"id":"b-1ee6c337c1b2d9e320f9a671","type":"p","scope":"block-91e584a383d94204a682f2b511a70e7a","url":"https://codingai-lec03.pages.dev/course/notebooks/95169c3e298c4c819373bb4534f3dff9.html#b-1ee6c337c1b2d9e320f9a671","content":"`cup left of blue cup` 分为目标类别cup、参照物blue cup、关系left-of。左红杯框为[15,19,50,58]、中心x=32.5；右蓝杯为[109,19,144,58]、中心x=126.5；书为[60,65,90,80]。采用半开边界框，关系定义为中心x严格小于，且目标不能是参照物本身。","sha256":"31b0ec800c021f4bf78aead857c84965c79eb07ffd3c3ac7c56a9f0d920dfed1","chapter":{"id":"III","title":"视觉与语言"},"unit":{"id":"language-local-and-question","title":"语言怎样选择当前需要的视觉信息？"},"page":{"id":"reading-block-91e584a383d94204a682f2b511a70e7a","title":"蓝杯是参照物，目标是它左边的杯子","number":2},"sectionTitle":"蓝杯是参照物，目标是它左边的杯子","currentQuestion":"蓝杯是参照物，目标是它左边的杯子","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":"b-25daecf78a143b5052e6b870","nextBlock":"b-8866e181a9bcad8bef65bce9","prerequisites":[{"id":"b-25daecf78a143b5052e6b870","url":"#b-25daecf78a143b5052e6b870","role":"section-title"},{"id":"b-fe5d080f907d9badb9e038d8","url":"#b-fe5d080f907d9badb9e038d8","role":"problem-context"}],"relatedBlocks":[]},{"id":"b-8866e181a9bcad8bef65bce9","type":"p","scope":"block-91e584a383d94204a682f2b511a70e7a","url":"https://codingai-lec03.pages.dev/course/notebooks/95169c3e298c4c819373bb4534f3dff9.html#b-8866e181a9bcad8bef65bce9","content":"先找唯一蓝杯，再筛其他cup：红杯满足32.5<126.5；书虽在左但类别不符。蓝杯缺失、多义或没有唯一目标时返回无唯一目标，不用参照杯顶替。若改成“整个框都在左侧”，需另用边界比较；关系语义要先固定。[完整坐标算例](/course/blocks/grounding-target-reference-relation.html)。","sha256":"0125d825d461dff4940899636ef2d3fe45ee0995675e43a390f6ef0ff3459929","chapter":{"id":"III","title":"视觉与语言"},"unit":{"id":"language-local-and-question","title":"语言怎样选择当前需要的视觉信息？"},"page":{"id":"reading-block-91e584a383d94204a682f2b511a70e7a","title":"蓝杯是参照物，目标是它左边的杯子","number":2},"sectionTitle":"蓝杯是参照物，目标是它左边的杯子","currentQuestion":"蓝杯是参照物，目标是它左边的杯子","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":"b-1ee6c337c1b2d9e320f9a671","nextBlock":"b-bee0a4821c221f238d1671b9","prerequisites":[{"id":"b-25daecf78a143b5052e6b870","url":"#b-25daecf78a143b5052e6b870","role":"section-title"},{"id":"b-fe5d080f907d9badb9e038d8","url":"#b-fe5d080f907d9badb9e038d8","role":"problem-context"}],"relatedBlocks":[]},{"id":"b-bee0a4821c221f238d1671b9","type":"h2","scope":"block-91e584a383d94204a682f2b511a70e7a","url":"https://codingai-lec03.pages.dev/course/notebooks/95169c3e298c4c819373bb4534f3dff9.html#b-bee0a4821c221f238d1671b9","content":"## 输入变化确实经过像素","sha256":"010ebd449316c650880ce1e33e6a791f011cbfdf4378eaeca5eed0e8ff1003ea","chapter":{"id":"III","title":"视觉与语言"},"unit":{"id":"language-local-and-question","title":"语言怎样选择当前需要的视觉信息？"},"page":{"id":"reading-block-91e584a383d94204a682f2b511a70e7a","title":"蓝杯是参照物，目标是它左边的杯子","number":2},"sectionTitle":"蓝杯是参照物，目标是它左边的杯子","currentQuestion":"蓝杯是参照物，目标是它左边的杯子","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":"b-8866e181a9bcad8bef65bce9","nextBlock":"b-604d9638c0c7127d5f501221","prerequisites":[{"id":"b-25daecf78a143b5052e6b870","url":"#b-25daecf78a143b5052e6b870","role":"section-title"},{"id":"b-fe5d080f907d9badb9e038d8","url":"#b-fe5d080f907d9badb9e038d8","role":"problem-context"}],"relatedBlocks":[]},{"id":"b-604d9638c0c7127d5f501221","type":"p","scope":"block-91e584a383d94204a682f2b511a70e7a","url":"https://codingai-lec03.pages.dev/course/notebooks/95169c3e298c4c819373bb4534f3dff9.html#b-604d9638c0c7127d5f501221","content":"[LAB 15](https://deepnote.com/workspace/ise3309jingao-5ba66b0a-4283-4f5a-ae06-3f68d3201e99/project/38755e4e-23f6-4f5f-8c68-8a2e4114b1c1/notebook/7dc83786d067465aa76264475db2c62d?utm_source=openai&utm_medium=mcp&utm_campaign=openaimcp&utm_content=7dc83786d067465aa76264475db2c62d&utm_term=get_notebook) 从RGB颜色阈值和连通域提框，再执行关系规则。它读取了图像，但不是预训练神经grounding。已有2026-09-22结果如下，本轮未重跑。","sha256":"b862cf21e15ec46a718158ed3a9e968312c4b922fe578f4b7d31f7fb91984d5f","chapter":{"id":"III","title":"视觉与语言"},"unit":{"id":"language-local-and-question","title":"语言怎样选择当前需要的视觉信息？"},"page":{"id":"reading-block-91e584a383d94204a682f2b511a70e7a","title":"蓝杯是参照物，目标是它左边的杯子","number":2},"sectionTitle":"蓝杯是参照物，目标是它左边的杯子","currentQuestion":"蓝杯是参照物，目标是它左边的杯子","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":"b-bee0a4821c221f238d1671b9","nextBlock":"b-44f6b644072afd768ea3269e","prerequisites":[{"id":"b-bee0a4821c221f238d1671b9","url":"#b-bee0a4821c221f238d1671b9","role":"section-title"},{"id":"b-fe5d080f907d9badb9e038d8","url":"#b-fe5d080f907d9badb9e038d8","role":"problem-context"}],"relatedBlocks":[]},{"id":"b-44f6b644072afd768ea3269e","type":"table","scope":"block-91e584a383d94204a682f2b511a70e7a","url":"https://codingai-lec03.pages.dev/course/notebooks/95169c3e298c4c819373bb4534f3dff9.html#b-44f6b644072afd768ea3269e","content":"| 条件 | 问句 | 目标 |\n| --- | --- | --- |\n| 左红右蓝 | 蓝杯左边的杯子 | 左红 |\n| 遮住左目标 | 同上 | 无唯一可见目标 |\n| 只交换红蓝像素 | 同上 | 无目标，蓝参照物在最左 |\n| 同一交换图改关系 | 蓝杯右边的杯子 | 右红 |","sha256":"df7f73ddc106b40fdf4921795bfdc8ca59a90a09e089b8078a3707a00175ce09","chapter":{"id":"III","title":"视觉与语言"},"unit":{"id":"language-local-and-question","title":"语言怎样选择当前需要的视觉信息？"},"page":{"id":"reading-block-91e584a383d94204a682f2b511a70e7a","title":"蓝杯是参照物，目标是它左边的杯子","number":2},"sectionTitle":"蓝杯是参照物，目标是它左边的杯子","currentQuestion":"蓝杯是参照物，目标是它左边的杯子","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"table":{"headers":["条件","问句","目标"],"rows":[["条件","问句","目标"],["左红右蓝","蓝杯左边的杯子","左红"],["遮住左目标","同上","无唯一可见目标"],["只交换红蓝像素","同上","无目标，蓝参照物在最左"],["同一交换图改关系","蓝杯右边的杯子","右红"]]},"previousBlock":"b-604d9638c0c7127d5f501221","nextBlock":"b-6a7f80bc2117ba5e1b4b6d78","prerequisites":[{"id":"b-bee0a4821c221f238d1671b9","url":"#b-bee0a4821c221f238d1671b9","role":"section-title"},{"id":"b-fe5d080f907d9badb9e038d8","url":"#b-fe5d080f907d9badb9e038d8","role":"problem-context"}],"relatedBlocks":[]},{"id":"b-6a7f80bc2117ba5e1b4b6d78","type":"p","scope":"block-91e584a383d94204a682f2b511a70e7a","url":"https://codingai-lec03.pages.dev/course/notebooks/95169c3e298c4c819373bb4534f3dff9.html#b-6a7f80bc2117ba5e1b4b6d78","content":"![像素干预与返回框](/assets/display/969fa088b392b7fb8a846fb3.webp)","sha256":"de3dccea21f6c1e8c0cb018f1b6c858731e5aa0f6afce8812c286e4f9e87447d","chapter":{"id":"III","title":"视觉与语言"},"unit":{"id":"language-local-and-question","title":"语言怎样选择当前需要的视觉信息？"},"page":{"id":"reading-block-91e584a383d94204a682f2b511a70e7a","title":"蓝杯是参照物，目标是它左边的杯子","number":2},"sectionTitle":"蓝杯是参照物，目标是它左边的杯子","currentQuestion":"蓝杯是参照物，目标是它左边的杯子","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":"b-44f6b644072afd768ea3269e","nextBlock":"b-c5856840ff545853637b3701","prerequisites":[{"id":"b-bee0a4821c221f238d1671b9","url":"#b-bee0a4821c221f238d1671b9","role":"section-title"},{"id":"b-fe5d080f907d9badb9e038d8","url":"#b-fe5d080f907d9badb9e038d8","role":"problem-context"}],"relatedBlocks":[]},{"id":"b-c5856840ff545853637b3701","type":"p","scope":"block-91e584a383d94204a682f2b511a70e7a","url":"https://codingai-lec03.pages.dev/course/notebooks/95169c3e298c4c819373bb4534f3dff9.html#b-c5856840ff545853637b3701","content":"代码断言干预区域外像素不变并保存差分。旧关键词规则错选蓝杯保留为失败；颜色规则只适用于有限场景，不能扩大为任意杯子识别。[干预、代码与证据身份](/course/blocks/grounding-pixel-interventions.html)。","sha256":"711c38fb9ddf160f916c3b47509108309c3b93e9454902055559cc273c112c0a","chapter":{"id":"III","title":"视觉与语言"},"unit":{"id":"language-local-and-question","title":"语言怎样选择当前需要的视觉信息？"},"page":{"id":"reading-block-91e584a383d94204a682f2b511a70e7a","title":"蓝杯是参照物，目标是它左边的杯子","number":2},"sectionTitle":"蓝杯是参照物，目标是它左边的杯子","currentQuestion":"蓝杯是参照物，目标是它左边的杯子","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":"b-6a7f80bc2117ba5e1b4b6d78","nextBlock":null,"prerequisites":[{"id":"b-bee0a4821c221f238d1671b9","url":"#b-bee0a4821c221f238d1671b9","role":"section-title"},{"id":"b-fe5d080f907d9badb9e038d8","url":"#b-fe5d080f907d9badb9e038d8","role":"problem-context"}],"relatedBlocks":[]},{"id":"b-2cca6a1ec1e4c8c973a64d86","type":"h2","scope":"block-8d37281fe37f43618a865966b0297ef6","url":"https://codingai-lec03.pages.dev/course/notebooks/95169c3e298c4c819373bb4534f3dff9.html#b-2cca6a1ec1e4c8c973a64d86","content":"## GLIP 把语言接入对象级定位","sha256":"65100119afb6475cd7fa42899997aa1fd6b677b73c87e372086b97657e796edb","chapter":{"id":"III","title":"视觉与语言"},"unit":{"id":"language-local-and-question","title":"语言怎样选择当前需要的视觉信息？"},"page":{"id":"reading-block-8d37281fe37f43618a865966b0297ef6","title":"GLIP 把语言接入对象级定位","number":3},"sectionTitle":"GLIP 把语言接入对象级定位","currentQuestion":"GLIP 把语言接入对象级定位","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":null,"nextBlock":"b-318e15f5f756bdd11401e267","prerequisites":[{"id":"b-318e15f5f756bdd11401e267","url":"#b-318e15f5f756bdd11401e267","role":"problem-context"}],"relatedBlocks":[]},{"id":"b-318e15f5f756bdd11401e267","type":"p","scope":"block-8d37281fe37f43618a865966b0297ef6","url":"https://codingai-lec03.pages.dev/course/notebooks/95169c3e298c4c819373bb4534f3dff9.html#b-318e15f5f756bdd11401e267","content":"GLIP 统一对象检测与短语定位，用视觉—文字交互和区域对应监督学习位置。它不是把CLIP整图分数画成框。原论文在其协议下直接评估COCO为49.8 AP、LVIS为26.9 AP，并说明预训练未使用COCO图像。这是语言感知区域表示的正面迁移证据，不是本课程图上的推理。[方法、数据与论文来源](/course/blocks/glip-language-objective.html)。","sha256":"769e106acd90a694434b7d5cc7214cd9f0ac370b2137f0450f2095ba8e8ecf54","chapter":{"id":"III","title":"视觉与语言"},"unit":{"id":"language-local-and-question","title":"语言怎样选择当前需要的视觉信息？"},"page":{"id":"reading-block-8d37281fe37f43618a865966b0297ef6","title":"GLIP 把语言接入对象级定位","number":3},"sectionTitle":"GLIP 把语言接入对象级定位","currentQuestion":"GLIP 把语言接入对象级定位","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":"b-2cca6a1ec1e4c8c973a64d86","nextBlock":"b-4f87760bb0fc291affd89087","prerequisites":[{"id":"b-2cca6a1ec1e4c8c973a64d86","url":"#b-2cca6a1ec1e4c8c973a64d86","role":"section-title"}],"relatedBlocks":[]},{"id":"b-4f87760bb0fc291affd89087","type":"p","scope":"block-8d37281fe37f43618a865966b0297ef6","url":"https://codingai-lec03.pages.dev/course/notebooks/95169c3e298c4c819373bb4534f3dff9.html#b-4f87760bb0fc291affd89087","content":"语言定位先给框，再交分割接口时，要先核对对象身份：上游若选了蓝参照杯，后续精确分割蓝杯仍答错。目标对而边界差，才是另一段接口的问题。","sha256":"5fccf2697f7bb75f1c5b9f28774ade65ffb5193515ffc679352f0e3fa00803d4","chapter":{"id":"III","title":"视觉与语言"},"unit":{"id":"language-local-and-question","title":"语言怎样选择当前需要的视觉信息？"},"page":{"id":"reading-block-8d37281fe37f43618a865966b0297ef6","title":"GLIP 把语言接入对象级定位","number":3},"sectionTitle":"GLIP 把语言接入对象级定位","currentQuestion":"GLIP 把语言接入对象级定位","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":"b-318e15f5f756bdd11401e267","nextBlock":"b-4d86a0d3ac0420ae212e1429","prerequisites":[{"id":"b-2cca6a1ec1e4c8c973a64d86","url":"#b-2cca6a1ec1e4c8c973a64d86","role":"section-title"},{"id":"b-318e15f5f756bdd11401e267","url":"#b-318e15f5f756bdd11401e267","role":"problem-context"}],"relatedBlocks":[]},{"id":"b-4d86a0d3ac0420ae212e1429","type":"h2","scope":"block-8d37281fe37f43618a865966b0297ef6","url":"https://codingai-lec03.pages.dev/course/notebooks/95169c3e298c4c819373bb4534f3dff9.html#b-4d86a0d3ac0420ae212e1429","content":"## 原始 LLaVA 接入的是视觉向量","sha256":"7ff543e39aa88bbec9e002f3eceee9cbd26923179c4a1beb9a5025c6fad29bb7","chapter":{"id":"III","title":"视觉与语言"},"unit":{"id":"language-local-and-question","title":"语言怎样选择当前需要的视觉信息？"},"page":{"id":"reading-block-8d37281fe37f43618a865966b0297ef6","title":"GLIP 把语言接入对象级定位","number":3},"sectionTitle":"GLIP 把语言接入对象级定位","currentQuestion":"GLIP 把语言接入对象级定位","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":"b-4f87760bb0fc291affd89087","nextBlock":"b-221400e15aae416b3eab076a","prerequisites":[{"id":"b-2cca6a1ec1e4c8c973a64d86","url":"#b-2cca6a1ec1e4c8c973a64d86","role":"section-title"},{"id":"b-318e15f5f756bdd11401e267","url":"#b-318e15f5f756bdd11401e267","role":"problem-context"}],"relatedBlocks":[]},{"id":"b-221400e15aae416b3eab076a","type":"p","scope":"block-8d37281fe37f43618a865966b0297ef6","url":"https://codingai-lec03.pages.dev/course/notebooks/95169c3e298c4c819373bb4534f3dff9.html#b-221400e15aae416b3eab076a","content":"CLIP ViT-L/14 的网格特征经可训练线性投影接入 Vicuna，与文字指令一起预测回答。不是先压成一个对象标签，也不是将一项整图相似度输入语言模型。按行写，$Z_v\\in\\mathbb R^{n\\times d_v}$、$W\\in\\mathbb R^{d_v\\times d_l}$，因此 $H_v=Z_vW\\in\\mathbb R^{n\\times d_l}$。","sha256":"abf0150f51dfcccae9133778cb1863a0ccd5f7bbc9387b377717fd90bffbd7e1","chapter":{"id":"III","title":"视觉与语言"},"unit":{"id":"language-local-and-question","title":"语言怎样选择当前需要的视觉信息？"},"page":{"id":"reading-block-8d37281fe37f43618a865966b0297ef6","title":"GLIP 把语言接入对象级定位","number":3},"sectionTitle":"GLIP 把语言接入对象级定位","currentQuestion":"GLIP 把语言接入对象级定位","sourceSlides":[],"knowledgeBlock":null,"formulas":[{"latex":"Z_v\\in\\mathbb R^{n\\times d_v}","definitionContext":"GLIP 把语言接入对象级定位"},{"latex":"W\\in\\mathbb R^{d_v\\times d_l}","definitionContext":"GLIP 把语言接入对象级定位"},{"latex":"H_v=Z_vW\\in\\mathbb R^{n\\times d_l}","definitionContext":"GLIP 把语言接入对象级定位"}],"previousBlock":"b-4d86a0d3ac0420ae212e1429","nextBlock":"b-f1c1c7613d15a5b7d96684aa","prerequisites":[{"id":"b-4d86a0d3ac0420ae212e1429","url":"#b-4d86a0d3ac0420ae212e1429","role":"section-title"},{"id":"b-318e15f5f756bdd11401e267","url":"#b-318e15f5f756bdd11401e267","role":"problem-context"}],"relatedBlocks":[]},{"id":"b-f1c1c7613d15a5b7d96684aa","type":"p","scope":"block-8d37281fe37f43618a865966b0297ef6","url":"https://codingai-lec03.pages.dev/course/notebooks/95169c3e298c4c819373bb4534f3dff9.html#b-f1c1c7613d15a5b7d96684aa","content":"按行堆叠 n 个视觉向量。每个向量有 `d_v` 个分量，因此 `Z_v` 是 `n×d_v` 矩阵；线性投影 `W` 是 `d_v×d_l` 矩阵。乘积 `H_v=Z_vW` 的形状是 `n×d_l`，视觉位置数量 n 保持不变，每个位置转换成语言模型能接收的 `d_l` 维向量。","sha256":"3e8a946035d93cd2627ddc293d4c6f83a2e06f370f81c1adc0bc6050185b8a47","chapter":{"id":"III","title":"视觉与语言"},"unit":{"id":"language-local-and-question","title":"语言怎样选择当前需要的视觉信息？"},"page":{"id":"reading-block-8d37281fe37f43618a865966b0297ef6","title":"GLIP 把语言接入对象级定位","number":3},"sectionTitle":"GLIP 把语言接入对象级定位","currentQuestion":"GLIP 把语言接入对象级定位","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":"b-221400e15aae416b3eab076a","nextBlock":"b-59af4e040c68e3228114b3e8","prerequisites":[{"id":"b-4d86a0d3ac0420ae212e1429","url":"#b-4d86a0d3ac0420ae212e1429","role":"section-title"},{"id":"b-318e15f5f756bdd11401e267","url":"#b-318e15f5f756bdd11401e267","role":"problem-context"}],"relatedBlocks":[]},{"id":"b-59af4e040c68e3228114b3e8","type":"p","scope":"block-8d37281fe37f43618a865966b0297ef6","url":"https://codingai-lec03.pages.dev/course/notebooks/95169c3e298c4c819373bb4534f3dff9.html#b-59af4e040c68e3228114b3e8","content":"CLIP的比较空间不是Vicuna的词向量空间。维度匹配保证计算合法，任务训练建立使用关系；投影不会恢复视觉前处理已丢失的小字或细节。[接口的完整说明](/course/blocks/llava-visual-vector-interface.html)。","sha256":"daaf0b69b1451fbccc45c0ac1a036b0e91a18bbd91e55650a2bba15dff988047","chapter":{"id":"III","title":"视觉与语言"},"unit":{"id":"language-local-and-question","title":"语言怎样选择当前需要的视觉信息？"},"page":{"id":"reading-block-8d37281fe37f43618a865966b0297ef6","title":"GLIP 把语言接入对象级定位","number":3},"sectionTitle":"GLIP 把语言接入对象级定位","currentQuestion":"GLIP 把语言接入对象级定位","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":"b-f1c1c7613d15a5b7d96684aa","nextBlock":"b-6c97a641981ee9cb940b33af","prerequisites":[{"id":"b-4d86a0d3ac0420ae212e1429","url":"#b-4d86a0d3ac0420ae212e1429","role":"section-title"},{"id":"b-318e15f5f756bdd11401e267","url":"#b-318e15f5f756bdd11401e267","role":"problem-context"}],"relatedBlocks":[]},{"id":"b-6c97a641981ee9cb940b33af","type":"table","scope":"block-8d37281fe37f43618a865966b0297ef6","url":"https://codingai-lec03.pages.dev/course/notebooks/95169c3e298c4c819373bb4534f3dff9.html#b-6c97a641981ee9cb940b33af","content":"| 原始阶段 | 视觉编码器 | 线性投影 | 语言模型 |\n| --- | --- | --- | --- |\n| 特征对齐 | 冻结 | 更新 | 冻结 |\n| 视觉指令 | 仍冻结 | 更新 | 更新 |","sha256":"58a6267f320edf88f22da6dab0b12c07dd2293ae03712eb2a3c3a906c0d48eb9","chapter":{"id":"III","title":"视觉与语言"},"unit":{"id":"language-local-and-question","title":"语言怎样选择当前需要的视觉信息？"},"page":{"id":"reading-block-8d37281fe37f43618a865966b0297ef6","title":"GLIP 把语言接入对象级定位","number":3},"sectionTitle":"GLIP 把语言接入对象级定位","currentQuestion":"GLIP 把语言接入对象级定位","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"table":{"headers":["原始阶段","视觉编码器","线性投影","语言模型"],"rows":[["原始阶段","视觉编码器","线性投影","语言模型"],["特征对齐","冻结","更新","冻结"],["视觉指令","仍冻结","更新","更新"]]},"previousBlock":"b-59af4e040c68e3228114b3e8","nextBlock":"b-e2b78d0c0c7937a7078aa855","prerequisites":[{"id":"b-4d86a0d3ac0420ae212e1429","url":"#b-4d86a0d3ac0420ae212e1429","role":"section-title"},{"id":"b-318e15f5f756bdd11401e267","url":"#b-318e15f5f756bdd11401e267","role":"problem-context"}],"relatedBlocks":[]},{"id":"b-e2b78d0c0c7937a7078aa855","type":"p","scope":"block-8d37281fe37f43618a865966b0297ef6","url":"https://codingai-lec03.pages.dev/course/notebooks/95169c3e298c4c819373bb4534f3dff9.html#b-e2b78d0c0c7937a7078aa855","content":"第一阶段先建立接口，第二阶段训练按图像与指令使用信息。两个阶段视觉都冻结；不能从论文“End-to-End”的标题推断视觉也更新。目标计在助手回答和停止位置，问题和图像是条件。[训练范围与原论文位置](/course/blocks/llava-two-stage-freezing.html)。","sha256":"ddbf9516cdb4c41c64652b966f94ecd8e2eb33ed6dbc8617bf610d708dc13fd9","chapter":{"id":"III","title":"视觉与语言"},"unit":{"id":"language-local-and-question","title":"语言怎样选择当前需要的视觉信息？"},"page":{"id":"reading-block-8d37281fe37f43618a865966b0297ef6","title":"GLIP 把语言接入对象级定位","number":3},"sectionTitle":"GLIP 把语言接入对象级定位","currentQuestion":"GLIP 把语言接入对象级定位","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":"b-6c97a641981ee9cb940b33af","nextBlock":"b-e552752a67eec1ef8b4db9ad","prerequisites":[{"id":"b-4d86a0d3ac0420ae212e1429","url":"#b-4d86a0d3ac0420ae212e1429","role":"section-title"},{"id":"b-318e15f5f756bdd11401e267","url":"#b-318e15f5f756bdd11401e267","role":"problem-context"}],"relatedBlocks":[]},{"id":"b-e552752a67eec1ef8b4db9ad","type":"h2","scope":"block-8d37281fe37f43618a865966b0297ef6","url":"https://codingai-lec03.pages.dev/course/notebooks/95169c3e298c4c819373bb4534f3dff9.html#b-e552752a67eec1ef8b4db9ad","content":"## 指令中的事实仍需追溯","sha256":"c65a370a8f6724513cacb5f23766c25784e0a7edb2f1019a61398dd2676a1e3c","chapter":{"id":"III","title":"视觉与语言"},"unit":{"id":"language-local-and-question","title":"语言怎样选择当前需要的视觉信息？"},"page":{"id":"reading-block-8d37281fe37f43618a865966b0297ef6","title":"GLIP 把语言接入对象级定位","number":3},"sectionTitle":"GLIP 把语言接入对象级定位","currentQuestion":"GLIP 把语言接入对象级定位","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":"b-e2b78d0c0c7937a7078aa855","nextBlock":"b-0102b4746bb52e56c9c8ee83","prerequisites":[{"id":"b-4d86a0d3ac0420ae212e1429","url":"#b-4d86a0d3ac0420ae212e1429","role":"section-title"},{"id":"b-318e15f5f756bdd11401e267","url":"#b-318e15f5f756bdd11401e267","role":"problem-context"}],"relatedBlocks":[]},{"id":"b-0102b4746bb52e56c9c8ee83","type":"p","scope":"block-8d37281fe37f43618a865966b0297ef6","url":"https://codingai-lec03.pages.dev/course/notebooks/95169c3e298c4c819373bb4534f3dff9.html#b-0102b4746bb52e56c9c8ee83","content":"原始LLaVA的部分指令资料由文本GPT-4依据caption、类别和框坐标制作，该步骤没有直接看原图。随后训练LLaVA才输入原图。若记录遗漏谁接触球、杯把如何遮挡书页，长回答也不能补足监督；应补标注、改问题或承认覆盖范围。","sha256":"9ccbfc05b07ddd4aaccfb97c7aa85e45d4d99601d4c1329e7184eea5fdffb5ca","chapter":{"id":"III","title":"视觉与语言"},"unit":{"id":"language-local-and-question","title":"语言怎样选择当前需要的视觉信息？"},"page":{"id":"reading-block-8d37281fe37f43618a865966b0297ef6","title":"GLIP 把语言接入对象级定位","number":3},"sectionTitle":"GLIP 把语言接入对象级定位","currentQuestion":"GLIP 把语言接入对象级定位","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":"b-e552752a67eec1ef8b4db9ad","nextBlock":"b-1b291c61c2b4a0871e9c9035","prerequisites":[{"id":"b-e552752a67eec1ef8b4db9ad","url":"#b-e552752a67eec1ef8b4db9ad","role":"section-title"},{"id":"b-318e15f5f756bdd11401e267","url":"#b-318e15f5f756bdd11401e267","role":"problem-context"}],"relatedBlocks":[]},{"id":"b-1b291c61c2b4a0871e9c9035","type":"p","scope":"block-8d37281fe37f43618a865966b0297ef6","url":"https://codingai-lec03.pages.dev/course/notebooks/95169c3e298c4c819373bb4534f3dff9.html#b-1b291c61c2b4a0871e9c9035","content":"这种数据制作能把已有图文资料转成多种交互任务，其价值不只是数量，而是定义系统练习怎样使用图像。[数据来源与一个可做的检查](/course/blocks/llava-instruction-source.html)。循环网络与attention内部留第五讲，不作为本册起步前置。","sha256":"45987936f4a58bce97e53965bf722c5eae4b94758b1bcc9fbb2a564ad724edd6","chapter":{"id":"III","title":"视觉与语言"},"unit":{"id":"language-local-and-question","title":"语言怎样选择当前需要的视觉信息？"},"page":{"id":"reading-block-8d37281fe37f43618a865966b0297ef6","title":"GLIP 把语言接入对象级定位","number":3},"sectionTitle":"GLIP 把语言接入对象级定位","currentQuestion":"GLIP 把语言接入对象级定位","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":"b-0102b4746bb52e56c9c8ee83","nextBlock":null,"prerequisites":[{"id":"b-e552752a67eec1ef8b4db9ad","url":"#b-e552752a67eec1ef8b4db9ad","role":"section-title"},{"id":"b-318e15f5f756bdd11401e267","url":"#b-318e15f5f756bdd11401e267","role":"problem-context"}],"relatedBlocks":[]},{"id":"b-3689f55c577551cb3d752dc4","type":"h2","scope":"block-30c185dacfcf4fb59e6a8b9c3ae7be20","url":"https://codingai-lec03.pages.dev/course/notebooks/95169c3e298c4c819373bb4534f3dff9.html#b-3689f55c577551cb3d752dc4","content":"## 组合接口与选读边界","sha256":"ea081a32377fa4e7891b015a053f4c782391d173702b678a690a469e35517814","chapter":{"id":"III","title":"视觉与语言"},"unit":{"id":"language-local-and-question","title":"语言怎样选择当前需要的视觉信息？"},"page":{"id":"reading-block-30c185dacfcf4fb59e6a8b9c3ae7be20","title":"组合接口与选读边界","number":4},"sectionTitle":"组合接口与选读边界","currentQuestion":"组合接口与选读边界","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":null,"nextBlock":"b-8a022d44bef14d5a50bb2b88","prerequisites":[{"id":"b-8a022d44bef14d5a50bb2b88","url":"#b-8a022d44bef14d5a50bb2b88","role":"problem-context"}],"relatedBlocks":[]},{"id":"b-8a022d44bef14d5a50bb2b88","type":"p","scope":"block-30c185dacfcf4fb59e6a8b9c3ae7be20","url":"https://codingai-lec03.pages.dev/course/notebooks/95169c3e298c4c819373bb4534f3dff9.html#b-8a022d44bef14d5a50bb2b88","content":"语言定位可以先产出框，再将框交给 SAM 一类提示式分割接口。若语言阶段把参照杯当成目标，后续 mask 可能边界很准但对象错误；若框身份正确而边界差，应检查分割与分辨率。由此把错误沿接口定位，见 [提示式分割](/course/pages/l3-r2-p47.html) 和 [语言定位到局部输出](/course/blocks/glip-language-objective.html)。","sha256":"5fa92c124d5680ee446ca2019153d88daebd10b674b4494c5556722c5f7076de","chapter":{"id":"III","title":"视觉与语言"},"unit":{"id":"language-local-and-question","title":"语言怎样选择当前需要的视觉信息？"},"page":{"id":"reading-block-30c185dacfcf4fb59e6a8b9c3ae7be20","title":"组合接口与选读边界","number":4},"sectionTitle":"组合接口与选读边界","currentQuestion":"组合接口与选读边界","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":"b-3689f55c577551cb3d752dc4","nextBlock":"b-5180a42f4f08d641741569d2","prerequisites":[{"id":"b-3689f55c577551cb3d752dc4","url":"#b-3689f55c577551cb3d752dc4","role":"section-title"}],"relatedBlocks":[]},{"id":"b-5180a42f4f08d641741569d2","type":"p","scope":"block-30c185dacfcf4fb59e6a8b9c3ae7be20","url":"https://codingai-lec03.pages.dev/course/notebooks/95169c3e298c4c819373bb4534f3dff9.html#b-5180a42f4f08d641741569d2","content":"**第五讲预备阅读。** BLIP-2 的 Q-Former、Flamingo 的视觉重采样和跨模态交互，提供与原始 LLaVA 线性投影不同的信息选择方式。它们留作接口比较，不是第三讲必须提前完成的 attention 专题。","sha256":"4fb0f211cc5ea6b1f697b08c4e3a4ab0675c3aa6420552c28873e1c9b47164ed","chapter":{"id":"III","title":"视觉与语言"},"unit":{"id":"language-local-and-question","title":"语言怎样选择当前需要的视觉信息？"},"page":{"id":"reading-block-30c185dacfcf4fb59e6a8b9c3ae7be20","title":"组合接口与选读边界","number":4},"sectionTitle":"组合接口与选读边界","currentQuestion":"组合接口与选读边界","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":"b-8a022d44bef14d5a50bb2b88","nextBlock":"b-397da040c299a54386043eb7","prerequisites":[{"id":"b-3689f55c577551cb3d752dc4","url":"#b-3689f55c577551cb3d752dc4","role":"section-title"},{"id":"b-8a022d44bef14d5a50bb2b88","url":"#b-8a022d44bef14d5a50bb2b88","role":"problem-context"}],"relatedBlocks":[]},{"id":"b-397da040c299a54386043eb7","type":"p","scope":"block-30c185dacfcf4fb59e6a8b9c3ae7be20","url":"https://codingai-lec03.pages.dev/course/notebooks/95169c3e298c4c819373bb4534f3dff9.html#b-397da040c299a54386043eb7","content":"压缩视觉向量可降低后续计算，但小字或小物体可能被丢失；提高分辨率也带来成本。判断时用同图换问、遮挡目标和关系交换等具体对照，而不把流畅回答本身当作细节保留的证明。","sha256":"611142a0bb5d0a7d4cf516bc65f90ef3dc4787c6b456d246bb1b2683bfce1556","chapter":{"id":"III","title":"视觉与语言"},"unit":{"id":"language-local-and-question","title":"语言怎样选择当前需要的视觉信息？"},"page":{"id":"reading-block-30c185dacfcf4fb59e6a8b9c3ae7be20","title":"组合接口与选读边界","number":4},"sectionTitle":"组合接口与选读边界","currentQuestion":"组合接口与选读边界","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":"b-5180a42f4f08d641741569d2","nextBlock":"b-389c1e99ac7585f649feb66b","prerequisites":[{"id":"b-3689f55c577551cb3d752dc4","url":"#b-3689f55c577551cb3d752dc4","role":"section-title"},{"id":"b-8a022d44bef14d5a50bb2b88","url":"#b-8a022d44bef14d5a50bb2b88","role":"problem-context"}],"relatedBlocks":[]},{"id":"b-389c1e99ac7585f649feb66b","type":"p","scope":"block-30c185dacfcf4fb59e6a8b9c3ae7be20","url":"https://codingai-lec03.pages.dev/course/notebooks/95169c3e298c4c819373bb4534f3dff9.html#b-389c1e99ac7585f649feb66b","content":"[BLIP-2 原论文](https://arxiv.org/abs/2301.12597) · [Flamingo 原论文](https://arxiv.org/abs/2204.14198)。回到 [三条主线的组件复用](/course/pages/l3-r2-p66.html)，然后进入 [第四讲 · 从条件多解走向生成模型](https://deepnote.com/project/6f83a923-d155-47be-be58-680db701ff7f/notebook/694b7a43ea4948f5a197c014c2fbad3b?utm_source=openai&utm_medium=mcp&utm_campaign=openaimcp&utm_content=694b7a43ea4948f5a197c014c2fbad3b&utm_term=get_notebook)。","sha256":"6bd6829cbb9417b5e60425f8792e5d6b4dd1d1e3ca7aa225e6e61fe3b920ebfd","chapter":{"id":"III","title":"视觉与语言"},"unit":{"id":"language-local-and-question","title":"语言怎样选择当前需要的视觉信息？"},"page":{"id":"reading-block-30c185dacfcf4fb59e6a8b9c3ae7be20","title":"组合接口与选读边界","number":4},"sectionTitle":"组合接口与选读边界","currentQuestion":"组合接口与选读边界","sourceSlides":[],"knowledgeBlock":null,"formulas":[],"previousBlock":"b-397da040c299a54386043eb7","nextBlock":null,"prerequisites":[{"id":"b-3689f55c577551cb3d752dc4","url":"#b-3689f55c577551cb3d752dc4","role":"section-title"},{"id":"b-8a022d44bef14d5a50bb2b88","url":"#b-8a022d44bef14d5a50bb2b88","role":"problem-context"}],"relatedBlocks":[]}]}