[ { "key": "P01-20240202-110250:12b423526ed0f56376d30211352f130cb91e53a3", "video_id": "P01-20240202-110250", "question": "Which refrigerated item on the inside door could be a sweet alternative to the item in the glass at 06:01?", "answer_choice_0": "Iced Tea", "answer_choice_1": "Coffee", "answer_choice_2": "Smoothie", "answer_choice_3": "Chocolate milk", "answer_choice_4": "Fruit juice", "answer_id": 4, "answer": "Fruit juice", "question_type": [ "Object Recognition", "Counterfactual", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "juicer bowl-4870a5041fe72cb4", "glass-df43aa6f802d1561", "cone-429e077d1a0a5f1d", "left half of third orange-56a389eb59ae14b8", "right half of third orange-11785d3856c2848f", "right half of second orange-3bccd9fca84d4a0b", "left half of second orange-b671de6a85a09507", "right half of first orange-f421e886dd28b482", "left half of first orange-37b4dde2b16602ca", "knife-00bcef63aa22aa44", "orange towel-c8cf4b84f82a6a73", "lid of milk frother-c60c10f7d1acb901", "green sponge-c79d145ffded5b56", "black small part of the frother-3cd95e6ebb38b27e", "one wooden stirrer-8070a48d1b8bc081", "food bin-e5572d6184029f52", "one bag-a955b01a2895c6f5", "green roll of recyclable bags-344ce01b15288a18", "box-b9deb700d00c2759", "juicer part-52a8bee137386c66", "white plastic part of the food processor-cb41545a46ea78d0", "top lid of food processor-b3fb93f07305013c", "plug of food processor-762824ba9d53a1cc", "food processor-e556b10f1cd07c2b", "blue chopping board-e5dc98cfa1bdfbd0", "third orange-2defaf6678fa65eb", "second orange-36a5f41a2ff0b00f", "first orange-0dc220cf42cdb3d8", "pack of oranges-98dfab7aa5cc00f7", "scissors-1ac283178491f48e", "milk frother base-cc058509b79d0313", "milk bottle-b90716c096d4c280", "coffee capsule-e5066ca31ca315c2", "mug-e44a21e6e04d424c" ], "reasoning": "1. I watched the video to find what item is poured into a glass (glass-df43aa6f802d1561) at 06:01. \n2. I determined the item poured at 06:01 was orange juice. \n3. Next, I distinguished between the items visible on the refrigerator door at 00:20 as unflavored milk (milk bottle-b90716c096d4c280), centered on the bottom row, next to a bottle of fruit juice . \n4. Ketchup, ranch dressing, olives, and other indistinguishable items are positioned on the fridge door's top row. \n5. Iced Tea, coffee, smoothie, and chocolate milk are not visible in the fridge .\n6. No other visible refrigerated item could be a sweet alternative to the orange juice.\n7. I determined, then, that the fruit juice is the only refrigerated item that could possibly be a sweet alternative to the glass (glass-df43aa6f802d1561) of orange juice.", "referenced_object_masks": [ "milk bottle-b90716c096d4c280", "glass-df43aa6f802d1561" ] }, { "key": "P01-20240202-110250:1f5d3214b2a6f15edd54be5790677431c769bf9b", "video_id": "P01-20240202-110250", "question": "If I doubled the number of times I used the knife, how many times would I have used it?", "answer_choice_0": "6", "answer_choice_1": "3", "answer_choice_2": "4", "answer_choice_3": "8", "answer_choice_4": "7", "answer_id": 3, "answer": "8", "question_type": [ "Counting", "Numerical Reasoning", "Object Recognition", "Counterfactual", "Action Recognition" ], "objects_of_interest": [ "juicer bowl-4870a5041fe72cb4", "glass-df43aa6f802d1561", "cone-429e077d1a0a5f1d", "left half of third orange-56a389eb59ae14b8", "right half of third orange-11785d3856c2848f", "right half of second orange-3bccd9fca84d4a0b", "left half of second orange-b671de6a85a09507", "right half of first orange-f421e886dd28b482", "left half of first orange-37b4dde2b16602ca", "knife-00bcef63aa22aa44", "orange towel-c8cf4b84f82a6a73", "lid of milk frother-c60c10f7d1acb901", "green sponge-c79d145ffded5b56", "black small part of the frother-3cd95e6ebb38b27e", "one wooden stirrer-8070a48d1b8bc081", "food bin-e5572d6184029f52", "one bag-a955b01a2895c6f5", "green roll of recyclable bags-344ce01b15288a18", "box-b9deb700d00c2759", "juicer part-52a8bee137386c66", "white plastic part of the food processor-cb41545a46ea78d0", "top lid of food processor-b3fb93f07305013c", "plug of food processor-762824ba9d53a1cc", "food processor-e556b10f1cd07c2b", "blue chopping board-e5dc98cfa1bdfbd0", "third orange-2defaf6678fa65eb", "second orange-36a5f41a2ff0b00f", "first orange-0dc220cf42cdb3d8", "pack of oranges-98dfab7aa5cc00f7", "scissors-1ac283178491f48e", "milk frother base-cc058509b79d0313", "milk bottle-b90716c096d4c280", "coffee capsule-e5066ca31ca315c2", "mug-e44a21e6e04d424c" ], "reasoning": "1. I watched the video to determine the number of times the user used the knife (knife-00bcef63aa22aa44).\n2. The user used the knife (knife-00bcef63aa22aa44) to cut the first orange (first orange-0dc220cf42cdb3d8) in half at 03:04, the second orange (second orange-36a5f41a2ff0b00f) in half at 03:42, and the third orange (third orange-2defaf6678fa65eb) in half at 04:16.\n3. At 05:04, the user used the knife to stir the pulp left in the orange juicer.\n4. That is a total of 4 times the user used the knife.\n5. If the user doubled the number of times they used the knife, multiplying 4x2, the product would be 8.\n6. Therefore, the user would have used the knife 8 times.", "referenced_object_masks": [ "first orange-0dc220cf42cdb3d8", "knife-00bcef63aa22aa44", "second orange-36a5f41a2ff0b00f", "third orange-2defaf6678fa65eb" ] }, { "key": "P01-20240202-110250:26c1a473a675615727bf1d22dc7b9d9578188929", "video_id": "P01-20240202-110250", "question": "If I quadrupled the number of beverages I made, including the frothy milk, what final number would I end up with?", "answer_choice_0": "3", "answer_choice_1": "12", "answer_choice_2": "8", "answer_choice_3": "4", "answer_choice_4": "10", "answer_id": 1, "answer": "12", "question_type": [ "Counting", "Event Occurence", "Object Recognition", "Numerical Reasoning", "Counterfactual" ], "objects_of_interest": [ "juicer bowl-4870a5041fe72cb4", "glass-df43aa6f802d1561", "cone-429e077d1a0a5f1d", "left half of third orange-56a389eb59ae14b8", "right half of third orange-11785d3856c2848f", "right half of second orange-3bccd9fca84d4a0b", "left half of second orange-b671de6a85a09507", "right half of first orange-f421e886dd28b482", "left half of first orange-37b4dde2b16602ca", "knife-00bcef63aa22aa44", "orange towel-c8cf4b84f82a6a73", "lid of milk frother-c60c10f7d1acb901", "green sponge-c79d145ffded5b56", "black small part of the frother-3cd95e6ebb38b27e", "one wooden stirrer-8070a48d1b8bc081", "food bin-e5572d6184029f52", "one bag-a955b01a2895c6f5", "green roll of recyclable bags-344ce01b15288a18", "box-b9deb700d00c2759", "juicer part-52a8bee137386c66", "white plastic part of the food processor-cb41545a46ea78d0", "top lid of food processor-b3fb93f07305013c", "plug of food processor-762824ba9d53a1cc", "food processor-e556b10f1cd07c2b", "blue chopping board-e5dc98cfa1bdfbd0", "third orange-2defaf6678fa65eb", "second orange-36a5f41a2ff0b00f", "first orange-0dc220cf42cdb3d8", "pack of oranges-98dfab7aa5cc00f7", "scissors-1ac283178491f48e", "milk frother base-cc058509b79d0313", "milk bottle-b90716c096d4c280", "coffee capsule-e5066ca31ca315c2", "mug-e44a21e6e04d424c" ], "reasoning": "1. First, I determined the number of beverages the user made. \n2. Fresh coffee was started using a coffee pod (coffee capsule-e5066ca31ca315c2) at 00:15. \n3. Milk was poured into a milk frother (black small part of the frother-3cd95e6ebb38b27e) at 00:28. \n4. Orange juice was started in a juicer at 03:08. \n5. No other beverages were made, so I concluded that the user made a total of 3 beverages.\n6. Next, to quadruple the number of beverages (3), I multiplied 3x4=12.", "referenced_object_masks": [ "black small part of the frother-3cd95e6ebb38b27e", "coffee capsule-e5066ca31ca315c2" ] }, { "key": "P01-20240202-110250:4cfa4d90d5a94e4049d02774d91a52bb6d267201", "video_id": "P01-20240202-110250", "question": "Out of all the whole oranges I used, which took the least amount of time to juice?", "answer_choice_0": "Second orange", "answer_choice_1": "Fifth orange", "answer_choice_2": "Third orange", "answer_choice_3": "First orange", "answer_choice_4": "Fourth orange", "answer_id": 0, "answer": "Second orange", "question_type": [ "Action Recognition", "Object Recognition", "Counting", "Numerical Reasoning", "Temporal Reasoning" ], "objects_of_interest": [ "juicer bowl-4870a5041fe72cb4", "glass-df43aa6f802d1561", "cone-429e077d1a0a5f1d", "left half of third orange-56a389eb59ae14b8", "right half of third orange-11785d3856c2848f", "right half of second orange-3bccd9fca84d4a0b", "left half of second orange-b671de6a85a09507", "right half of first orange-f421e886dd28b482", "left half of first orange-37b4dde2b16602ca", "knife-00bcef63aa22aa44", "orange towel-c8cf4b84f82a6a73", "lid of milk frother-c60c10f7d1acb901", "green sponge-c79d145ffded5b56", "black small part of the frother-3cd95e6ebb38b27e", "one wooden stirrer-8070a48d1b8bc081", "food bin-e5572d6184029f52", "one bag-a955b01a2895c6f5", "green roll of recyclable bags-344ce01b15288a18", "box-b9deb700d00c2759", "juicer part-52a8bee137386c66", "white plastic part of the food processor-cb41545a46ea78d0", "top lid of food processor-b3fb93f07305013c", "plug of food processor-762824ba9d53a1cc", "food processor-e556b10f1cd07c2b", "blue chopping board-e5dc98cfa1bdfbd0", "third orange-2defaf6678fa65eb", "second orange-36a5f41a2ff0b00f", "first orange-0dc220cf42cdb3d8", "pack of oranges-98dfab7aa5cc00f7", "scissors-1ac283178491f48e", "milk frother base-cc058509b79d0313", "milk bottle-b90716c096d4c280", "coffee capsule-e5066ca31ca315c2", "mug-e44a21e6e04d424c" ], "reasoning": "1. I watched the video to see the user started juicing the oranges (pack of oranges-98dfab7aa5cc00f7) at 03:04. \n2. The user juiced the first orange, starting with the left half (left half of first orange-37b4dde2b16602ca) from 03:08-03:22, for a total of 14 seconds. \n3. The user juiced the second half (right half of the first orange-f421e886dd28b482) from 03:25-03:38, for a total of 13 seconds. \n4. The first orange (first orange-0dc220cf42cdb3d8) took approximately 27 seconds to juice. \n5. The user juiced the second orange, starting with the left half (left half of second orange-b671de6a85a09507) from 03:46-03:57, for a total of 11 seconds. \n6. The user juiced the second half (right half of second orange-3bccd9fca84d4a0b,) from 04:01-04:11, for a total of 10 seconds. \n7. The second orange (second orange-36a5f41a2ff0b00f) took approximately 21 seconds. \n8. The user juiced the third orange, starting with the left half (left half of third orange-56a389eb59ae14b8) from 04:21-04:33 and then continued from 04:36-04:39, for a total of 15 seconds. \n9. The user juiced the third half (right half of third orange-11785d3856c2848f) from 04:43-04:53 and continued from 04:56-05:01, for a total of 15 seconds. \n10. The third orange (third orange-2defaf6678fa65eb) took the user approximately 30 seconds to juice. \n11. The user did not juice anymore oranges. \n12. Therefore, the second orange (second orange-36a5f41a2ff0b00f) took the user the least amount of time to juice.", "referenced_object_masks": [ "first orange-0dc220cf42cdb3d8", "pack of oranges-98dfab7aa5cc00f7", "right half of second orange-3bccd9fca84d4a0b", "right half of third orange-11785d3856c2848f", "left half of second orange-b671de6a85a09507", "second orange-36a5f41a2ff0b00f", "left half of first orange-37b4dde2b16602ca", "right half of the first orange-f421e886dd28b482", "left half of third orange-56a389eb59ae14b8", "third orange-2defaf6678fa65eb" ] }, { "key": "P01-20240202-110250:f5853bd1ee0c56240a304f140b0b3d77a0fea673", "video_id": "P01-20240202-110250", "question": "In which order did I use the following tools: scissors, knife, orange towel, one bag, wooden stirrer?", "answer_choice_0": "One bag, Scissors, Knife, Wooden stirrer, Orange towel", "answer_choice_1": "Wooden stirrer, Orange towel, One bag, Scissors, Knife", "answer_choice_2": "Scissors, One bag, Wooden stirrer, Orange towel, Knife", "answer_choice_3": "Knife, Wooden stirrer, Orange towel, One bag, Scissors", "answer_choice_4": "Orange towel, Knife, Scissors, Wooden stirrer, One bag", "answer_id": 2, "answer": "Scissors, One bag, Wooden stirrer, Orange towel, Knife", "question_type": [ "Temporal Reasoning", "Object Recognition", "Event Occurence" ], "objects_of_interest": [ "juicer bowl-4870a5041fe72cb4", "glass-df43aa6f802d1561", "cone-429e077d1a0a5f1d", "left half of third orange-56a389eb59ae14b8", "right half of third orange-11785d3856c2848f", "right half of second orange-3bccd9fca84d4a0b", "left half of second orange-b671de6a85a09507", "right half of first orange-f421e886dd28b482", "left half of first orange-37b4dde2b16602ca", "knife-00bcef63aa22aa44", "orange towel-c8cf4b84f82a6a73", "lid of milk frother-c60c10f7d1acb901", "green sponge-c79d145ffded5b56", "black small part of the frother-3cd95e6ebb38b27e", "one wooden stirrer-8070a48d1b8bc081", "food bin-e5572d6184029f52", "one bag-a955b01a2895c6f5", "green roll of recyclable bags-344ce01b15288a18", "box-b9deb700d00c2759", "juicer part-52a8bee137386c66", "white plastic part of the food processor-cb41545a46ea78d0", "top lid of food processor-b3fb93f07305013c", "plug of food processor-762824ba9d53a1cc", "food processor-e556b10f1cd07c2b", "blue chopping board-e5dc98cfa1bdfbd0", "third orange-2defaf6678fa65eb", "second orange-36a5f41a2ff0b00f", "first orange-0dc220cf42cdb3d8", "pack of oranges-98dfab7aa5cc00f7", "scissors-1ac283178491f48e", "milk frother base-cc058509b79d0313", "milk bottle-b90716c096d4c280", "coffee capsule-e5066ca31ca315c2", "mug-e44a21e6e04d424c" ], "reasoning": "1. I watched the video to determine the order the user utilized the aforementioned tools. \n2. The user was seen working with the scissors (scissors-1ac283178491f48e) at 00:52. \n3. Next, the user handled one plastic bag (one bag-a955b01a2895c6f5) at 01:46. \n4. After handling a plastic bag, the user was seen using a wooden stirrer (one wooden stirrer-8070a48d1b8bc081) at 02:26. \n5. Next, the user dried their hands on an orange towel (orange towel-c8cf4b84f82a6a73) at 03:01 and again at 06:33. \n6. Finally, the user utilized a knife (knife-00bcef63aa22aa44) for the first time at 03:04.\n7. Therefore, answer C includes the correct order in which the user handled the aforementioned tools.", "referenced_object_masks": [ "knife-00bcef63aa22aa44", "scissors-1ac283178491f48e", "one wooden stirrer-8070a48d1b8bc081", "orange towel-c8cf4b84f82a6a73", "one bag-a955b01a2895c6f5" ] }, { "key": "P01-20240202-161354:4973689c55350d9640b6b39d372cf018c5959879", "video_id": "P01-20240202-161354", "question": "What is the difference between the final number on the scale and the one on the oven timer?", "answer_choice_0": "125", "answer_choice_1": "101", "answer_choice_2": "114", "answer_choice_3": "113", "answer_choice_4": "110", "answer_id": 2, "answer": "114", "question_type": [ "Numerical Reasoning", "Object Recognition", "Reading" ], "objects_of_interest": [ "salt bottle-eb9e167cc2740d23", "pan4-acda939c038e21c3", "pan3-181a9557e45e46f6", "galss2-001d4f1c5bdf97a7", "water filter-841dd6ff987bcbab", "strainer-5ebec360a68f00c4", "jar of black pepper-8b3e8b1921153770", "pan2-adbb354796919372", "plate2-f6de37701c53aca8", "plastic container of grapes-fa029002e77523d0", "plastic box of tomatoes-66e249fe8f75b64e", "yoghurt-d3adc7ccca10c21f", "plastic bag-85256a4c5b1f56b0", "parmesan cheese-733a851847a89247", "butter-548b751c64ce1ca6", "pasta box-6ae79daa7ca78194", "pan-00124b4e87cb7346", "plastic container-924b442033b331f9", "sponge-b3483c58c494e31e", "left glove-311d20193ffa8145", "right hand glove-4f8930764a85086a", "kettle-6712c70397c667be", "food processing lid-8cebaa57ebc85608", "food processor-2446996a209d953b", "glass-8234519be9f5a614", "second mug-1e41716a12636480", "mug-ef5986951d96d648", "plate-a2ced7e9c83ddafa", "whole food processor-facd86505c717e65", "weighing scales-3b8a02b6d19b4146" ], "reasoning": "1. I watched the video until I saw the user finish adding dry pasta to the bowl (plate2-f6de37701c53aca8) set atop the scale (weighing scales-3b8a02b6d19b4146) at 03:09.\n2. At 03:09 I can read the number on the scale (weighing scales-3b8a02b6d19b4146) as 125, but at 03:14 I can see it has ticked up to 126. It remains at 126 until it turns off at 03:58.\n3. At 05:20 the user begins setting the oven timer . I can read that they set it to 12 minutes at 05:25.\n4. I subtracted 12 from 126 to reach 114.", "referenced_object_masks": [ "weighing scales-3b8a02b6d19b4146", "plate2-f6de37701c53aca8" ] }, { "key": "P01-20240202-161354:6e7212050d21e0fd0733ca6fa03bf07fad25b4f5", "video_id": "P01-20240202-161354", "question": "If I had filled up the kettle while wearing gloves, how many objects handled with gloves would I have run under the faucet?", "answer_choice_0": "5", "answer_choice_1": "3", "answer_choice_2": "7", "answer_choice_3": "4", "answer_choice_4": "6", "answer_id": 0, "answer": "5", "question_type": [ "Action Recognition", "Counterfactual", "Counting", "Event Occurence", "Object Recognition", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "salt bottle-eb9e167cc2740d23", "pan4-acda939c038e21c3", "pan3-181a9557e45e46f6", "galss2-001d4f1c5bdf97a7", "water filter-841dd6ff987bcbab", "strainer-5ebec360a68f00c4", "jar of black pepper-8b3e8b1921153770", "pan2-adbb354796919372", "plate2-f6de37701c53aca8", "plastic container of grapes-fa029002e77523d0", "plastic box of tomatoes-66e249fe8f75b64e", "yoghurt-d3adc7ccca10c21f", "plastic bag-85256a4c5b1f56b0", "parmesan cheese-733a851847a89247", "butter-548b751c64ce1ca6", "pasta box-6ae79daa7ca78194", "pan-00124b4e87cb7346", "plastic container-924b442033b331f9", "sponge-b3483c58c494e31e", "left glove-311d20193ffa8145", "right hand glove-4f8930764a85086a", "kettle-6712c70397c667be", "food processing lid-8cebaa57ebc85608", "food processor-2446996a209d953b", "glass-8234519be9f5a614", "second mug-1e41716a12636480", "mug-ef5986951d96d648", "plate-a2ced7e9c83ddafa", "whole food processor-facd86505c717e65", "weighing scales-3b8a02b6d19b4146" ], "reasoning": "1. I watched the video and saw the user pick up the kettle (kettle-6712c70397c667be) at 00:34 without wearing gloves (right hand glove-4f8930764a85086a) (left glove-311d20193ffa8145) and move it under the faucet to fill it at 00:36.\n2. I continued watching and saw the user put on the gloves (right hand glove-4f8930764a85086a) (left glove-311d20193ffa8145) at 00:57.\n3. At 01:02 I saw the user pick up the first object, a glass (glass-8234519be9f5a614), and rinse it under the faucet before placing it in the dishwasher.\n4. At 01:16 I saw the user pick up the second object, a mug (mug-ef5986951d96d648) and rinse it under the faucet before placing it in the dishwasher.\n5. At 01:27 I saw the user pick up the third object, another mug (second mug-1e41716a12636480) and rinse it under the faucet before placing it in the dishwasher.\n6. At 01:36 I saw the user pick up the fourth object, a plate (plate-a2ced7e9c83ddafa) and rinse it under the faucet before placing it in the dishwasher.\n7. From 01:53-01:55 the user removes both gloves (right hand glove-4f8930764a85086a) (left glove-311d20193ffa8145) and does not wear them again for the duration of the video.\n8. I took note of a bowl at the bottom of the sink, which remained there from the time the user filled up the kettle (kettle-6712c70397c667be) to when both gloves (right hand glove-4f8930764a85086a) (left glove-311d20193ffa8145) were removed. Since the bowl was not handled even once (with or without gloves), it does not come into consideration.\n9. Supposing the user had worn gloves (right hand glove-4f8930764a85086a) (left glove-311d20193ffa8145) while filling the kettle (kettle-6712c70397c667be), that would make it the fifth object handled with gloves (right hand glove-4f8930764a85086a) (left glove-311d20193ffa8145) that the user ran under the faucet.", "referenced_object_masks": [ "right hand glove-4f8930764a85086a", "left glove-311d20193ffa8145", "kettle-6712c70397c667be", "mug-ef5986951d96d648", "second mug-1e41716a12636480", "glass-8234519be9f5a614", "plate-a2ced7e9c83ddafa" ] }, { "key": "P01-20240202-161354:835ff0decd5454702163c32ca916d57139d91c72", "video_id": "P01-20240202-161354", "question": "What color is the food inside the 5th thing I touched in the refrigerator?", "answer_choice_0": "White", "answer_choice_1": "Red", "answer_choice_2": "Green", "answer_choice_3": "Silver", "answer_choice_4": "Clear", "answer_id": 1, "answer": "Red", "question_type": [ "Counting", "Action Recognition", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "salt bottle-eb9e167cc2740d23", "pan4-acda939c038e21c3", "pan3-181a9557e45e46f6", "galss2-001d4f1c5bdf97a7", "water filter-841dd6ff987bcbab", "strainer-5ebec360a68f00c4", "jar of black pepper-8b3e8b1921153770", "pan2-adbb354796919372", "plate2-f6de37701c53aca8", "plastic container of grapes-fa029002e77523d0", "plastic box of tomatoes-66e249fe8f75b64e", "yoghurt-d3adc7ccca10c21f", "plastic bag-85256a4c5b1f56b0", "parmesan cheese-733a851847a89247", "butter-548b751c64ce1ca6", "pasta box-6ae79daa7ca78194", "pan-00124b4e87cb7346", "plastic container-924b442033b331f9", "sponge-b3483c58c494e31e", "left glove-311d20193ffa8145", "right hand glove-4f8930764a85086a", "kettle-6712c70397c667be", "food processing lid-8cebaa57ebc85608", "food processor-2446996a209d953b", "glass-8234519be9f5a614", "second mug-1e41716a12636480", "mug-ef5986951d96d648", "plate-a2ced7e9c83ddafa", "whole food processor-facd86505c717e65", "weighing scales-3b8a02b6d19b4146" ], "reasoning": "1. I watched the video until the user opened the refrigerator at 02:05 and began to pull things out.\n2. The first thing they touched was a silver package of butter (butter-548b751c64ce1ca6) at 02:07.\n3. The second thing they touched was a clear plastic package of parmesan cheese (parmesan cheese-733a851847a89247) with a green label at 02:08.\n4. The third thing they touched was a clear plastic bag of food (plastic bag-85256a4c5b1f56b0) at 02:17.\n5. The fourth thing they touched was a white plastic tub of yogurt (yoghurt-d3adc7ccca10c21f) at 02:19.\n6. The fifth thing they touched was a package of tomatoes (plastic box of tomatoes-66e249fe8f75b64e) at 02:21.\n7. The food inside the fifth item is red.", "referenced_object_masks": [ "butter-548b751c64ce1ca6", "plastic bag-85256a4c5b1f56b0", "parmesan cheese-733a851847a89247", "plastic box of tomatoes-66e249fe8f75b64e", "yoghurt-d3adc7ccca10c21f" ] }, { "key": "P01-20240202-161354:b845021d3b0ec6adac335bd2f854aa2820cf0c31", "video_id": "P01-20240202-161354", "question": "What would have happened if I had unknowingly filled my water glass with water from the kettle rather than the pitcher?", "answer_choice_0": "I would have burned my mouth", "answer_choice_1": "I would have filled the pot with it", "answer_choice_2": "The water would have spilled in the dishwasher", "answer_choice_3": "The water would have overflowed", "answer_choice_4": "I would have poured it down the drain", "answer_id": 0, "answer": "I would have burned my mouth", "question_type": [ "Cause and Effect", "Counterfactual", "Action Recognition", "Object Recognition" ], "objects_of_interest": [ "salt bottle-eb9e167cc2740d23", "pan4-acda939c038e21c3", "pan3-181a9557e45e46f6", "galss2-001d4f1c5bdf97a7", "water filter-841dd6ff987bcbab", "strainer-5ebec360a68f00c4", "jar of black pepper-8b3e8b1921153770", "pan2-adbb354796919372", "plate2-f6de37701c53aca8", "plastic container of grapes-fa029002e77523d0", "plastic box of tomatoes-66e249fe8f75b64e", "yoghurt-d3adc7ccca10c21f", "plastic bag-85256a4c5b1f56b0", "parmesan cheese-733a851847a89247", "butter-548b751c64ce1ca6", "pasta box-6ae79daa7ca78194", "pan-00124b4e87cb7346", "plastic container-924b442033b331f9", "sponge-b3483c58c494e31e", "left glove-311d20193ffa8145", "right hand glove-4f8930764a85086a", "kettle-6712c70397c667be", "food processing lid-8cebaa57ebc85608", "food processor-2446996a209d953b", "glass-8234519be9f5a614", "second mug-1e41716a12636480", "mug-ef5986951d96d648", "plate-a2ced7e9c83ddafa", "whole food processor-facd86505c717e65", "weighing scales-3b8a02b6d19b4146" ], "reasoning": "1. I watched until I saw the user fill their water glass (galss2-001d4f1c5bdf97a7) with water from the pitcher (water filter-841dd6ff987bcbab) at 03:44. At 03:48 the user brings the glass (galss2-001d4f1c5bdf97a7) up in their right hand to drink from it.\n2. I watched the user fill the kettle (kettle-6712c70397c667be) with water at 00:36. At 00:55 the user sets it back on its base and turns it on to heat.\n3. At 04:29 I can see that the water the user pours from the kettle (kettle-6712c70397c667be) is hot.\n4. If the user had filled the glass (galss2-001d4f1c5bdf97a7) with hot water from the kettle (kettle-6712c70397c667be), they would have burned their mouth when drinking from it at 03:48.", "referenced_object_masks": [ "water filter-841dd6ff987bcbab", "kettle-6712c70397c667be", "galss2-001d4f1c5bdf97a7" ] }, { "key": "P01-20240202-161354:b9f19f203842665262cfaf978348d269adbcd9da", "video_id": "P01-20240202-161354", "question": "Where does the white mug with blue text move throughout the video?", "answer_choice_0": "From the end of the counter, into the dishwasher, to the drying rack", "answer_choice_1": "From the end of the counter, to the other side of the counter, to under the faucet, into the dishwasher", "answer_choice_2": "From the end of the counter, to under the faucet, to the drying rack", "answer_choice_3": "From the end of the counter, to the other side of the counter, into the sink, into the dishwasher", "answer_choice_4": "From the end of the counter, into the sink, to the dishwasher", "answer_id": 1, "answer": "From the end of the counter, to the other side of the counter, to under the faucet, into the dishwasher", "question_type": [ "Object Recognition", "Egocentric & Spatial Perception", "State Changes" ], "objects_of_interest": [ "salt bottle-eb9e167cc2740d23", "pan4-acda939c038e21c3", "pan3-181a9557e45e46f6", "galss2-001d4f1c5bdf97a7", "water filter-841dd6ff987bcbab", "strainer-5ebec360a68f00c4", "jar of black pepper-8b3e8b1921153770", "pan2-adbb354796919372", "plate2-f6de37701c53aca8", "plastic container of grapes-fa029002e77523d0", "plastic box of tomatoes-66e249fe8f75b64e", "yoghurt-d3adc7ccca10c21f", "plastic bag-85256a4c5b1f56b0", "parmesan cheese-733a851847a89247", "butter-548b751c64ce1ca6", "pasta box-6ae79daa7ca78194", "pan-00124b4e87cb7346", "plastic container-924b442033b331f9", "sponge-b3483c58c494e31e", "left glove-311d20193ffa8145", "right hand glove-4f8930764a85086a", "kettle-6712c70397c667be", "food processing lid-8cebaa57ebc85608", "food processor-2446996a209d953b", "glass-8234519be9f5a614", "second mug-1e41716a12636480", "mug-ef5986951d96d648", "plate-a2ced7e9c83ddafa", "whole food processor-facd86505c717e65", "weighing scales-3b8a02b6d19b4146" ], "reasoning": "1. I watched the video and first saw the white mug with blue text (second mug-1e41716a12636480) sitting at the far end of the counter at 00:05.\n2. At 00:07, the user picks up the mug (second mug-1e41716a12636480) in their right hand and carries it over to the other side of the counter and sets it down beside the sink at 00:11.\n3. At 01:27, the user picks up the mug (second mug-1e41716a12636480) in their right hand, transfers it to their left, and rinses it out under the faucet.\n4. At 01:32 the user places the mug (second mug-1e41716a12636480) in the top rack of the dishwasher, where it remains for the rest of the video.", "referenced_object_masks": [ "second mug-1e41716a12636480" ] }, { "key": "P01-20240202-161948:668b0ae63c2c16a9ceab25120d70e1c3fadee0f7", "video_id": "P01-20240202-161948", "question": "What is the difference in weight between the amount of parmesan I shred the first time to the amount of parmesan I shred the second time?", "answer_choice_0": "3g", "answer_choice_1": "10g", "answer_choice_2": "30 g", "answer_choice_3": "1g", "answer_choice_4": "5g", "answer_id": 3, "answer": "1g", "question_type": [ "Temporal Reasoning", "Numerical Reasoning", "Object Recognition", "Reading" ], "objects_of_interest": [ "saucepan2-d0b77bb6544b2bba", "strainer-ebfe9ba1357add70", "cupboard door3-576b342829111817", "cone of juicer-a776f493443b8c93", "juicer bowl-54c35f4ae46be41a", "cupboard door2-fb6f5d66a6817614", "bowl2-83db2469d2d7bd18", "bowl-8bc47fe0860915f0", "lid of cheese grater-59907c8c4dfbb77c", "third drawer-99db157ae90cd81a", "pasta package-773eaeb446455039", "fridge door-e29958e100cf5c1b", "cheese package-9c76b0563f793a55", "second slice of butter-d1a8b2db02ba7d50", "one slice of butter-8c41128675cf475f", "rest of cheese-5194ca35a5b103f5", "handful of cheese-24a3c683c389b867", "ladle-3893e2a831626941", "drawer2-e94e2a0455aaf512", "plate2-34185b3bbba4e1db", "cupboard-75467a6c947c4da1", "fork-af53942bba314e4c", "spatula-f4a201fecc6f8ce8", "black pepper jar-25f6de5b6eb63905", "cheese block-75fa7d416006e42b", "cheese grater-d2f447fb1751d8c4", "scale-4f3cf7402f66f83e", "plate-8b1b3dfc4a2a7b22", "salt bottle-20321a8dcbf55b59", "knife-5896f57aaefbc467", "top drawer-7225fd5c18aa5010", "packet of butter-a1219e6aa5769241", "saucepan-0e817d8e029ec20a", "notepad-250fd6c1df977def", "pen-0a8eb18592b5f93d", "new bowl with cheese-dd24dad5d22e4afe" ], "reasoning": "1. At 01:37, the user finishes grating the first batch of parmesan (cheese block-75fa7d416006e42b) over a bowl (plate-8b1b3dfc4a2a7b22), which lies on the scale (scale-4f3cf7402f66f83e). At this time, the scale (scale-4f3cf7402f66f83e) reads \"15\".\n2. The user finishes grating parmesan (cheese block-75fa7d416006e42b) a second time at 04:18. When she takes a note at 04:20, the scale (scale-4f3cf7402f66f83e) reads \"16\".\n3. I subtract 16 grams from 15 grams and I get 1 gram.", "referenced_object_masks": [ "scale-4f3cf7402f66f83e", "plate-8b1b3dfc4a2a7b22", "cheese block-75fa7d416006e42b" ] }, { "key": "P01-20240202-161948:66d19970304f75b24ef8528b0bdf91621ea9985b", "video_id": "P01-20240202-161948", "question": "How many times do I either walk or reach to the left of the oven?", "answer_choice_0": "4 times", "answer_choice_1": "10 times", "answer_choice_2": "8 times", "answer_choice_3": "5 times", "answer_choice_4": "3 times", "answer_id": 2, "answer": "8 times", "question_type": [ "Counting", "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness", "Action Recognition" ], "objects_of_interest": [ "saucepan2-d0b77bb6544b2bba", "strainer-ebfe9ba1357add70", "cupboard door3-576b342829111817", "cone of juicer-a776f493443b8c93", "juicer bowl-54c35f4ae46be41a", "cupboard door2-fb6f5d66a6817614", "bowl2-83db2469d2d7bd18", "bowl-8bc47fe0860915f0", "lid of cheese grater-59907c8c4dfbb77c", "third drawer-99db157ae90cd81a", "pasta package-773eaeb446455039", "fridge door-e29958e100cf5c1b", "cheese package-9c76b0563f793a55", "second slice of butter-d1a8b2db02ba7d50", "one slice of butter-8c41128675cf475f", "rest of cheese-5194ca35a5b103f5", "handful of cheese-24a3c683c389b867", "ladle-3893e2a831626941", "drawer2-e94e2a0455aaf512", "plate2-34185b3bbba4e1db", "cupboard-75467a6c947c4da1", "fork-af53942bba314e4c", "spatula-f4a201fecc6f8ce8", "black pepper jar-25f6de5b6eb63905", "cheese block-75fa7d416006e42b", "cheese grater-d2f447fb1751d8c4", "scale-4f3cf7402f66f83e", "plate-8b1b3dfc4a2a7b22", "salt bottle-20321a8dcbf55b59", "knife-5896f57aaefbc467", "top drawer-7225fd5c18aa5010", "packet of butter-a1219e6aa5769241", "saucepan-0e817d8e029ec20a", "notepad-250fd6c1df977def", "pen-0a8eb18592b5f93d", "new bowl with cheese-dd24dad5d22e4afe" ], "reasoning": "1. As the user walks into the kitchen, I notice the oven in the center of the kitchen, just under the counter, at 00:02.\n2. At 00:23, the user walks to the left of the oven to grab a butter knife (knife-5896f57aaefbc467).\n3. At 01:40, the user reaches to the left of the oven to grab a wooden spatula (spatula-f4a201fecc6f8ce8).\n4. At 01:47, the user reaches to the left of the oven to grab a fork (fork-af53942bba314e4c).\n5. At 05:38, the user reaches left of the oven to grab a ladle (ladle-3893e2a831626941).\n6. At 07:44, the user walks to the left to put a package of butter (packet of butter-a1219e6aa5769241) and a package of parmesan (cheese package-9c76b0563f793a55) into the fridge.\n7. At 07:54, the user reaches to the left of the oven to put a box of penne (pasta package-773eaeb446455039) away.\n8. At 09:27, the user walks to the left of the oven to put a salt bottle (salt bottle-20321a8dcbf55b59) away.\n9. At 12:49, the user moves to the left of the oven to take her completed bowl of pasta.\n10. This makes for a total of 8 times, the user either reached or walked to the left of the oven .", "referenced_object_masks": [ "cheese package-9c76b0563f793a55", "packet of butter-a1219e6aa5769241", "salt bottle-20321a8dcbf55b59", "knife-5896f57aaefbc467", "ladle-3893e2a831626941", "fork-af53942bba314e4c", "spatula-f4a201fecc6f8ce8", "pasta package-773eaeb446455039" ] }, { "key": "P01-20240202-161948:808d8d3319740f36c5d5f00c61a4d2a4cd398c43", "video_id": "P01-20240202-161948", "question": "Rounded to the nearest tenth, how many twists per second do I make on the pepper grinder during my first bout of pepper grinding?", "answer_choice_0": "1.6 twists per second", "answer_choice_1": "3.2 twists per second", "answer_choice_2": "1.0 twists per second", "answer_choice_3": "0.6 twists per second", "answer_choice_4": "2.4 twists per second", "answer_id": 0, "answer": "1.6 twists per second", "question_type": [ "Temporal Reasoning", "Action Recognition", "Numerical Reasoning", "Counting" ], "objects_of_interest": [ "saucepan2-d0b77bb6544b2bba", "strainer-ebfe9ba1357add70", "cupboard door3-576b342829111817", "cone of juicer-a776f493443b8c93", "juicer bowl-54c35f4ae46be41a", "cupboard door2-fb6f5d66a6817614", "bowl2-83db2469d2d7bd18", "bowl-8bc47fe0860915f0", "lid of cheese grater-59907c8c4dfbb77c", "third drawer-99db157ae90cd81a", "pasta package-773eaeb446455039", "fridge door-e29958e100cf5c1b", "cheese package-9c76b0563f793a55", "second slice of butter-d1a8b2db02ba7d50", "one slice of butter-8c41128675cf475f", "rest of cheese-5194ca35a5b103f5", "handful of cheese-24a3c683c389b867", "ladle-3893e2a831626941", "drawer2-e94e2a0455aaf512", "plate2-34185b3bbba4e1db", "cupboard-75467a6c947c4da1", "fork-af53942bba314e4c", "spatula-f4a201fecc6f8ce8", "black pepper jar-25f6de5b6eb63905", "cheese block-75fa7d416006e42b", "cheese grater-d2f447fb1751d8c4", "scale-4f3cf7402f66f83e", "plate-8b1b3dfc4a2a7b22", "salt bottle-20321a8dcbf55b59", "knife-5896f57aaefbc467", "top drawer-7225fd5c18aa5010", "packet of butter-a1219e6aa5769241", "saucepan-0e817d8e029ec20a", "notepad-250fd6c1df977def", "pen-0a8eb18592b5f93d", "new bowl with cheese-dd24dad5d22e4afe" ], "reasoning": "1. I notice the user cranking the pepper grinder (black pepper jar-25f6de5b6eb63905) from 02:07 to 02:46, a total of 39 seconds. \n2. However, the user does spend 1 second shaking the pepper grinder (black pepper jar-25f6de5b6eb63905) and not twisting it, so I will subtract 1 second from the total time. Therefore, the user twists for about 38 seconds. \n3. I count the number of twists made during this time and came to a total of about 62 twists.\n4. Calculating 62 over 38 seconds gives me about 1.6 twists per second.", "referenced_object_masks": [ "black pepper jar-25f6de5b6eb63905" ] }, { "key": "P01-20240202-161948:8f1f1cf580c6b438bc727fec6e9d12dac2605778", "video_id": "P01-20240202-161948", "question": "How many total grams of butter do I add to the saucepan?", "answer_choice_0": "20g", "answer_choice_1": "25g", "answer_choice_2": "30g", "answer_choice_3": "10g", "answer_choice_4": "15g", "answer_id": 1, "answer": "25g", "question_type": [ "Numerical Reasoning", "Temporal Reasoning", "Object Recognition", "Reading" ], "objects_of_interest": [ "saucepan2-d0b77bb6544b2bba", "strainer-ebfe9ba1357add70", "cupboard door3-576b342829111817", "cone of juicer-a776f493443b8c93", "juicer bowl-54c35f4ae46be41a", "cupboard door2-fb6f5d66a6817614", "bowl2-83db2469d2d7bd18", "bowl-8bc47fe0860915f0", "lid of cheese grater-59907c8c4dfbb77c", "third drawer-99db157ae90cd81a", "pasta package-773eaeb446455039", "fridge door-e29958e100cf5c1b", "cheese package-9c76b0563f793a55", "second slice of butter-d1a8b2db02ba7d50", "one slice of butter-8c41128675cf475f", "rest of cheese-5194ca35a5b103f5", "handful of cheese-24a3c683c389b867", "ladle-3893e2a831626941", "drawer2-e94e2a0455aaf512", "plate2-34185b3bbba4e1db", "cupboard-75467a6c947c4da1", "fork-af53942bba314e4c", "spatula-f4a201fecc6f8ce8", "black pepper jar-25f6de5b6eb63905", "cheese block-75fa7d416006e42b", "cheese grater-d2f447fb1751d8c4", "scale-4f3cf7402f66f83e", "plate-8b1b3dfc4a2a7b22", "salt bottle-20321a8dcbf55b59", "knife-5896f57aaefbc467", "top drawer-7225fd5c18aa5010", "packet of butter-a1219e6aa5769241", "saucepan-0e817d8e029ec20a", "notepad-250fd6c1df977def", "pen-0a8eb18592b5f93d", "new bowl with cheese-dd24dad5d22e4afe" ], "reasoning": "1.To answer this question, I look for times during the cooking process where butter is weighed and subsequently added to the saucepan (saucepan-0e817d8e029ec20a). \n2. I notice a scale at 00:06 (scale-4f3cf7402f66f83e). \n3. From 00:30 to 00:40, I notice the user cut two pats of butter from the packet of butter (packet of butter-a1219e6aa5769241) and place them inside the saucepan (saucepan-0e817d8e029ec20a) which lies on the scale (scale-4f3cf7402f66f83e). At 00:040, the scale (scale-4f3cf7402f66f83e) reads \"9\". \n4. From 05:05 to 05:15, I also notice the user cut one slice of butter (one slice of butter-8c41128675cf475f) and a second slice of butter (second slice of butter-d1a8b2db02ba7d50) from the packet of butter (packet of butter-a1219e6aa5769241), and weigh them on the scale (scale-4f3cf7402f66f83e). At 05:15, The scale reads \"16\".\n5. At 06:43, the user puts one slice of butter (one slice of butter-8c41128675cf475f) in the saucepan (saucepan-0e817d8e029ec20a), and at 07:17, the user adds the second slice of butter (second slice of butter-d1a8b2db02ba7d50) into the saucepan (saucepan-0e817d8e029ec20a). \n6. This means that 9 grams of butter are added to the saucepan (saucepan-0e817d8e029ec20a) at one time, and 16 grams are added at another. I add these two numbers and I get 25 grams of butter.", "referenced_object_masks": [ "one slice of butter-8c41128675cf475f", "second slice of butter-d1a8b2db02ba7d50", "packet of butter-a1219e6aa5769241", "saucepan-0e817d8e029ec20a", "scale-4f3cf7402f66f83e" ] }, { "key": "P01-20240202-161948:dface47dc50386b998b0cbffbf251ec072685516", "video_id": "P01-20240202-161948", "question": "What objects in the kitchen do I move, but never use?", "answer_choice_0": "I move and use every object in the kitchen", "answer_choice_1": "Pepper grinder, cheese grater, and juicer", "answer_choice_2": "Fork, saucepan, salt bottle", "answer_choice_3": "Salt bottle, box of penne, juicer", "answer_choice_4": "Cheese grater, butter knife, box of penne", "answer_id": 3, "answer": "Salt bottle, box of penne, juicer", "question_type": [ "Object Recognition", "Action Recognition", "State Changes" ], "objects_of_interest": [ "saucepan2-d0b77bb6544b2bba", "strainer-ebfe9ba1357add70", "cupboard door3-576b342829111817", "cone of juicer-a776f493443b8c93", "juicer bowl-54c35f4ae46be41a", "cupboard door2-fb6f5d66a6817614", "bowl2-83db2469d2d7bd18", "bowl-8bc47fe0860915f0", "lid of cheese grater-59907c8c4dfbb77c", "third drawer-99db157ae90cd81a", "pasta package-773eaeb446455039", "fridge door-e29958e100cf5c1b", "cheese package-9c76b0563f793a55", "second slice of butter-d1a8b2db02ba7d50", "one slice of butter-8c41128675cf475f", "rest of cheese-5194ca35a5b103f5", "handful of cheese-24a3c683c389b867", "ladle-3893e2a831626941", "drawer2-e94e2a0455aaf512", "plate2-34185b3bbba4e1db", "cupboard-75467a6c947c4da1", "fork-af53942bba314e4c", "spatula-f4a201fecc6f8ce8", "black pepper jar-25f6de5b6eb63905", "cheese block-75fa7d416006e42b", "cheese grater-d2f447fb1751d8c4", "scale-4f3cf7402f66f83e", "plate-8b1b3dfc4a2a7b22", "salt bottle-20321a8dcbf55b59", "knife-5896f57aaefbc467", "top drawer-7225fd5c18aa5010", "packet of butter-a1219e6aa5769241", "saucepan-0e817d8e029ec20a", "notepad-250fd6c1df977def", "pen-0a8eb18592b5f93d", "new bowl with cheese-dd24dad5d22e4afe" ], "reasoning": "1. I notice the user enter a kitchen with a counter full of appliances and objects, at 00:03. The user begins to make a pasta sauce with pepper, parmesan, butter and pasta water. \n2. At 00:54, I notice the user move a salt bottle (salt bottle-20321a8dcbf55b59) above the scale (scale-4f3cf7402f66f83e) -- but the user never salts with this bottle.\n3. Additionally, from 07:51 to 07:55, the user grabs the box of penne (pasta package-773eaeb446455039) and puts it in the drawer, but never pours pasta out of it. \n4. Finally, from 09:57 to 10:05, I notice the user grab a juicer bowl (juicer bowl-54c35f4ae46be41a) and the cone of the juicer (cone of juicer-a776f493443b8c93) and put them into a cupboard. The user doesn't juice with this item, so I count it among the objects moved, but not used.\n5. Hence, the salt bottle, the box of penne, and the juicer are the objects that are moved, but never used.", "referenced_object_masks": [ "salt bottle-20321a8dcbf55b59", "cone of juicer-a776f493443b8c93", "pasta package-773eaeb446455039", "juicer bowl-54c35f4ae46be41a", "scale-4f3cf7402f66f83e" ] }, { "key": "P01-20240202-171220:1c7c54a25838e1c02fc0ebcbba8a702b3a842160", "video_id": "P01-20240202-171220", "question": "For how long are two electrical appliances on simultaneously?", "answer_choice_0": "1 minute, 34 seconds", "answer_choice_1": "36 seconds", "answer_choice_2": "3 minutes", "answer_choice_3": "2 minutes, 3 seconds", "answer_choice_4": "58 seconds", "answer_id": 4, "answer": "58 seconds", "question_type": [ "Temporal Reasoning", "Object Recognition", "Action Recognition", "Situational Awareness", "Numerical Reasoning" ], "objects_of_interest": [ "knife2-067f49caca9d1aff", "plastic spoon-06f797d4571f2b86", "strainer2-145af72efec29ee7", "mug2-878f03d7b38887ac", "container's cover-0749a379142903b2", "small gold color container-a060d880e6450095", "second cover-9471a44d2c431bb8", "cover of flask-851a735d1af406e2", "flask-9fb1d013fbbfb9ed", "glass2-053199e520c0f556", "mug-4af1b1d04e17deff", "kettle-885fa930de759506", "bag of bagels-ac266267bdee2863", "notepad-c6d57bd31af924bb", "glass-1e1e627cdef3b1f4", "water filter jug-b1b8d0f845af0e69", "plastic bag-c9466433cb43fa2d", "wooden stick-706d23768993e4c9", "food processing bin-ceacf569e9559dd2", "plug-0e6e6a0e27d713b4", "disk-8aae67747df26d12", "candy floss machine-771ffcc774722447", "second sponge-a9ed77e8e0c896fc", "pot3-e506804ec6e96883", "strainer-41302bb5824a0f27", "bottle of washing up liquid-adc726968328d1fa", "sponge-41e14d735f1bad62", "left glove-f606e9a6184e55df", "right glove-fb98ceef3538db60", "pot-8eb9783d5ebd751a", "knife-2c3afc468fb10cb2", "container-e511e9a22559c0d2", "plate2-9e93691684fa34a2", "empty pot-e61cc9d1001e6807", "plate-9a8f738729f20c04", "ladle-3f22f687f702f00a", "scale-d6036d36b016b549", "bowl-529e99d14e2ebc44", "tissue-7af41a37f4989671" ], "reasoning": "1. At 02:42, I notice the user turn on the cotton candy machine (candy floss machine-771ffcc774722447).\n2. Following 02:42, the sound of the turned-on cotton candy maker (candy floss machine-771ffcc774722447) can be heard.\n3. At 03:38, I notice the user put a kettle (kettle-885fa930de759506) on a stand that causes the water inside to heat up.\n4. The sound of the kettle (kettle-885fa930de759506) heating the water can be heard following the 03:38 minute mark.\n3. At 04:36, the user takes the kettle (kettle-885fa930de759506) off the stand.\n4.Two electronic appliances were simultaneously turned on, therefore, from 03:38 to 04:36.\n5. The difference between the two times is 58 seconds.", "referenced_object_masks": [ "kettle-885fa930de759506", "candy floss machine-771ffcc774722447" ] }, { "key": "P01-20240202-171220:3c7fb751e1d1eca12053c30d4778de1754725007", "video_id": "P01-20240202-171220", "question": "How much time passes between the first time I put something in the trash and the third time (not counting putting the crumbs in the compost bin)?", "answer_choice_0": "6 minutes", "answer_choice_1": "2 minutes, 5 seconds", "answer_choice_2": "3 minutes, 35 seconds", "answer_choice_3": "5 minutes, 50 seconds", "answer_choice_4": "4 minutes", "answer_id": 4, "answer": "4 minutes", "question_type": [ "Action Recognition", "Temporal Reasoning", "Numerical Reasoning" ], "objects_of_interest": [ "knife2-067f49caca9d1aff", "plastic spoon-06f797d4571f2b86", "strainer2-145af72efec29ee7", "mug2-878f03d7b38887ac", "container's cover-0749a379142903b2", "small gold color container-a060d880e6450095", "second cover-9471a44d2c431bb8", "cover of flask-851a735d1af406e2", "flask-9fb1d013fbbfb9ed", "glass2-053199e520c0f556", "mug-4af1b1d04e17deff", "kettle-885fa930de759506", "bag of bagels-ac266267bdee2863", "notepad-c6d57bd31af924bb", "glass-1e1e627cdef3b1f4", "water filter jug-b1b8d0f845af0e69", "plastic bag-c9466433cb43fa2d", "wooden stick-706d23768993e4c9", "food processing bin-ceacf569e9559dd2", "plug-0e6e6a0e27d713b4", "disk-8aae67747df26d12", "candy floss machine-771ffcc774722447", "second sponge-a9ed77e8e0c896fc", "pot3-e506804ec6e96883", "strainer-41302bb5824a0f27", "bottle of washing up liquid-adc726968328d1fa", "sponge-41e14d735f1bad62", "left glove-f606e9a6184e55df", "right glove-fb98ceef3538db60", "pot-8eb9783d5ebd751a", "knife-2c3afc468fb10cb2", "container-e511e9a22559c0d2", "plate2-9e93691684fa34a2", "empty pot-e61cc9d1001e6807", "plate-9a8f738729f20c04", "ladle-3f22f687f702f00a", "scale-d6036d36b016b549", "bowl-529e99d14e2ebc44", "tissue-7af41a37f4989671" ], "reasoning": "1. At 00:05, I notice the user throw a tissue (tissue-7af41a37f4989671) into the trash bin.\n2. At 03:17, I notice the user throw a post-it from a notepad (notepad-c6d57bd31af924bb) into the trash bin.\n3. At 04:05, I notice the user throw the plastic seal of a small gold container (notepad-c6d57bd31af924bb) into the trash.\n4. The time elapsing between 00:05 and 04:05 is 4 minutes.\n5. Therefore, the answer is 4 minutes.", "referenced_object_masks": [ "tissue-7af41a37f4989671", "notepad-c6d57bd31af924bb" ] }, { "key": "P01-20240202-171220:3e4997aae7b0c71274039a4186058a07c925043d", "video_id": "P01-20240202-171220", "question": "In what order do I drink water, put the second knife into the dishwasher, and pour hot water into the flask?", "answer_choice_0": "Put the second knife into the dishwasher, drink water, pour hot water into the flask", "answer_choice_1": "Drink water, pour hot water into the flask, put the second knife into the dishwasher", "answer_choice_2": "Drink water, put the second knife into the dishwasher, pour hot water", "answer_choice_3": "Pour hot water into the flask, drink water, put the second knife into the dishwasher", "answer_choice_4": "Put the second knife into the dishwasher, pour hot water into the flask, drink water", "answer_id": 1, "answer": "Drink water, pour hot water into the flask, put the second knife into the dishwasher", "question_type": [ "Action Recognition", "Temporal Reasoning", "Event Occurence", "Object Recognition" ], "objects_of_interest": [ "knife2-067f49caca9d1aff", "plastic spoon-06f797d4571f2b86", "strainer2-145af72efec29ee7", "mug2-878f03d7b38887ac", "container's cover-0749a379142903b2", "small gold color container-a060d880e6450095", "second cover-9471a44d2c431bb8", "cover of flask-851a735d1af406e2", "flask-9fb1d013fbbfb9ed", "glass2-053199e520c0f556", "mug-4af1b1d04e17deff", "kettle-885fa930de759506", "bag of bagels-ac266267bdee2863", "notepad-c6d57bd31af924bb", "glass-1e1e627cdef3b1f4", "water filter jug-b1b8d0f845af0e69", "plastic bag-c9466433cb43fa2d", "wooden stick-706d23768993e4c9", "food processing bin-ceacf569e9559dd2", "plug-0e6e6a0e27d713b4", "disk-8aae67747df26d12", "candy floss machine-771ffcc774722447", "second sponge-a9ed77e8e0c896fc", "pot3-e506804ec6e96883", "strainer-41302bb5824a0f27", "bottle of washing up liquid-adc726968328d1fa", "sponge-41e14d735f1bad62", "left glove-f606e9a6184e55df", "right glove-fb98ceef3538db60", "pot-8eb9783d5ebd751a", "knife-2c3afc468fb10cb2", "container-e511e9a22559c0d2", "plate2-9e93691684fa34a2", "empty pot-e61cc9d1001e6807", "plate-9a8f738729f20c04", "ladle-3f22f687f702f00a", "scale-d6036d36b016b549", "bowl-529e99d14e2ebc44", "tissue-7af41a37f4989671" ], "reasoning": "1. At 00:59, I notice the user putting the first knife (knife-2c3afc468fb10cb2) into the dishwasher.\n2. At 03:06, I notice the user drinking water.\n3. At 04:39, I notice the user pouring hot water into the flask (flask-9fb1d013fbbfb9ed).\n4. At 05:50, I notice the user putting the second knife (knife2-067f49caca9d1aff) into the dishwasher.\n5. The correct order of activities, therefore, is drinking water, pouring hot water into the flask, and putting the second knife into the dishwasher.", "referenced_object_masks": [ "flask-9fb1d013fbbfb9ed", "knife-2c3afc468fb10cb2", "knife2-067f49caca9d1aff" ] }, { "key": "P01-20240202-171220:c9ae258b2075bb0832c470cfdf5f3f98ba366a58", "video_id": "P01-20240202-171220", "question": "What surface colors does the kitchen scale manage to touch?", "answer_choice_0": "Black and silver", "answer_choice_1": "Brown and silver", "answer_choice_2": "White and black", "answer_choice_3": "Brown and black", "answer_choice_4": "Brown and white", "answer_id": 1, "answer": "Brown and silver", "question_type": [ "Object Recognition", "Situational Awareness", "State Changes" ], "objects_of_interest": [ "knife2-067f49caca9d1aff", "plastic spoon-06f797d4571f2b86", "strainer2-145af72efec29ee7", "mug2-878f03d7b38887ac", "container's cover-0749a379142903b2", "small gold color container-a060d880e6450095", "second cover-9471a44d2c431bb8", "cover of flask-851a735d1af406e2", "flask-9fb1d013fbbfb9ed", "glass2-053199e520c0f556", "mug-4af1b1d04e17deff", "kettle-885fa930de759506", "bag of bagels-ac266267bdee2863", "notepad-c6d57bd31af924bb", "glass-1e1e627cdef3b1f4", "water filter jug-b1b8d0f845af0e69", "plastic bag-c9466433cb43fa2d", "wooden stick-706d23768993e4c9", "food processing bin-ceacf569e9559dd2", "plug-0e6e6a0e27d713b4", "disk-8aae67747df26d12", "candy floss machine-771ffcc774722447", "second sponge-a9ed77e8e0c896fc", "pot3-e506804ec6e96883", "strainer-41302bb5824a0f27", "bottle of washing up liquid-adc726968328d1fa", "sponge-41e14d735f1bad62", "left glove-f606e9a6184e55df", "right glove-fb98ceef3538db60", "pot-8eb9783d5ebd751a", "knife-2c3afc468fb10cb2", "container-e511e9a22559c0d2", "plate2-9e93691684fa34a2", "empty pot-e61cc9d1001e6807", "plate-9a8f738729f20c04", "ladle-3f22f687f702f00a", "scale-d6036d36b016b549", "bowl-529e99d14e2ebc44", "tissue-7af41a37f4989671" ], "reasoning": "1. I notice the scale (scale-d6036d36b016b549) at 00:15.\n2. At 00:15, the scale (scale-d6036d36b016b549) is touching the brown wooden countertop.\n3. At 00:22, the user puts the scale (scale-d6036d36b016b549) onto a microwave .\n4. The microwave has a silver surface.\n5. The scale (scale-d6036d36b016b549), therefore, touches both a brown surface and a silver, metallic surface.", "referenced_object_masks": [ "scale-d6036d36b016b549" ] }, { "key": "P01-20240202-171220:ffdb7fa7b06fe78e29027fd30994b64fa8de4e27", "video_id": "P01-20240202-171220", "question": "Which of the following actions do I perform while I am not wearing gloves: wiping down the countertop, rinsing a pot, putting a plate in the dishwasher, cleaning a colander?", "answer_choice_0": "Wiping down the countertop", "answer_choice_1": "Putting a plate in the dishwasher", "answer_choice_2": "Throwing crumbs into the compost bin", "answer_choice_3": "Rinsing a pot", "answer_choice_4": "Cleaning a colander", "answer_id": 1, "answer": "Putting a plate in the dishwasher", "question_type": [ "Temporal Reasoning", "Action Recognition", "Event Occurence" ], "objects_of_interest": [ "knife2-067f49caca9d1aff", "plastic spoon-06f797d4571f2b86", "strainer2-145af72efec29ee7", "mug2-878f03d7b38887ac", "container's cover-0749a379142903b2", "small gold color container-a060d880e6450095", "second cover-9471a44d2c431bb8", "cover of flask-851a735d1af406e2", "flask-9fb1d013fbbfb9ed", "glass2-053199e520c0f556", "mug-4af1b1d04e17deff", "kettle-885fa930de759506", "bag of bagels-ac266267bdee2863", "notepad-c6d57bd31af924bb", "glass-1e1e627cdef3b1f4", "water filter jug-b1b8d0f845af0e69", "plastic bag-c9466433cb43fa2d", "wooden stick-706d23768993e4c9", "food processing bin-ceacf569e9559dd2", "plug-0e6e6a0e27d713b4", "disk-8aae67747df26d12", "candy floss machine-771ffcc774722447", "second sponge-a9ed77e8e0c896fc", "pot3-e506804ec6e96883", "strainer-41302bb5824a0f27", "bottle of washing up liquid-adc726968328d1fa", "sponge-41e14d735f1bad62", "left glove-f606e9a6184e55df", "right glove-fb98ceef3538db60", "pot-8eb9783d5ebd751a", "knife-2c3afc468fb10cb2", "container-e511e9a22559c0d2", "plate2-9e93691684fa34a2", "empty pot-e61cc9d1001e6807", "plate-9a8f738729f20c04", "ladle-3f22f687f702f00a", "scale-d6036d36b016b549", "bowl-529e99d14e2ebc44", "tissue-7af41a37f4989671" ], "reasoning": "1. I notice the user with gloves (left glove-f606e9a6184e55df) wiping down counter tops from 01:51 to 02:01.\n2. I notice the user with gloves (left glove-f606e9a6184e55df) rinsing a pot (pot-8eb9783d5ebd751a) at 01:27.\n3. I notice the user putting two plates (plate2-9e93691684fa34a2) in the dishwasher at 00:53.\n4. At 00:53, the user is not wearing gloves (left glove-f606e9a6184e55df).\n5. I notice the user with gloves (left glove-f606e9a6184e55df) cleaning a colander (strainer-41302bb5824a0f27) at 01:15.\n6. I notice the user with gloves (left glove-f606e9a6184e55df) throwing crumbs into a bin (food processing bin-ceacf569e9559dd2).\n7. The answer is the activity where the user puts a plate into the dishwasher, because it is the only activity performed where the user isn't wearing gloves.", "referenced_object_masks": [ "strainer-41302bb5824a0f27", "plate2-9e93691684fa34a2", "left glove-f606e9a6184e55df", "pot-8eb9783d5ebd751a", "food processing bin-ceacf569e9559dd2" ] }, { "key": "P01-20240202-175627:0efffa8829be3bea0153d201f1b4bcd0c1a39baa", "video_id": "P01-20240202-175627", "question": "What's the difference between the number of times I am fully facing the sink and the number of times I wash a plate?", "answer_choice_0": "0", "answer_choice_1": "3", "answer_choice_2": "2", "answer_choice_3": "4", "answer_choice_4": "5", "answer_id": 3, "answer": "4", "question_type": [ "Counting", "Event Occurence", "Object Recognition", "Egocentric & Spatial Perception", "Situational Awareness", "Counterfactual" ], "objects_of_interest": [ "other lemon half-d134090029249324", "upper lid-d2b0afd0d4e8df9a", "spoon-d88502fa6ce2f59e", "plug-2b3a2e16f21f818b", "knife-8d85fc91438f7177", "one lemon-feeb6dd46dbe194e", "lemon mesh-4666dd39d59a5ccd", "sponge-113b2c59847af82d", "candy floss bowl-696f9460c9eddc90", "cooled disk-d81311bbc9c12b16", "plate-f7e2f23ac89a7b56", "strainer-74ba607f126fef96", "mug-3bbc12c9cb4b4c43", "lid-d8ad03f3b3853036", "thermo flask-893dcb489672b9ab" ], "reasoning": "1. First, I scrub the video to determine the number of times the user is fully facing the sink. \n2. I determine the user is visible fully facing the sink during 00:15, 00:43, 00:57, and 02:14, for a total of 4 times. \n3. Next, I determine the number of times the user washes a plate (plate-f7e2f23ac89a7b56). \n4. I conclude the user only fills a plate (plate-f7e2f23ac89a7b56) with water at 00:43 but never washes it, resulting in 0 times. \n5. To determine the difference between the number of times the user is at the sink (4) and the number of times the user washes a plate (0), I subtract 4-0=4.", "referenced_object_masks": [ "plate-f7e2f23ac89a7b56" ] }, { "key": "P01-20240202-175627:3173b8270a5b4fa998f7228b8bdf2ae9efe6c4c9", "video_id": "P01-20240202-175627", "question": "If I reverse the original order the following events happen: I shut the fridge for the last time, put the plate in the dishwasher, removed the plug from the wall, washed the candy bowl, and unscrewed the thermos lid, what event would be third?", "answer_choice_0": "Put plate in dishwasher", "answer_choice_1": "Unscrewed lid", "answer_choice_2": "Washed bowl", "answer_choice_3": "Pulled plug", "answer_choice_4": "Shut fridge", "answer_id": 3, "answer": "Pulled plug", "question_type": [ "Counterfactual", "Action Recognition", "Counting", "Event Occurence", "Object Recognition" ], "objects_of_interest": [ "other lemon half-d134090029249324", "upper lid-d2b0afd0d4e8df9a", "spoon-d88502fa6ce2f59e", "plug-2b3a2e16f21f818b", "knife-8d85fc91438f7177", "one lemon-feeb6dd46dbe194e", "lemon mesh-4666dd39d59a5ccd", "sponge-113b2c59847af82d", "candy floss bowl-696f9460c9eddc90", "cooled disk-d81311bbc9c12b16", "plate-f7e2f23ac89a7b56", "strainer-74ba607f126fef96", "mug-3bbc12c9cb4b4c43", "lid-d8ad03f3b3853036", "thermo flask-893dcb489672b9ab" ], "reasoning": "1. I scrubbed the video to determine the original order of events the user completed the aforementioned events. \n2. I determined the user first unscrewed the thermos lid (lid-d8ad03f3b3853036) at 00:08.\n3. Second, the user washed the bowl (candy floss bowl-696f9460c9eddc90) at 00:58.\n4. Next, the user removed the plug (plug-2b3a2e16f21f818b) from the wall outlet at 02:11.\n5. After that, the user put the plate (plate-f7e2f23ac89a7b56) into the dishwasher at 02:35.\n6. Finally, the user shut the fridge door for the last time at 03:20.\n7. If the user reversed the events, shutting the fridge door would be first. \n8. Placing the plate (plate-f7e2f23ac89a7b56) in the dishwasher would be second. \n9. Unplugging the plug (plug-2b3a2e16f21f818b) from the wall would be third.\n10. Washing the bowl (candy floss bowl-696f9460c9eddc90) would be fourth. \n11. Unscrewing the thermos lid (lid-d8ad03f3b3853036) would be fifth.\n12. Therefore, unplugging the plug would be the third event in reverse order.", "referenced_object_masks": [ "plate-f7e2f23ac89a7b56", "candy floss bowl-696f9460c9eddc90", "lid-d8ad03f3b3853036", "plug-2b3a2e16f21f818b" ] }, { "key": "P01-20240202-175627:a3f5d391b69030d026d230fec9c6aa8878ebac70", "video_id": "P01-20240202-175627", "question": "What is the ratio of the number of times I scrub the candyfloss bowl in a clockwise direction to the number of times I scrub it in a counterclockwise direction with the sponge?", "answer_choice_0": "8:2", "answer_choice_1": "6:5", "answer_choice_2": "10:4", "answer_choice_3": "1:9", "answer_choice_4": "2:13", "answer_id": 1, "answer": "6:5", "question_type": [ "Action Recognition", "Event Occurence", "Object Recognition" ], "objects_of_interest": [ "other lemon half-d134090029249324", "upper lid-d2b0afd0d4e8df9a", "spoon-d88502fa6ce2f59e", "plug-2b3a2e16f21f818b", "knife-8d85fc91438f7177", "one lemon-feeb6dd46dbe194e", "lemon mesh-4666dd39d59a5ccd", "sponge-113b2c59847af82d", "candy floss bowl-696f9460c9eddc90", "cooled disk-d81311bbc9c12b16", "plate-f7e2f23ac89a7b56", "strainer-74ba607f126fef96", "mug-3bbc12c9cb4b4c43", "lid-d8ad03f3b3853036", "thermo flask-893dcb489672b9ab" ], "reasoning": "1. First I looked for the user to wash the candyfloss bowl (candy floss bowl-696f9460c9eddc90) with a sponge (sponge-113b2c59847af82d), which they began to do at 01:17. They wash the bowl (candy floss bowl-696f9460c9eddc90) until 01:22 when they drop the sponge (sponge-113b2c59847af82d). \n2. Next, I counted the number of scrubs the user makes in a clockwise direction with the sponge (sponge-113b2c59847af82d) between 01:17-01:22 and got a total of 6 scrubs. \n3. Then, I counted the number of scrubs the user makes in a counterclockwise direction with the sponge (sponge-113b2c59847af82d) between 01:17-01:22 and got a total of 5.\n5. Therefore, the ratio of clockwise scrubs to counterclockwise scrubs is 6:5.", "referenced_object_masks": [ "sponge-113b2c59847af82d", "candy floss bowl-696f9460c9eddc90" ] }, { "key": "P01-20240202-175627:a4c3419646b456c4dec11c4579e30c28744a96fc", "video_id": "P01-20240202-175627", "question": "If I'd cut 3 more lemons, in addition to what I cut previously, how many lemon halves would I have in total?", "answer_choice_0": "10 halves", "answer_choice_1": "4 halves", "answer_choice_2": "2 halves", "answer_choice_3": "6 halves", "answer_choice_4": "8 halves", "answer_id": 4, "answer": "8 halves", "question_type": [ "Counterfactual", "Counting", "Numerical Reasoning", "Object Recognition" ], "objects_of_interest": [ "other lemon half-d134090029249324", "upper lid-d2b0afd0d4e8df9a", "spoon-d88502fa6ce2f59e", "plug-2b3a2e16f21f818b", "knife-8d85fc91438f7177", "one lemon-feeb6dd46dbe194e", "lemon mesh-4666dd39d59a5ccd", "sponge-113b2c59847af82d", "candy floss bowl-696f9460c9eddc90", "cooled disk-d81311bbc9c12b16", "plate-f7e2f23ac89a7b56", "strainer-74ba607f126fef96", "mug-3bbc12c9cb4b4c43", "lid-d8ad03f3b3853036", "thermo flask-893dcb489672b9ab" ], "reasoning": "1. First, I determined how many lemons the user cuts and uses in the video. \n2. The user retrieves 1 whole lemon (one lemon-feeb6dd46dbe194e) from the fridge at 01:50, cuts it at 01:59, and squeezes half of it into a mug (mug-3bbc12c9cb4b4c43) at 02:02, leaving the other half (other lemon half-d134090029249324) on the cutting board. \n3. This is the only lemon (one lemon-feeb6dd46dbe194e) the user handles.\n4. If the user were to cut 3 more whole lemons, which equals 6 lemon halves, and add it to the 1 lemon (one lemon-feeb6dd46dbe194e) already used (2 halves), the total would be 4 lemons, equal to 8 halves.", "referenced_object_masks": [ "mug-3bbc12c9cb4b4c43", "one lemon-feeb6dd46dbe194e", "other lemon half-d134090029249324" ] }, { "key": "P01-20240202-175627:ed85a97631144c1d16e9254377e1dc03385417bc", "video_id": "P01-20240202-175627", "question": "What do I spoon out of my mug and throw away?", "answer_choice_0": "Tea leaves", "answer_choice_1": "Lemon peel", "answer_choice_2": "Tea bag", "answer_choice_3": "Lemon seed", "answer_choice_4": "Sugar cube", "answer_id": 3, "answer": "Lemon seed", "question_type": [ "Action Recognition", "Event Occurence", "Object Recognition" ], "objects_of_interest": [ "other lemon half-d134090029249324", "upper lid-d2b0afd0d4e8df9a", "spoon-d88502fa6ce2f59e", "plug-2b3a2e16f21f818b", "knife-8d85fc91438f7177", "one lemon-feeb6dd46dbe194e", "lemon mesh-4666dd39d59a5ccd", "sponge-113b2c59847af82d", "candy floss bowl-696f9460c9eddc90", "cooled disk-d81311bbc9c12b16", "plate-f7e2f23ac89a7b56", "strainer-74ba607f126fef96", "mug-3bbc12c9cb4b4c43", "lid-d8ad03f3b3853036", "thermo flask-893dcb489672b9ab" ], "reasoning": "1. I locate the section of the video where the user is spooning an object out of their mug (mug-3bbc12c9cb4b4c43). \n2. This event occurs at 02:52-02:57.\n3. I rewind the video back to determine what is taking place before that event. \n4. At 02:02, the user squeezes a lemon (one lemon-feeb6dd46dbe194e) into their mug (mug-3bbc12c9cb4b4c43) of tea.\n5. I can then determine that the user is spooning a lemon seed from their mug that accidentally fell into their mug (mug-3bbc12c9cb4b4c43) during the squeezing of the lemon (one lemon-feeb6dd46dbe194e) at 02:02.", "referenced_object_masks": [ "one lemon-feeb6dd46dbe194e", "mug-3bbc12c9cb4b4c43" ] }, { "key": "P01-20240202-195538:34bf5d0f004062a391784d717ede13e2a021b94c", "video_id": "P01-20240202-195538", "question": "What color would I get if I mixed the colors of all of the gloves I wore?", "answer_choice_0": "Red", "answer_choice_1": "Green", "answer_choice_2": "Orange", "answer_choice_3": "Yellow", "answer_choice_4": "Pink", "answer_id": 2, "answer": "Orange", "question_type": [ "Counterfactual", "Object Recognition", "Event Occurence" ], "objects_of_interest": [ "other oven glove-48f1b071f87ba9d3", "two oven gloves-7485a525b76b6702", "other mug-05de89741c17a8ec", "grey draining tray-38d9a04dadd729cf", "other sponge-7915067a751aeec1", "water filter jug-1f58269e7af47ad8", "pen-41e677fe52669fd9", "notebook-55051fbad89f51d1", "bowl4-c1e677150cd75a84", "some plates-a40cdeee936031e1", "glass-6ae7af9a565257b5", "sink hole cover-6dc0c7424c8a4db0", "bowl2-0b889f5b754cda95", "container-cc390db8f0cfe0f0", "plate-10fe2fc829ea25c1", "fork-15399a321619eae6", "pan-f130c55ab4d82397", "washing up liquid bottle-49ff84964587f5fe", "sponge-11673443d224486b", "spatula-29a11aeea70f84db", "left glove-113f86144358265d", "right glove-cd27a95c0265b8ea", "spoon-b991e034f5c8eed0", "plastic container of soup-f5d7d262a0fac16b", "bowl-0ca2d3e6c64cb047", "candy floss machine-19256a1d63acabcc", "disk-c062f2a00587fe9c", "candy floss bowl-541469e8f03660d2", "strainer-c65da629615df617", "lid-27476d1f9881898b", "flask-181b2e6a7a7b60c9", "mug-caeb494b6574cb34", "inner flask lid-31e4d552813d3887", "new pot-d6f591bd2f8b2dad" ], "reasoning": "1. I watched the video and saw the user don yellow gloves (right glove-cd27a95c0265b8ea, left glove-113f86144358265d) at 02:56.\n2. I continued watching and saw them don red gloves (two oven gloves-7485a525b76b6702, other oven glove-48f1b071f87ba9d3) at 07:38.\n3. If I mixed red and yellow I would get orange.", "referenced_object_masks": [ "right glove-cd27a95c0265b8ea", "left glove-113f86144358265d", "two oven gloves-7485a525b76b6702", "other oven glove-48f1b071f87ba9d3" ] }, { "key": "P01-20240202-195538:4287d4e9af3a8c7dbeeaf4449b6cd42730599a10", "video_id": "P01-20240202-195538", "question": "How many items containing metal did I rinse under the faucet and then place somewhere other than the dishwasher?", "answer_choice_0": "9", "answer_choice_1": "5", "answer_choice_2": "11", "answer_choice_3": "8", "answer_choice_4": "3", "answer_id": 4, "answer": "3", "question_type": [ "Counting", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "other oven glove-48f1b071f87ba9d3", "two oven gloves-7485a525b76b6702", "other mug-05de89741c17a8ec", "grey draining tray-38d9a04dadd729cf", "other sponge-7915067a751aeec1", "water filter jug-1f58269e7af47ad8", "pen-41e677fe52669fd9", "notebook-55051fbad89f51d1", "bowl4-c1e677150cd75a84", "some plates-a40cdeee936031e1", "glass-6ae7af9a565257b5", "sink hole cover-6dc0c7424c8a4db0", "bowl2-0b889f5b754cda95", "container-cc390db8f0cfe0f0", "plate-10fe2fc829ea25c1", "fork-15399a321619eae6", "pan-f130c55ab4d82397", "washing up liquid bottle-49ff84964587f5fe", "sponge-11673443d224486b", "spatula-29a11aeea70f84db", "left glove-113f86144358265d", "right glove-cd27a95c0265b8ea", "spoon-b991e034f5c8eed0", "plastic container of soup-f5d7d262a0fac16b", "bowl-0ca2d3e6c64cb047", "candy floss machine-19256a1d63acabcc", "disk-c062f2a00587fe9c", "candy floss bowl-541469e8f03660d2", "strainer-c65da629615df617", "lid-27476d1f9881898b", "flask-181b2e6a7a7b60c9", "mug-caeb494b6574cb34", "inner flask lid-31e4d552813d3887", "new pot-d6f591bd2f8b2dad" ], "reasoning": "1. I watched the user run the tea strainer (strainer-c65da629615df617), which contained both metal and plastic, under the faucet at 00:50. They then placed it on the drying rack at 00:52, so I counted this as the first item. \n2. At 00:54, the user rinsed out a tea mug (flask-181b2e6a7a7b60c9) with a metal interior and then placed it on the drying rack at 01:02. I counted this as the second item. \n3. At 04:34 I saw the user rinse the sink trap (sink hole cover-6dc0c7424c8a4db0) under the faucet and then plug the sink with it at 04:40. I counted this as the third item. \n4. Therefore, the user rinsed a total of 3 metal items under the faucet that they then placed somewhere other than the dishwasher.", "referenced_object_masks": [ "flask-181b2e6a7a7b60c9", "strainer-c65da629615df617", "sink hole cover-6dc0c7424c8a4db0" ] }, { "key": "P01-20240202-195538:5ddaaa24f530100f476650a2d99d1ccac4d6d941", "video_id": "P01-20240202-195538", "question": "How many more times did I fully stir my soup clockwise than there are oranges on the counter?", "answer_choice_0": "9", "answer_choice_1": "12", "answer_choice_2": "10", "answer_choice_3": "11", "answer_choice_4": "8", "answer_id": 3, "answer": "11", "question_type": [ "Counting", "Action Recognition", "Object Recognition", "Numerical Reasoning" ], "objects_of_interest": [ "other oven glove-48f1b071f87ba9d3", "two oven gloves-7485a525b76b6702", "other mug-05de89741c17a8ec", "grey draining tray-38d9a04dadd729cf", "other sponge-7915067a751aeec1", "water filter jug-1f58269e7af47ad8", "pen-41e677fe52669fd9", "notebook-55051fbad89f51d1", "bowl4-c1e677150cd75a84", "some plates-a40cdeee936031e1", "glass-6ae7af9a565257b5", "sink hole cover-6dc0c7424c8a4db0", "bowl2-0b889f5b754cda95", "container-cc390db8f0cfe0f0", "plate-10fe2fc829ea25c1", "fork-15399a321619eae6", "pan-f130c55ab4d82397", "washing up liquid bottle-49ff84964587f5fe", "sponge-11673443d224486b", "spatula-29a11aeea70f84db", "left glove-113f86144358265d", "right glove-cd27a95c0265b8ea", "spoon-b991e034f5c8eed0", "plastic container of soup-f5d7d262a0fac16b", "bowl-0ca2d3e6c64cb047", "candy floss machine-19256a1d63acabcc", "disk-c062f2a00587fe9c", "candy floss bowl-541469e8f03660d2", "strainer-c65da629615df617", "lid-27476d1f9881898b", "flask-181b2e6a7a7b60c9", "mug-caeb494b6574cb34", "inner flask lid-31e4d552813d3887", "new pot-d6f591bd2f8b2dad" ], "reasoning": "1. I watched the user remove their soup bowl (bowl-0ca2d3e6c64cb047) from the microwave and stir it clockwise 12 times from 05:36-05:45.\n2. I can see the user stir the soup clockwise 1 additional time from 08:50-08:51.\n3. I can clearly see 2 oranges in the mesh bag atop the counter at 02:37.\n4. 13 stirs minus 2 oranges equals 11.", "referenced_object_masks": [ "bowl-0ca2d3e6c64cb047" ] }, { "key": "P01-20240202-195538:8b234fd4e2c5feb1530e204939504c23c194bf04", "video_id": "P01-20240202-195538", "question": "Excluding items already loaded into the dishwasher before the start of the video, if I had put my flask and strainer in the dishwasher, how many items would I have loaded into it?", "answer_choice_0": "7", "answer_choice_1": "8", "answer_choice_2": "6", "answer_choice_3": "10", "answer_choice_4": "9", "answer_id": 4, "answer": "9", "question_type": [ "Counterfactual", "Counting", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "other oven glove-48f1b071f87ba9d3", "two oven gloves-7485a525b76b6702", "other mug-05de89741c17a8ec", "grey draining tray-38d9a04dadd729cf", "other sponge-7915067a751aeec1", "water filter jug-1f58269e7af47ad8", "pen-41e677fe52669fd9", "notebook-55051fbad89f51d1", "bowl4-c1e677150cd75a84", "some plates-a40cdeee936031e1", "glass-6ae7af9a565257b5", "sink hole cover-6dc0c7424c8a4db0", "bowl2-0b889f5b754cda95", "container-cc390db8f0cfe0f0", "plate-10fe2fc829ea25c1", "fork-15399a321619eae6", "pan-f130c55ab4d82397", "washing up liquid bottle-49ff84964587f5fe", "sponge-11673443d224486b", "spatula-29a11aeea70f84db", "left glove-113f86144358265d", "right glove-cd27a95c0265b8ea", "spoon-b991e034f5c8eed0", "plastic container of soup-f5d7d262a0fac16b", "bowl-0ca2d3e6c64cb047", "candy floss machine-19256a1d63acabcc", "disk-c062f2a00587fe9c", "candy floss bowl-541469e8f03660d2", "strainer-c65da629615df617", "lid-27476d1f9881898b", "flask-181b2e6a7a7b60c9", "mug-caeb494b6574cb34", "inner flask lid-31e4d552813d3887", "new pot-d6f591bd2f8b2dad" ], "reasoning": "1. I watched the video and saw the user set the strainer (strainer-c65da629615df617) into the drying rack at 00:52. They then set the flask (flask-181b2e6a7a7b60c9) into the drying rack at 01:02.\n2. At 03:37 the user sets the first item, a pot (pan-f130c55ab4d82397), into the dishwasher.\n3. At 03:50 the user sets the second item, a fork (fork-15399a321619eae6), into the dishwasher.\n4. At 03:56, the user sets the third item, a bowl (plate-10fe2fc829ea25c1), into the dishwasher.\n5. At 04:07, the user sets the fourth item, a bowl , into the dishwasher.\n6. At 05:21, the user sets the fifth item, a mug (mug-caeb494b6574cb34), into the dishwasher.\n7. At 07:12, the user sets the sixth item, a spoon , into the dishwasher.\n8. At 07:18, the user sets the seventh and final item, a glass (glass-6ae7af9a565257b5), into the dishwasher.\n9. I added 7+2 to reach a total of 9 items.", "referenced_object_masks": [ "strainer-c65da629615df617", "pan-f130c55ab4d82397", "mug-caeb494b6574cb34", "plate-10fe2fc829ea25c1", "fork-15399a321619eae6", "flask-181b2e6a7a7b60c9", "glass-6ae7af9a565257b5" ] }, { "key": "P01-20240202-195538:a1184f5e927f45633861a6b86feb83cd72fd5a42", "video_id": "P01-20240202-195538", "question": "How many different vessels hold soup throughout the video?", "answer_choice_0": "2", "answer_choice_1": "4", "answer_choice_2": "1", "answer_choice_3": "3", "answer_choice_4": "5", "answer_id": 3, "answer": "3", "question_type": [ "Counting", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "other oven glove-48f1b071f87ba9d3", "two oven gloves-7485a525b76b6702", "other mug-05de89741c17a8ec", "grey draining tray-38d9a04dadd729cf", "other sponge-7915067a751aeec1", "water filter jug-1f58269e7af47ad8", "pen-41e677fe52669fd9", "notebook-55051fbad89f51d1", "bowl4-c1e677150cd75a84", "some plates-a40cdeee936031e1", "glass-6ae7af9a565257b5", "sink hole cover-6dc0c7424c8a4db0", "bowl2-0b889f5b754cda95", "container-cc390db8f0cfe0f0", "plate-10fe2fc829ea25c1", "fork-15399a321619eae6", "pan-f130c55ab4d82397", "washing up liquid bottle-49ff84964587f5fe", "sponge-11673443d224486b", "spatula-29a11aeea70f84db", "left glove-113f86144358265d", "right glove-cd27a95c0265b8ea", "spoon-b991e034f5c8eed0", "plastic container of soup-f5d7d262a0fac16b", "bowl-0ca2d3e6c64cb047", "candy floss machine-19256a1d63acabcc", "disk-c062f2a00587fe9c", "candy floss bowl-541469e8f03660d2", "strainer-c65da629615df617", "lid-27476d1f9881898b", "flask-181b2e6a7a7b60c9", "mug-caeb494b6574cb34", "inner flask lid-31e4d552813d3887", "new pot-d6f591bd2f8b2dad" ], "reasoning": "1. I watched the user pull out a container of soup (plastic container of soup-f5d7d262a0fac16b) from the refrigerator at 01:53.\n2. At 02:08 I can see the user spoon some of the soup into a bowl (bowl-0ca2d3e6c64cb047).\n3. I continued watching as the user placed the bowl (bowl-0ca2d3e6c64cb047) of soup into the microwave to warm up. The user removes it and at 07:56, they spoon it into a new bowl (bowl4-c1e677150cd75a84), the third vessel.", "referenced_object_masks": [ "bowl-0ca2d3e6c64cb047", "bowl4-c1e677150cd75a84", "plastic container of soup-f5d7d262a0fac16b" ] }, { "key": "P01-20240203-093333:0d1e23e629eb264ba20c104ccffc301983ef8d2a", "video_id": "P01-20240203-093333", "question": "Given what I do with the can in the fridge, should I use it for anything I make in the kitchen?", "answer_choice_0": "Yes, you opened a can of yeast and you are clearly making bread", "answer_choice_1": "No, you accidentally spilled its contents on the floor and ruined it", "answer_choice_2": "No, you threw it away because it is expired", "answer_choice_3": "No, you shook your head after grabbing it, so clearly you don't want it", "answer_choice_4": "Yes, you drank its contents and clearly liked the taste", "answer_id": 2, "answer": "No, you threw it away because it is expired", "question_type": [ "Goal Reasoning", "Counterfactual", "Action Recognition" ], "objects_of_interest": [ "sponge-4ad8895244419f48", "center piece-a3a784bd19011302", "wooden stirrer-737a4cc5450367cc", "lid-8c02dbfadaa70cc5", "milk frother-c5cf435ffba447a5", "can of yeast-5c573fc88948c561", "milk bottle-70560e0cf827e275", "coffee capsule-734c74771eb5ac6f", "box of coffee capsules-611f83c8d12e0d66", "water filter jug-9a6a101762cceb45", "water tank-80942cbe1122e817", "mug-5e6e92894731cbbb", "pot-4bf068ddbf84109b", "empty plate-b34bcbed5870e580" ], "reasoning": "1. I look for the time the user picked out a can (can of yeast-5c573fc88948c561) from the fridge, which happens at 00:53.\n2. The can (can of yeast-5c573fc88948c561) is a can of yeast.\n2. For a long time, from 00:56 to 01:03, the user looks at the back of the can (can of yeast-5c573fc88948c561), where the expiration date is labelled.\n3. At 01:07, the user throws the can (can of yeast-5c573fc88948c561) away.\n4. This series of events suggests that the can (can of yeast-5c573fc88948c561) is expired.\n5. Given this information, therefore, the user shouldn't use it for anything they could be making in the kitchen.\n6. Therefore, the answer is no, you threw it away because it is expired.", "referenced_object_masks": [ "can of yeast-5c573fc88948c561" ] }, { "key": "P01-20240203-093333:58cb8f79fdafbfae95b42105c38fcafad8413292", "video_id": "P01-20240203-093333", "question": "About how many times do I see the coffee machine blink, when all its 4 lights are blinking simultaneously?", "answer_choice_0": "The coffee machine never blinks with 4 lights", "answer_choice_1": "22 times", "answer_choice_2": "28 times", "answer_choice_3": "12 times", "answer_choice_4": "8 times", "answer_id": 1, "answer": "22 times", "question_type": [ "Object Recognition", "Action Recognition", "Counting", "State Changes", "Numerical Reasoning" ], "objects_of_interest": [ "sponge-4ad8895244419f48", "center piece-a3a784bd19011302", "wooden stirrer-737a4cc5450367cc", "lid-8c02dbfadaa70cc5", "milk frother-c5cf435ffba447a5", "can of yeast-5c573fc88948c561", "milk bottle-70560e0cf827e275", "coffee capsule-734c74771eb5ac6f", "box of coffee capsules-611f83c8d12e0d66", "water filter jug-9a6a101762cceb45", "water tank-80942cbe1122e817", "mug-5e6e92894731cbbb", "pot-4bf068ddbf84109b", "empty plate-b34bcbed5870e580" ], "reasoning": "1. I notice the user walk toward the coffee machine and turn it on at 00:19.\n2. When the coffee machine turns on, four of its lights start flashing simultaneously.\n3. Counting the first flash, the coffee machine flashes 4 times from 00:19-00:22.\n4. When the coffee machine next comes into view from 00:27-00:31, it blinks 5 more times.\n5. When the coffee machine comes into view again from 00:36-00:48, it flashes 13 more times.\n6. When the coffee machine returns into view at 01:09, its lights aren't flashing anymore.\n7. From 01:09 on, whenever the coffee machine is seen flashing, it isn't flashing with 4 lights at the same time.\n8: I add 5+4+13=22.\n9. Therefore, 22 times is the answer.", "referenced_object_masks": [] }, { "key": "P01-20240203-093333:652b3572fe1897059a695738bceb66db47c86028", "video_id": "P01-20240203-093333", "question": "How many times do I stop and stare for more than 7 seconds?", "answer_choice_0": "3 times", "answer_choice_1": "4 times", "answer_choice_2": "2 times", "answer_choice_3": "1 time", "answer_choice_4": "5 times", "answer_id": 0, "answer": "3 times", "question_type": [ "Egocentric & Spatial Perception", "Temporal Reasoning", "State Changes", "Action Recognition", "Counting", "Numerical Reasoning" ], "objects_of_interest": [ "sponge-4ad8895244419f48", "center piece-a3a784bd19011302", "wooden stirrer-737a4cc5450367cc", "lid-8c02dbfadaa70cc5", "milk frother-c5cf435ffba447a5", "can of yeast-5c573fc88948c561", "milk bottle-70560e0cf827e275", "coffee capsule-734c74771eb5ac6f", "box of coffee capsules-611f83c8d12e0d66", "water filter jug-9a6a101762cceb45", "water tank-80942cbe1122e817", "mug-5e6e92894731cbbb", "pot-4bf068ddbf84109b", "empty plate-b34bcbed5870e580" ], "reasoning": "1. I notice the user occasionally stopping and staring during the video.\n2. From 0:56 to 01:03, the user stares at the back of a can of yeast (can of yeast-5c573fc88948c561).\n3. This amount of time equals 7 seconds.\n4. From 01:31-01:45, the user is staring at the coffee machine and the milk frother (milk frother-c5cf435ffba447a5).\n5. This amount of time equals 14 seconds.\n6. Finally, from 02:35-02:53, the user stares out of the kitchen, through a hallway and partially into another room with a window and chairs.\n7. This amount of time equals 18 seconds.\n8. All of these times spent staring are over 7 seconds.\n9. Therefore, the answer is 3 times.", "referenced_object_masks": [ "can of yeast-5c573fc88948c561", "milk frother-c5cf435ffba447a5" ] }, { "key": "P01-20240203-093333:c1a7f94e60849329f0387271e3f98dab16d0a430", "video_id": "P01-20240203-093333", "question": "Once I walk into the kitchen, where do I spend the next 30 seconds walking?", "answer_choice_0": "To the fridge, the coffee machine, the sink, back to the refrigerator, and the water pitcher", "answer_choice_1": "To the sink, the fridge, the shelf by the trash can, the coffee machine, and back to the sink", "answer_choice_2": "To the shelf by the trash can, the fridge, the coffee machine, the water pitcher, and back to the coffee machine", "answer_choice_3": "To the sink, the coffee machine, the water pitcher, back to the coffee machine, and the shelf by the trash can", "answer_choice_4": "To the coffee machine, the shelf by the trash can, back to the coffee machine, the refrigerator, and the water pitcher", "answer_id": 3, "answer": "To the sink, the coffee machine, the water pitcher, back to the coffee machine, and the shelf by the trash can", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Temporal Reasoning", "Object Recognition", "Situational Awareness", "State Changes" ], "objects_of_interest": [ "sponge-4ad8895244419f48", "center piece-a3a784bd19011302", "wooden stirrer-737a4cc5450367cc", "lid-8c02dbfadaa70cc5", "milk frother-c5cf435ffba447a5", "can of yeast-5c573fc88948c561", "milk bottle-70560e0cf827e275", "coffee capsule-734c74771eb5ac6f", "box of coffee capsules-611f83c8d12e0d66", "water filter jug-9a6a101762cceb45", "water tank-80942cbe1122e817", "mug-5e6e92894731cbbb", "pot-4bf068ddbf84109b", "empty plate-b34bcbed5870e580" ], "reasoning": "1. I notice the user enters the kitchen at the 00:05 mark of the video, so I examine the timeframe between 00:05 and 00:35.\n2. The user first moves to the sink , at 00:07, to put down a plate (empty plate-b34bcbed5870e580).\n3. After 00:07, the user moves to a coffee machine at 00:17.\n4. After 00:17, the user moves to a water pitcher (water filter jug-9a6a101762cceb45) by the sink at 00:24.\n5. After 00:24, the user moves back to the coffee machine at 00:30.\n6. After 00:30, the user moves to a shelf by the trash can to grab a box of coffee pods (box of coffee capsules-611f83c8d12e0d66) at 00:33.\n7. By 00:35, the user is still taking the coffee pods (box of coffee capsules-611f83c8d12e0d66) from the shelf.\n8. Therefore, the user moves to the sink, the coffee machine, the water pitcher, back to the coffee machine, and the shelf by the trash can.", "referenced_object_masks": [ "box of coffee capsules-611f83c8d12e0d66", "water filter jug-9a6a101762cceb45", "empty plate-b34bcbed5870e580" ] }, { "key": "P01-20240203-093333:e85f9f15cbe0a7f9c2aec2c5a22a908b37a5a595", "video_id": "P01-20240203-093333", "question": "How much combined time does it take for me to both pour my frothed milk and to duck low to retrieve my coffee pods from the shelf?", "answer_choice_0": "19 seconds", "answer_choice_1": "10 seconds", "answer_choice_2": "24 seconds", "answer_choice_3": "35 seconds", "answer_choice_4": "14 seconds", "answer_id": 4, "answer": "14 seconds", "question_type": [ "Egocentric & Spatial Perception", "Numerical Reasoning", "Action Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "sponge-4ad8895244419f48", "center piece-a3a784bd19011302", "wooden stirrer-737a4cc5450367cc", "lid-8c02dbfadaa70cc5", "milk frother-c5cf435ffba447a5", "can of yeast-5c573fc88948c561", "milk bottle-70560e0cf827e275", "coffee capsule-734c74771eb5ac6f", "box of coffee capsules-611f83c8d12e0d66", "water filter jug-9a6a101762cceb45", "water tank-80942cbe1122e817", "mug-5e6e92894731cbbb", "pot-4bf068ddbf84109b", "empty plate-b34bcbed5870e580" ], "reasoning": "1. I notice the user ducking to retrieve coffee pods (box of coffee capsules-611f83c8d12e0d66) at 00:33 and righting themselves at 00:35.\n2. This makes for a total of 2 seconds spent ducking.\n3. I notice the user pouring the frothed milk from the milk frother (milk frother-c5cf435ffba447a5) from 01:55-02:07.\n4. This makes for a total of 12 seconds spent pouring frothed milk.\n5. I add 2 seconds to 12 seconds to get 14 seconds.\n6. Therefore, 14 seconds is the answer.", "referenced_object_masks": [ "box of coffee capsules-611f83c8d12e0d66", "milk frother-c5cf435ffba447a5" ] }, { "key": "P01-20240203-121517:3f3fb716b2538e97420270944f51e099f718bfe1", "video_id": "P01-20240203-121517", "question": "Of the items I touched, which one is placed highest?", "answer_choice_0": "A water glass", "answer_choice_1": "A sugar jar", "answer_choice_2": "A carton of eggs", "answer_choice_3": "A small bowl", "answer_choice_4": "A bottle of orange juice", "answer_id": 1, "answer": "A sugar jar", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Object Recognition" ], "objects_of_interest": [ "spoon-25b216bcdb2db565", "bottle of salt-1f3ee0f27874ba9a", "glass-a7167e04a4ae18f2", "small glass-1829c9bb549e70ff", "inner bowl-0cdccc8c4fcacc15", "blue bag of flour-2341c5a60db66cc0", "bag of pasta-e89bb55af47a11dc", "green bag of flour-1917a46b8ec0662b", "large bowl-7f1816c23a6f1778", "plate2-bd87a245e46e6a52", "cover of jar-fc24f7a7b2c62222", "lid-10d9b429ceec667c", "lid of sugar jar-c40e0af62510a984", "sugar jar-d2b5292d9b2479ab", "teaspoon-b7c71c24697697be", "bag of flour-f4073161ef877806", "packet of crisps-81990c4abec0c09c", "second packet of herbs-6f28efd28ffeddd8", "bag of herbs-b9b30f5091a08b6f", "onions-7c8054c5ac54b7ee", "mesh of oranges-d517b2261f5f2c12", "plastic package of salad-e90fc9eadf0510a3", "bag of carrots-474dfe25eb2aabf7", "box of chicken stock-83a1d496488ffd40", "third shopping bag-138268fba41e5c5f", "cheese package-e4d3dedefd866491", "eggs box-39da197d6ab20dd4", "bag-1c87f22a3805d1c0", "jar of tahini-15f584c48870ecca", "can of condensed milk-714ebf2d4358f8b3", "butter-71ac54c3c9f0c5b2", "jar of nutella-7348265e18c9f5a8", "minced meat-cf040fe728706b2b", "yeast can-ccead79ab5acab41", "packet of pasta-d3b5abd8e0e7ce36", "plastic box-3ec1d5a0b4b89c5b", "two lemon halves-5e9fb970a316da3a", "bottle of milk-36999353b8440a58", "packet of biscuits-c404bf2724d62bfb", "bowl-58d1ee1b83b5c249", "plate-8b62a1b10b38a94d", "water jug filter-0e14179ad7a577a0", "kettle-50cebab09e7b6ae3", "one package of biscuits-cf0b371baa8b559b", "brown grocery bag-0b69256534f104cf", "white grocery bag-195ab3ac6de0f58b", "new plastic box-ea05838dc57c92d7" ], "reasoning": "1. I watched the video and looked for moments where the user touched things and I could see their arm and hand stretched upward in the frame.\n2. I can see the user's right arm fully extended upward when they take the jar of sugar (sugar jar-d2b5292d9b2479ab) from the top shelf above the sink at 03:26. While the user reaches upwards throughout the video, there is no other time where their arm is fully extended upward.\n3. Therefore the item placed highest that the user touches is the jar of sugar (sugar jar-d2b5292d9b2479ab).", "referenced_object_masks": [ "sugar jar-d2b5292d9b2479ab" ] }, { "key": "P01-20240203-121517:732cbb723b6164e501a9ac443d94f52cb553adb8", "video_id": "P01-20240203-121517", "question": "How many round items are on the counter to my right when I'm measuring flour into a glass?", "answer_choice_0": "10", "answer_choice_1": "7", "answer_choice_2": "2", "answer_choice_3": "11", "answer_choice_4": "5", "answer_id": 0, "answer": "10", "question_type": [ "Action Recognition", "Counting", "Egocentric & Spatial Perception", "Event Occurence", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "spoon-25b216bcdb2db565", "bottle of salt-1f3ee0f27874ba9a", "glass-a7167e04a4ae18f2", "small glass-1829c9bb549e70ff", "inner bowl-0cdccc8c4fcacc15", "blue bag of flour-2341c5a60db66cc0", "bag of pasta-e89bb55af47a11dc", "green bag of flour-1917a46b8ec0662b", "large bowl-7f1816c23a6f1778", "plate2-bd87a245e46e6a52", "cover of jar-fc24f7a7b2c62222", "lid-10d9b429ceec667c", "lid of sugar jar-c40e0af62510a984", "sugar jar-d2b5292d9b2479ab", "teaspoon-b7c71c24697697be", "bag of flour-f4073161ef877806", "packet of crisps-81990c4abec0c09c", "second packet of herbs-6f28efd28ffeddd8", "bag of herbs-b9b30f5091a08b6f", "onions-7c8054c5ac54b7ee", "mesh of oranges-d517b2261f5f2c12", "plastic package of salad-e90fc9eadf0510a3", "bag of carrots-474dfe25eb2aabf7", "box of chicken stock-83a1d496488ffd40", "third shopping bag-138268fba41e5c5f", "cheese package-e4d3dedefd866491", "eggs box-39da197d6ab20dd4", "bag-1c87f22a3805d1c0", "jar of tahini-15f584c48870ecca", "can of condensed milk-714ebf2d4358f8b3", "butter-71ac54c3c9f0c5b2", "jar of nutella-7348265e18c9f5a8", "minced meat-cf040fe728706b2b", "yeast can-ccead79ab5acab41", "packet of pasta-d3b5abd8e0e7ce36", "plastic box-3ec1d5a0b4b89c5b", "two lemon halves-5e9fb970a316da3a", "bottle of milk-36999353b8440a58", "packet of biscuits-c404bf2724d62bfb", "bowl-58d1ee1b83b5c249", "plate-8b62a1b10b38a94d", "water jug filter-0e14179ad7a577a0", "kettle-50cebab09e7b6ae3", "one package of biscuits-cf0b371baa8b559b", "brown grocery bag-0b69256534f104cf", "white grocery bag-195ab3ac6de0f58b", "new plastic box-ea05838dc57c92d7" ], "reasoning": "1. I watched the video until I saw the user pour flour from a bag (green bag of flour-1917a46b8ec0662b) into a glass (glass-a7167e04a4ae18f2) set into a white bowl (large bowl-7f1816c23a6f1778) at 06:02.\n2. I went back in the video to find a clear view of the particular area of the counter from 02:27-02:35 in order to identify round objects. In that time span I can identify 5 oranges in the new mesh bag of oranges (mesh of oranges-d517b2261f5f2c12), 3 onions in the mesh bag of onions (onions-7c8054c5ac54b7ee), and 2 more oranges in the old mesh bag of oranges .\n3. Counting all of the onions and oranges gave me a total of 10.", "referenced_object_masks": [ "large bowl-7f1816c23a6f1778", "glass-a7167e04a4ae18f2", "green bag of flour-1917a46b8ec0662b", "onions-7c8054c5ac54b7ee", "mesh of oranges-d517b2261f5f2c12" ] }, { "key": "P01-20240203-121517:8b6efced9695a5d2bc91fdd42cf0745331e1d435", "video_id": "P01-20240203-121517", "question": "If I went to look into the refrigerator after filling the first bowl with water, what item would be directly behind and closest to me?", "answer_choice_0": "A bag of herbs", "answer_choice_1": "A microwave", "answer_choice_2": "A bag of oranges", "answer_choice_3": "A bowl of water", "answer_choice_4": "A cutting board", "answer_id": 0, "answer": "A bag of herbs", "question_type": [ "Counterfactual", "Action Recognition", "Egocentric & Spatial Perception", "Event Occurence", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "spoon-25b216bcdb2db565", "bottle of salt-1f3ee0f27874ba9a", "glass-a7167e04a4ae18f2", "small glass-1829c9bb549e70ff", "inner bowl-0cdccc8c4fcacc15", "blue bag of flour-2341c5a60db66cc0", "bag of pasta-e89bb55af47a11dc", "green bag of flour-1917a46b8ec0662b", "large bowl-7f1816c23a6f1778", "plate2-bd87a245e46e6a52", "cover of jar-fc24f7a7b2c62222", "lid-10d9b429ceec667c", "lid of sugar jar-c40e0af62510a984", "sugar jar-d2b5292d9b2479ab", "teaspoon-b7c71c24697697be", "bag of flour-f4073161ef877806", "packet of crisps-81990c4abec0c09c", "second packet of herbs-6f28efd28ffeddd8", "bag of herbs-b9b30f5091a08b6f", "onions-7c8054c5ac54b7ee", "mesh of oranges-d517b2261f5f2c12", "plastic package of salad-e90fc9eadf0510a3", "bag of carrots-474dfe25eb2aabf7", "box of chicken stock-83a1d496488ffd40", "third shopping bag-138268fba41e5c5f", "cheese package-e4d3dedefd866491", "eggs box-39da197d6ab20dd4", "bag-1c87f22a3805d1c0", "jar of tahini-15f584c48870ecca", "can of condensed milk-714ebf2d4358f8b3", "butter-71ac54c3c9f0c5b2", "jar of nutella-7348265e18c9f5a8", "minced meat-cf040fe728706b2b", "yeast can-ccead79ab5acab41", "packet of pasta-d3b5abd8e0e7ce36", "plastic box-3ec1d5a0b4b89c5b", "two lemon halves-5e9fb970a316da3a", "bottle of milk-36999353b8440a58", "packet of biscuits-c404bf2724d62bfb", "bowl-58d1ee1b83b5c249", "plate-8b62a1b10b38a94d", "water jug filter-0e14179ad7a577a0", "kettle-50cebab09e7b6ae3", "one package of biscuits-cf0b371baa8b559b", "brown grocery bag-0b69256534f104cf", "white grocery bag-195ab3ac6de0f58b", "new plastic box-ea05838dc57c92d7" ], "reasoning": "1. I watched to see when the user filled the first bowl with water and saw them pour water from a kettle (kettle-50cebab09e7b6ae3) into a bowl (bowl-58d1ee1b83b5c249) at 02:49.\n2. I determined that the end of the counter where the user sets their grocery bags down is directly across from the refrigerator when the user turns around to the right 180 degrees from 00:51-00:53.\n3. I saw the user putting away items from their grocery bags at that particular part of the counter from 00:46-02:33.\n4. At 02:20 I can see the user pull out a bag of herbs (second packet of herbs-6f28efd28ffeddd8) and place it on the edge of the counter nearest to the refrigerator .\n5. At 02:39, just before the user walks to the other end of the kitchen to fill the bowl (bowl-58d1ee1b83b5c249), I can see that the bag of herbs (second packet of herbs-6f28efd28ffeddd8) is still the closest object to the refrigerator.\n6. Therefore, the item directly behind the user and closest to them is the bag of herbs (second packet of herbs-6f28efd28ffeddd8).", "referenced_object_masks": [ "bowl-58d1ee1b83b5c249", "second packet of herbs-6f28efd28ffeddd8", "kettle-50cebab09e7b6ae3" ] }, { "key": "P01-20240203-121517:b07f23c2db6d1aae9557b9ab437725d3f8e4bdce", "video_id": "P01-20240203-121517", "question": "What's the difference between the number of times I tapped the bottom of the white bowl and the number of items I placed in the refrigerator?", "answer_choice_0": "1", "answer_choice_1": "3", "answer_choice_2": "4", "answer_choice_3": "0", "answer_choice_4": "2", "answer_id": 1, "answer": "3", "question_type": [ "Action Recognition", "Counting", "Numerical Reasoning", "Temporal Reasoning" ], "objects_of_interest": [ "spoon-25b216bcdb2db565", "bottle of salt-1f3ee0f27874ba9a", "glass-a7167e04a4ae18f2", "small glass-1829c9bb549e70ff", "inner bowl-0cdccc8c4fcacc15", "blue bag of flour-2341c5a60db66cc0", "bag of pasta-e89bb55af47a11dc", "green bag of flour-1917a46b8ec0662b", "large bowl-7f1816c23a6f1778", "plate2-bd87a245e46e6a52", "cover of jar-fc24f7a7b2c62222", "lid-10d9b429ceec667c", "lid of sugar jar-c40e0af62510a984", "sugar jar-d2b5292d9b2479ab", "teaspoon-b7c71c24697697be", "bag of flour-f4073161ef877806", "packet of crisps-81990c4abec0c09c", "second packet of herbs-6f28efd28ffeddd8", "bag of herbs-b9b30f5091a08b6f", "onions-7c8054c5ac54b7ee", "mesh of oranges-d517b2261f5f2c12", "plastic package of salad-e90fc9eadf0510a3", "bag of carrots-474dfe25eb2aabf7", "box of chicken stock-83a1d496488ffd40", "third shopping bag-138268fba41e5c5f", "cheese package-e4d3dedefd866491", "eggs box-39da197d6ab20dd4", "bag-1c87f22a3805d1c0", "jar of tahini-15f584c48870ecca", "can of condensed milk-714ebf2d4358f8b3", "butter-71ac54c3c9f0c5b2", "jar of nutella-7348265e18c9f5a8", "minced meat-cf040fe728706b2b", "yeast can-ccead79ab5acab41", "packet of pasta-d3b5abd8e0e7ce36", "plastic box-3ec1d5a0b4b89c5b", "two lemon halves-5e9fb970a316da3a", "bottle of milk-36999353b8440a58", "packet of biscuits-c404bf2724d62bfb", "bowl-58d1ee1b83b5c249", "plate-8b62a1b10b38a94d", "water jug filter-0e14179ad7a577a0", "kettle-50cebab09e7b6ae3", "one package of biscuits-cf0b371baa8b559b", "brown grocery bag-0b69256534f104cf", "white grocery bag-195ab3ac6de0f58b", "new plastic box-ea05838dc57c92d7" ], "reasoning": "1. I watched until I saw the user invert the bowl (large bowl-7f1816c23a6f1778) over the sink and tap its bottom 3 times from 06:12-06:13.\n2. I watched the video and saw the user place inside the refrigerator a jug of milk (bottle of milk-36999353b8440a58) at 00:58, a jar of Nutella (jar of nutella-7348265e18c9f5a8) at 01:23, a package of ground beef (minced meat-cf040fe728706b2b) at 01:25, a stick of butter (butter-71ac54c3c9f0c5b2) at 01:40, a package of cheese (cheese package-e4d3dedefd866491) at 01:56, and a package of eggs (eggs box-39da197d6ab20dd4) at 01:59 for a total of 6 items.\n3. I subtracted 3 from 6 to reach a difference of 3.", "referenced_object_masks": [ "large bowl-7f1816c23a6f1778", "eggs box-39da197d6ab20dd4", "bottle of milk-36999353b8440a58", "butter-71ac54c3c9f0c5b2", "minced meat-cf040fe728706b2b", "cheese package-e4d3dedefd866491", "jar of nutella-7348265e18c9f5a8" ] }, { "key": "P01-20240203-121517:de2123255fd597218e20d54f334ef3d5350de7dc", "video_id": "P01-20240203-121517", "question": "If I had picked up the kettle with my left hand, how many unique objects in total would I have touched with my left hand by the time I first picked up the kettle (including the kettle)?", "answer_choice_0": "2", "answer_choice_1": "5", "answer_choice_2": "3", "answer_choice_3": "1", "answer_choice_4": "4", "answer_id": 1, "answer": "5", "question_type": [ "Action Recognition", "Counterfactual", "Counting", "Egocentric & Spatial Perception", "Temporal Reasoning" ], "objects_of_interest": [ "spoon-25b216bcdb2db565", "bottle of salt-1f3ee0f27874ba9a", "glass-a7167e04a4ae18f2", "small glass-1829c9bb549e70ff", "inner bowl-0cdccc8c4fcacc15", "blue bag of flour-2341c5a60db66cc0", "bag of pasta-e89bb55af47a11dc", "green bag of flour-1917a46b8ec0662b", "large bowl-7f1816c23a6f1778", "plate2-bd87a245e46e6a52", "cover of jar-fc24f7a7b2c62222", "lid-10d9b429ceec667c", "lid of sugar jar-c40e0af62510a984", "sugar jar-d2b5292d9b2479ab", "teaspoon-b7c71c24697697be", "bag of flour-f4073161ef877806", "packet of crisps-81990c4abec0c09c", "second packet of herbs-6f28efd28ffeddd8", "bag of herbs-b9b30f5091a08b6f", "onions-7c8054c5ac54b7ee", "mesh of oranges-d517b2261f5f2c12", "plastic package of salad-e90fc9eadf0510a3", "bag of carrots-474dfe25eb2aabf7", "box of chicken stock-83a1d496488ffd40", "third shopping bag-138268fba41e5c5f", "cheese package-e4d3dedefd866491", "eggs box-39da197d6ab20dd4", "bag-1c87f22a3805d1c0", "jar of tahini-15f584c48870ecca", "can of condensed milk-714ebf2d4358f8b3", "butter-71ac54c3c9f0c5b2", "jar of nutella-7348265e18c9f5a8", "minced meat-cf040fe728706b2b", "yeast can-ccead79ab5acab41", "packet of pasta-d3b5abd8e0e7ce36", "plastic box-3ec1d5a0b4b89c5b", "two lemon halves-5e9fb970a316da3a", "bottle of milk-36999353b8440a58", "packet of biscuits-c404bf2724d62bfb", "bowl-58d1ee1b83b5c249", "plate-8b62a1b10b38a94d", "water jug filter-0e14179ad7a577a0", "kettle-50cebab09e7b6ae3", "one package of biscuits-cf0b371baa8b559b", "brown grocery bag-0b69256534f104cf", "white grocery bag-195ab3ac6de0f58b", "new plastic box-ea05838dc57c92d7" ], "reasoning": "1. I watched the video and began counting the objects the user touches with their left hand.\n2. I saw the user touch the tote bag (brown grocery bag-0b69256534f104cf) at 0:02, the other bag of groceries (white grocery bag-195ab3ac6de0f58b) at 00:06, the bag of tea cookies (one package of biscuits-cf0b371baa8b559b) at 00:09, and the water pitcher (water jug filter-0e14179ad7a577a0) at 00:20 for a total of 4 unique objects.\n3. At 00:21 I saw the user pick up the kettle (kettle-50cebab09e7b6ae3) with their right hand for the first time. If they had picked it up with their left, that would be the fifth unique object.", "referenced_object_masks": [ "white grocery bag-195ab3ac6de0f58b", "water jug filter-0e14179ad7a577a0", "kettle-50cebab09e7b6ae3", "one package of biscuits-cf0b371baa8b559b", "brown grocery bag-0b69256534f104cf" ] }, { "key": "P01-20240203-123350:07eff9213c7db5a41e46edebc8dd97c0cfb6cdf9", "video_id": "P01-20240203-123350", "question": "After placing the bowl of flour down on the counter for the first time, what is the third object I touch with my right hand?", "answer_choice_0": "Bowl of flour", "answer_choice_1": "Vegetable Oil", "answer_choice_2": "Olive Oil", "answer_choice_3": "Spoon", "answer_choice_4": "Balsamic vinegar", "answer_id": 2, "answer": "Olive Oil", "question_type": [ "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counting" ], "objects_of_interest": [ "carrot10-0aae52eddad55523", "carrot9-786e7f1c6a225c71", "carrot8-7623c13c407a327b", "carrot7-2b7a43a332504a51", "carrot6-975e4a3804a0ba76", "carrot5-c55e1a891787c67a", "carrot4-42df7816f7f589c5", "carrot3-c3a43afebd820a86", "carrot2-532aa17893a6e110", "plastic cover of peeler-6075adba6cba9311", "plastic tupperware-dea9178b19800ade", "carrot1-975ab66b998500ec", "mesh of oranges-fb26a0288c9637ea", "spoon2-8558830aa3073276", "plastic package of salad-17d1f6ca0909bf5b", "part of cover2-fbeb40cafec4369b", "chicken stock cube2-13a57f8bebccfb63", "part of cover-8adb9bf79df13865", "chicken stock cube-39ee8c4bc6e4e1cf", "box of chicken stock-1b6260f9d91edf71", "kettle-5b399fca36e15db5", "mug-1c8b74e0e887c6d7", "cover of plastic package-d7dc32e7d938b2aa", "paper-1577fe9bf3a6da7d", "garlic presser-120a303847616d27", "chopping board4-fc0e37ac72161ef7", "bag of carrots-9f74fd12b417bdc8", "scissors cover-a6205ce6ec86f841", "scissors-b7500debea02c36b", "peeler-ce2dccad611eec54", "plastic cover-e8516b1355630251", "plastic serving spoon-b763996485190f8c", "chopping board3-f3c34af39fab0708", "clove of garlic-86a50ba9146f88ef", "glass lid2-1a1c0412fbdc8f58", "glass lid-c142bd4e12156f19", "wooden spoon-daa08e1f95bbad19", "pan2-ccf19cacfc2943b6", "saucepan2-3c6a47036d8474c2", "packet of minced meat2-18c5e3e040333bea", "packet of minced meat-ac322553d2441535", "salt bottle-36852988b45e6bd3", "chopping board2-6932d696acd5032a", "knife2-ae61b780aa68a4a0", "chopping board-2e3e4f97b1242861", "knife-5a16c68c09537f9f", "onion2-a9afa5b7b76cd1b7", "onion-224e797dfe26cc13", "dirty glass-f0b6b06a86bb7d64", "mesh-e95446965746807f", "pan-4948d2e5651ada57", "saucepan-fbee1e79b449d5e0", "small notebook-b1e44e5112c3b86e", "white towel-c54866de0cab1d79", "blue towel-e98a9f7a908fd99e", "large bowl2-5f736a42eb1057e1", "bottle of oil cap-88edc5df14eaa961", "plastic bag-df3e5dbb5d0f3acf", "bag of flour-b148433d9dfdad75", "water filter jug-ee749c4996f36500", "bowl-5951775fa8b8bace", "spoon-decaa21c7b1c70d5", "can of yeast-4e4be53413d4f697", "bowl of flour-23decab0fbb85e18", "bottle of olive oil-08ebf6ae6cefc83a", "bottle of balsemic vinegar-d463235fcbcd8004", "bottle of vegetable oil-9bf7fc9adb0a25e6", "plate-0ea1342f1b55fbb9", "new onion half-2e97349d356bac9a" ], "reasoning": "1. I observe the user place down the bowl of flour (bowl of flour-23decab0fbb85e18) to the right of the stovetop for the first time at 00:22.\n2 I observe the user touch the bowl (bowl-5951775fa8b8bace) to the right of the bowl of flour (bowl of flour-23decab0fbb85e18) at 00:23 with their right hand.\n3. I observe the user touch the spoon (spoon-decaa21c7b1c70d5) in the bowl of flour (bowl of flour-23decab0fbb85e18) at 00:24 with their right hand.\n4. I observe the user turn around to grab the olive oil (bottle of olive oil-08ebf6ae6cefc83a) from 00:26 to 00:28 with their right hand.\n5. I determine that the 3rd object the user touches with their right hand is the olive oil.", "referenced_object_masks": [ "bottle of olive oil-08ebf6ae6cefc83a", "bowl of flour-23decab0fbb85e18", "bowl-5951775fa8b8bace", "spoon-decaa21c7b1c70d5" ] }, { "key": "P01-20240203-123350:2979ff9928783ad5e6a719712c699eb00d1dd4fc", "video_id": "P01-20240203-123350", "question": "When I place the water filter down the second time, where is it in relation to its original location?", "answer_choice_0": "Behind", "answer_choice_1": "In front of", "answer_choice_2": "To the left", "answer_choice_3": "Above", "answer_choice_4": "To the right", "answer_id": 2, "answer": "To the left", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counting" ], "objects_of_interest": [ "carrot10-0aae52eddad55523", "carrot9-786e7f1c6a225c71", "carrot8-7623c13c407a327b", "carrot7-2b7a43a332504a51", "carrot6-975e4a3804a0ba76", "carrot5-c55e1a891787c67a", "carrot4-42df7816f7f589c5", "carrot3-c3a43afebd820a86", "carrot2-532aa17893a6e110", "plastic cover of peeler-6075adba6cba9311", "plastic tupperware-dea9178b19800ade", "carrot1-975ab66b998500ec", "mesh of oranges-fb26a0288c9637ea", "spoon2-8558830aa3073276", "plastic package of salad-17d1f6ca0909bf5b", "part of cover2-fbeb40cafec4369b", "chicken stock cube2-13a57f8bebccfb63", "part of cover-8adb9bf79df13865", "chicken stock cube-39ee8c4bc6e4e1cf", "box of chicken stock-1b6260f9d91edf71", "kettle-5b399fca36e15db5", "mug-1c8b74e0e887c6d7", "cover of plastic package-d7dc32e7d938b2aa", "paper-1577fe9bf3a6da7d", "garlic presser-120a303847616d27", "chopping board4-fc0e37ac72161ef7", "bag of carrots-9f74fd12b417bdc8", "scissors cover-a6205ce6ec86f841", "scissors-b7500debea02c36b", "peeler-ce2dccad611eec54", "plastic cover-e8516b1355630251", "plastic serving spoon-b763996485190f8c", "chopping board3-f3c34af39fab0708", "clove of garlic-86a50ba9146f88ef", "glass lid2-1a1c0412fbdc8f58", "glass lid-c142bd4e12156f19", "wooden spoon-daa08e1f95bbad19", "pan2-ccf19cacfc2943b6", "saucepan2-3c6a47036d8474c2", "packet of minced meat2-18c5e3e040333bea", "packet of minced meat-ac322553d2441535", "salt bottle-36852988b45e6bd3", "chopping board2-6932d696acd5032a", "knife2-ae61b780aa68a4a0", "chopping board-2e3e4f97b1242861", "knife-5a16c68c09537f9f", "onion2-a9afa5b7b76cd1b7", "onion-224e797dfe26cc13", "dirty glass-f0b6b06a86bb7d64", "mesh-e95446965746807f", "pan-4948d2e5651ada57", "saucepan-fbee1e79b449d5e0", "small notebook-b1e44e5112c3b86e", "white towel-c54866de0cab1d79", "blue towel-e98a9f7a908fd99e", "large bowl2-5f736a42eb1057e1", "bottle of oil cap-88edc5df14eaa961", "plastic bag-df3e5dbb5d0f3acf", "bag of flour-b148433d9dfdad75", "water filter jug-ee749c4996f36500", "bowl-5951775fa8b8bace", "spoon-decaa21c7b1c70d5", "can of yeast-4e4be53413d4f697", "bowl of flour-23decab0fbb85e18", "bottle of olive oil-08ebf6ae6cefc83a", "bottle of balsemic vinegar-d463235fcbcd8004", "bottle of vegetable oil-9bf7fc9adb0a25e6", "plate-0ea1342f1b55fbb9", "new onion half-2e97349d356bac9a" ], "reasoning": "1. I observe the user pick up the water filter (water filter jug-ee749c4996f36500) from the counter directly below the silver knobs on the black cabinet above at 01:39.\n2. I observe the user place the water filter (water filter jug-ee749c4996f36500) onto the stovetop at 01:45.\n3. I observe the user pick up the water filter (water filter jug-ee749c4996f36500) and place it in front of the knife block at 07:22.\n4. I determine the user moves the water filter (water filter jug-ee749c4996f36500) to the left of its original location when they place it down the second time.", "referenced_object_masks": [ "water filter jug-ee749c4996f36500" ] }, { "key": "P01-20240203-123350:2ea09210936c600086c53e771727daab25b30b9f", "video_id": "P01-20240203-123350", "question": "While pouring from the kettle, how would I turn to grab a bottle of balsamic vinegar?", "answer_choice_0": "90 degrees clockwise", "answer_choice_1": "90 degrees counterclockwise", "answer_choice_2": "45 degrees to counterclockwise", "answer_choice_3": "45 degrees clockwise", "answer_choice_4": "180 degrees", "answer_id": 0, "answer": "90 degrees clockwise", "question_type": [ "Egocentric & Spatial Perception", "Counterfactual", "Object Recognition", "Action Recognition", "Situational Awareness" ], "objects_of_interest": [ "carrot10-0aae52eddad55523", "carrot9-786e7f1c6a225c71", "carrot8-7623c13c407a327b", "carrot7-2b7a43a332504a51", "carrot6-975e4a3804a0ba76", "carrot5-c55e1a891787c67a", "carrot4-42df7816f7f589c5", "carrot3-c3a43afebd820a86", "carrot2-532aa17893a6e110", "plastic cover of peeler-6075adba6cba9311", "plastic tupperware-dea9178b19800ade", "carrot1-975ab66b998500ec", "mesh of oranges-fb26a0288c9637ea", "spoon2-8558830aa3073276", "plastic package of salad-17d1f6ca0909bf5b", "part of cover2-fbeb40cafec4369b", "chicken stock cube2-13a57f8bebccfb63", "part of cover-8adb9bf79df13865", "chicken stock cube-39ee8c4bc6e4e1cf", "box of chicken stock-1b6260f9d91edf71", "kettle-5b399fca36e15db5", "mug-1c8b74e0e887c6d7", "cover of plastic package-d7dc32e7d938b2aa", "paper-1577fe9bf3a6da7d", "garlic presser-120a303847616d27", "chopping board4-fc0e37ac72161ef7", "bag of carrots-9f74fd12b417bdc8", "scissors cover-a6205ce6ec86f841", "scissors-b7500debea02c36b", "peeler-ce2dccad611eec54", "plastic cover-e8516b1355630251", "plastic serving spoon-b763996485190f8c", "chopping board3-f3c34af39fab0708", "clove of garlic-86a50ba9146f88ef", "glass lid2-1a1c0412fbdc8f58", "glass lid-c142bd4e12156f19", "wooden spoon-daa08e1f95bbad19", "pan2-ccf19cacfc2943b6", "saucepan2-3c6a47036d8474c2", "packet of minced meat2-18c5e3e040333bea", "packet of minced meat-ac322553d2441535", "salt bottle-36852988b45e6bd3", "chopping board2-6932d696acd5032a", "knife2-ae61b780aa68a4a0", "chopping board-2e3e4f97b1242861", "knife-5a16c68c09537f9f", "onion2-a9afa5b7b76cd1b7", "onion-224e797dfe26cc13", "dirty glass-f0b6b06a86bb7d64", "mesh-e95446965746807f", "pan-4948d2e5651ada57", "saucepan-fbee1e79b449d5e0", "small notebook-b1e44e5112c3b86e", "white towel-c54866de0cab1d79", "blue towel-e98a9f7a908fd99e", "large bowl2-5f736a42eb1057e1", "bottle of oil cap-88edc5df14eaa961", "plastic bag-df3e5dbb5d0f3acf", "bag of flour-b148433d9dfdad75", "water filter jug-ee749c4996f36500", "bowl-5951775fa8b8bace", "spoon-decaa21c7b1c70d5", "can of yeast-4e4be53413d4f697", "bowl of flour-23decab0fbb85e18", "bottle of olive oil-08ebf6ae6cefc83a", "bottle of balsemic vinegar-d463235fcbcd8004", "bottle of vegetable oil-9bf7fc9adb0a25e6", "plate-0ea1342f1b55fbb9", "new onion half-2e97349d356bac9a" ], "reasoning": "1. I watched the video to find the location of the bottle of balsamic vinegar (bottle of balsemic vinegar-d463235fcbcd8004) and found it at 00:13 on a shelf. \n2. I watched the video to find out when the user poured from a kettle (kettle-5b399fca36e15db5) and found that this occurred from \n19:25-19:28. \n3. I observed the surroundings from 19:25-19:28 and noted that the kettle (kettle-5b399fca36e15db5) was on the counter to the right of the stovetop . \n4. I continued watching and found a full view of the space at 19:36. I observed the entire kitchen and noted that the counter right of the stovetop ran perpendicular to the wall with the sink . Above and to the right of the sink was a shelf with the bottle of balsamic vinegar (bottle of balsemic vinegar-d463235fcbcd8004). \n4. Therefore, I determined that when the user faced the kettle (kettle-5b399fca36e15db5), they would have had to turn 90 degrees clockwise to grab the bottle of balsamic vinegar (bottle of balsemic vinegar-d463235fcbcd8004).", "referenced_object_masks": [ "bottle of balsemic vinegar-d463235fcbcd8004", "kettle-5b399fca36e15db5" ] }, { "key": "P01-20240203-123350:4d0605931122a91c5b06227389a6fd8b1f61c21e", "video_id": "P01-20240203-123350", "question": "How many times did I slice an onion before taking the meat out of the fridge?", "answer_choice_0": "6", "answer_choice_1": "15", "answer_choice_2": "9", "answer_choice_3": "3", "answer_choice_4": "12", "answer_id": 0, "answer": "6", "question_type": [ "Counting", "Temporal Reasoning", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "carrot10-0aae52eddad55523", "carrot9-786e7f1c6a225c71", "carrot8-7623c13c407a327b", "carrot7-2b7a43a332504a51", "carrot6-975e4a3804a0ba76", "carrot5-c55e1a891787c67a", "carrot4-42df7816f7f589c5", "carrot3-c3a43afebd820a86", "carrot2-532aa17893a6e110", "plastic cover of peeler-6075adba6cba9311", "plastic tupperware-dea9178b19800ade", "carrot1-975ab66b998500ec", "mesh of oranges-fb26a0288c9637ea", "spoon2-8558830aa3073276", "plastic package of salad-17d1f6ca0909bf5b", "part of cover2-fbeb40cafec4369b", "chicken stock cube2-13a57f8bebccfb63", "part of cover-8adb9bf79df13865", "chicken stock cube-39ee8c4bc6e4e1cf", "box of chicken stock-1b6260f9d91edf71", "kettle-5b399fca36e15db5", "mug-1c8b74e0e887c6d7", "cover of plastic package-d7dc32e7d938b2aa", "paper-1577fe9bf3a6da7d", "garlic presser-120a303847616d27", "chopping board4-fc0e37ac72161ef7", "bag of carrots-9f74fd12b417bdc8", "scissors cover-a6205ce6ec86f841", "scissors-b7500debea02c36b", "peeler-ce2dccad611eec54", "plastic cover-e8516b1355630251", "plastic serving spoon-b763996485190f8c", "chopping board3-f3c34af39fab0708", "clove of garlic-86a50ba9146f88ef", "glass lid2-1a1c0412fbdc8f58", "glass lid-c142bd4e12156f19", "wooden spoon-daa08e1f95bbad19", "pan2-ccf19cacfc2943b6", "saucepan2-3c6a47036d8474c2", "packet of minced meat2-18c5e3e040333bea", "packet of minced meat-ac322553d2441535", "salt bottle-36852988b45e6bd3", "chopping board2-6932d696acd5032a", "knife2-ae61b780aa68a4a0", "chopping board-2e3e4f97b1242861", "knife-5a16c68c09537f9f", "onion2-a9afa5b7b76cd1b7", "onion-224e797dfe26cc13", "dirty glass-f0b6b06a86bb7d64", "mesh-e95446965746807f", "pan-4948d2e5651ada57", "saucepan-fbee1e79b449d5e0", "small notebook-b1e44e5112c3b86e", "white towel-c54866de0cab1d79", "blue towel-e98a9f7a908fd99e", "large bowl2-5f736a42eb1057e1", "bottle of oil cap-88edc5df14eaa961", "plastic bag-df3e5dbb5d0f3acf", "bag of flour-b148433d9dfdad75", "water filter jug-ee749c4996f36500", "bowl-5951775fa8b8bace", "spoon-decaa21c7b1c70d5", "can of yeast-4e4be53413d4f697", "bowl of flour-23decab0fbb85e18", "bottle of olive oil-08ebf6ae6cefc83a", "bottle of balsemic vinegar-d463235fcbcd8004", "bottle of vegetable oil-9bf7fc9adb0a25e6", "plate-0ea1342f1b55fbb9", "new onion half-2e97349d356bac9a" ], "reasoning": "1. I watched the video and saw the user take meat (packet of minced meat2-18c5e3e040333bea) out of the fridge from 09:08-09:12.\n2. I noticed an onion being sliced on 2 separate occasions, once from 07:56-08:06, and another time from 08:11-08:20.\n3. From 07:56-08:06, I watched the user slice the onion (onion-224e797dfe26cc13) 3 times: once on each end and once down the middle. \n4. From 08:11-08:20, I watched the user slice the second onion (onion2-a9afa5b7b76cd1b7) 3 times: once on each end and once down the middle.\n5. There were no more onions sliced before 09:08, so I found a total of 6 slices occurred before the meat (packet of minced meat2-18c5e3e040333bea) was taken out.", "referenced_object_masks": [ "onion-224e797dfe26cc13", "packet of minced meat2-18c5e3e040333bea", "onion2-a9afa5b7b76cd1b7" ] }, { "key": "P01-20240203-123350:55cccc282d0aa40feebb874e63dd86032bbbd90e", "video_id": "P01-20240203-123350", "question": "If I had used vegetable oil in the large bowl with the bread dough instead of olive oil, how many times would I have picked up vegetable oil before adding meat to the pan?", "answer_choice_0": "1", "answer_choice_1": "0", "answer_choice_2": "3", "answer_choice_3": "2", "answer_choice_4": "4", "answer_id": 2, "answer": "3", "question_type": [ "Object Recognition", "Temporal Reasoning", "Counterfactual", "Counting" ], "objects_of_interest": [ "carrot10-0aae52eddad55523", "carrot9-786e7f1c6a225c71", "carrot8-7623c13c407a327b", "carrot7-2b7a43a332504a51", "carrot6-975e4a3804a0ba76", "carrot5-c55e1a891787c67a", "carrot4-42df7816f7f589c5", "carrot3-c3a43afebd820a86", "carrot2-532aa17893a6e110", "plastic cover of peeler-6075adba6cba9311", "plastic tupperware-dea9178b19800ade", "carrot1-975ab66b998500ec", "mesh of oranges-fb26a0288c9637ea", "spoon2-8558830aa3073276", "plastic package of salad-17d1f6ca0909bf5b", "part of cover2-fbeb40cafec4369b", "chicken stock cube2-13a57f8bebccfb63", "part of cover-8adb9bf79df13865", "chicken stock cube-39ee8c4bc6e4e1cf", "box of chicken stock-1b6260f9d91edf71", "kettle-5b399fca36e15db5", "mug-1c8b74e0e887c6d7", "cover of plastic package-d7dc32e7d938b2aa", "paper-1577fe9bf3a6da7d", "garlic presser-120a303847616d27", "chopping board4-fc0e37ac72161ef7", "bag of carrots-9f74fd12b417bdc8", "scissors cover-a6205ce6ec86f841", "scissors-b7500debea02c36b", "peeler-ce2dccad611eec54", "plastic cover-e8516b1355630251", "plastic serving spoon-b763996485190f8c", "chopping board3-f3c34af39fab0708", "clove of garlic-86a50ba9146f88ef", "glass lid2-1a1c0412fbdc8f58", "glass lid-c142bd4e12156f19", "wooden spoon-daa08e1f95bbad19", "pan2-ccf19cacfc2943b6", "saucepan2-3c6a47036d8474c2", "packet of minced meat2-18c5e3e040333bea", "packet of minced meat-ac322553d2441535", "salt bottle-36852988b45e6bd3", "chopping board2-6932d696acd5032a", "knife2-ae61b780aa68a4a0", "chopping board-2e3e4f97b1242861", "knife-5a16c68c09537f9f", "onion2-a9afa5b7b76cd1b7", "onion-224e797dfe26cc13", "dirty glass-f0b6b06a86bb7d64", "mesh-e95446965746807f", "pan-4948d2e5651ada57", "saucepan-fbee1e79b449d5e0", "small notebook-b1e44e5112c3b86e", "white towel-c54866de0cab1d79", "blue towel-e98a9f7a908fd99e", "large bowl2-5f736a42eb1057e1", "bottle of oil cap-88edc5df14eaa961", "plastic bag-df3e5dbb5d0f3acf", "bag of flour-b148433d9dfdad75", "water filter jug-ee749c4996f36500", "bowl-5951775fa8b8bace", "spoon-decaa21c7b1c70d5", "can of yeast-4e4be53413d4f697", "bowl of flour-23decab0fbb85e18", "bottle of olive oil-08ebf6ae6cefc83a", "bottle of balsemic vinegar-d463235fcbcd8004", "bottle of vegetable oil-9bf7fc9adb0a25e6", "plate-0ea1342f1b55fbb9", "new onion half-2e97349d356bac9a" ], "reasoning": "1. I watched the video and saw that the user began adding the meat (packet of minced meat2-18c5e3e040333bea) to the pan (pan-4948d2e5651ada57) at 15:41.\n2. I watched the video and saw that the user picked up the bottle of vegetable oil (bottle of vegetable oil-9bf7fc9adb0a25e6) at 00:13 and 10:50.\n3. This gives a total of 2 times that the bottle of vegetable oil (bottle of vegetable oil-9bf7fc9adb0a25e6) was picked up before the meat (packet of minced meat2-18c5e3e040333bea) was added to the pan (pan-4948d2e5651ada57).\n4. From 05:44-05:46, I see the user add olive oil from a bottle (bottle of olive oil-08ebf6ae6cefc83a) to a large bowl (large bowl2-5f736a42eb1057e1), and from 06:05-06:12, I see the user places the bread dough into the large bowl (large bowl2-5f736a42eb1057e1).\n5. If the user replaced the olive oil (bottle of olive oil-08ebf6ae6cefc83a) poured from 05:44-05:46 with vegetable oil (bottle of vegetable oil-9bf7fc9adb0a25e6), the user would have picked up the vegetable oil (bottle of vegetable oil-9bf7fc9adb0a25e6) 1 additional time.\n9. I add 2 to 1 and get 3.\n6. Therefore, if the vegetable oil (bottle of vegetable oil-9bf7fc9adb0a25e6) had also been used for the large bowl (large bowl2-5f736a42eb1057e1) with the bread dough , the total amount of times it would be picked up would be 3.", "referenced_object_masks": [ "bottle of vegetable oil-9bf7fc9adb0a25e6", "large bowl2-5f736a42eb1057e1", "packet of minced meat2-18c5e3e040333bea", "bottle of olive oil-08ebf6ae6cefc83a", "pan-4948d2e5651ada57" ] }, { "key": "P01-20240203-123350:7ada5a4ad92738ef0f005293314c719d59ef481c", "video_id": "P01-20240203-123350", "question": "When I rinse the second large bowl, where is the bowl of flour in relation to me?", "answer_choice_0": "Above and to the right", "answer_choice_1": "Behind and to the right", "answer_choice_2": "Directly ahead", "answer_choice_3": "Below", "answer_choice_4": "To the left", "answer_id": 1, "answer": "Behind and to the right", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counting" ], "objects_of_interest": [ "carrot10-0aae52eddad55523", "carrot9-786e7f1c6a225c71", "carrot8-7623c13c407a327b", "carrot7-2b7a43a332504a51", "carrot6-975e4a3804a0ba76", "carrot5-c55e1a891787c67a", "carrot4-42df7816f7f589c5", "carrot3-c3a43afebd820a86", "carrot2-532aa17893a6e110", "plastic cover of peeler-6075adba6cba9311", "plastic tupperware-dea9178b19800ade", "carrot1-975ab66b998500ec", "mesh of oranges-fb26a0288c9637ea", "spoon2-8558830aa3073276", "plastic package of salad-17d1f6ca0909bf5b", "part of cover2-fbeb40cafec4369b", "chicken stock cube2-13a57f8bebccfb63", "part of cover-8adb9bf79df13865", "chicken stock cube-39ee8c4bc6e4e1cf", "box of chicken stock-1b6260f9d91edf71", "kettle-5b399fca36e15db5", "mug-1c8b74e0e887c6d7", "cover of plastic package-d7dc32e7d938b2aa", "paper-1577fe9bf3a6da7d", "garlic presser-120a303847616d27", "chopping board4-fc0e37ac72161ef7", "bag of carrots-9f74fd12b417bdc8", "scissors cover-a6205ce6ec86f841", "scissors-b7500debea02c36b", "peeler-ce2dccad611eec54", "plastic cover-e8516b1355630251", "plastic serving spoon-b763996485190f8c", "chopping board3-f3c34af39fab0708", "clove of garlic-86a50ba9146f88ef", "glass lid2-1a1c0412fbdc8f58", "glass lid-c142bd4e12156f19", "wooden spoon-daa08e1f95bbad19", "pan2-ccf19cacfc2943b6", "saucepan2-3c6a47036d8474c2", "packet of minced meat2-18c5e3e040333bea", "packet of minced meat-ac322553d2441535", "salt bottle-36852988b45e6bd3", "chopping board2-6932d696acd5032a", "knife2-ae61b780aa68a4a0", "chopping board-2e3e4f97b1242861", "knife-5a16c68c09537f9f", "onion2-a9afa5b7b76cd1b7", "onion-224e797dfe26cc13", "dirty glass-f0b6b06a86bb7d64", "mesh-e95446965746807f", "pan-4948d2e5651ada57", "saucepan-fbee1e79b449d5e0", "small notebook-b1e44e5112c3b86e", "white towel-c54866de0cab1d79", "blue towel-e98a9f7a908fd99e", "large bowl2-5f736a42eb1057e1", "bottle of oil cap-88edc5df14eaa961", "plastic bag-df3e5dbb5d0f3acf", "bag of flour-b148433d9dfdad75", "water filter jug-ee749c4996f36500", "bowl-5951775fa8b8bace", "spoon-decaa21c7b1c70d5", "can of yeast-4e4be53413d4f697", "bowl of flour-23decab0fbb85e18", "bottle of olive oil-08ebf6ae6cefc83a", "bottle of balsemic vinegar-d463235fcbcd8004", "bottle of vegetable oil-9bf7fc9adb0a25e6", "plate-0ea1342f1b55fbb9", "new onion half-2e97349d356bac9a" ], "reasoning": "1. I observe the user remove the second large bowl (large bowl2-5f736a42eb1057e1) from the blue cabinet at 05:27.\n2. I observe the user rinse the second large bowl (large bowl2-5f736a42eb1057e1) at the sink at 05:31.\n3. I rewind until 05:01 to identify the user standing at the bowl of flour (bowl of flour-23decab0fbb85e18) on the counter in front of them.\n4. From 05:01 to 05:02, I observe the user turn to the left and walk forward towards the sink .\n5. I determine that the bowl of flour (bowl of flour-23decab0fbb85e18) is behind and to the right of the user when they are rinsing the second large bowl (large bowl2-5f736a42eb1057e1).", "referenced_object_masks": [ "bowl of flour-23decab0fbb85e18", "large bowl2-5f736a42eb1057e1" ] }, { "key": "P01-20240203-123350:a6dfe2458a145cae6f705ef021108c4f8da73e5d", "video_id": "P01-20240203-123350", "question": "What is the product of the number of carrots I peeled times the number of towels I put into the oven?", "answer_choice_0": "2", "answer_choice_1": "10", "answer_choice_2": "4", "answer_choice_3": "20", "answer_choice_4": "16", "answer_id": 3, "answer": "20", "question_type": [ "Numerical Reasoning", "Counting", "Object Recognition", "Action Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "carrot10-0aae52eddad55523", "carrot9-786e7f1c6a225c71", "carrot8-7623c13c407a327b", "carrot7-2b7a43a332504a51", "carrot6-975e4a3804a0ba76", "carrot5-c55e1a891787c67a", "carrot4-42df7816f7f589c5", "carrot3-c3a43afebd820a86", "carrot2-532aa17893a6e110", "plastic cover of peeler-6075adba6cba9311", "plastic tupperware-dea9178b19800ade", "carrot1-975ab66b998500ec", "mesh of oranges-fb26a0288c9637ea", "spoon2-8558830aa3073276", "plastic package of salad-17d1f6ca0909bf5b", "part of cover2-fbeb40cafec4369b", "chicken stock cube2-13a57f8bebccfb63", "part of cover-8adb9bf79df13865", "chicken stock cube-39ee8c4bc6e4e1cf", "box of chicken stock-1b6260f9d91edf71", "kettle-5b399fca36e15db5", "mug-1c8b74e0e887c6d7", "cover of plastic package-d7dc32e7d938b2aa", "paper-1577fe9bf3a6da7d", "garlic presser-120a303847616d27", "chopping board4-fc0e37ac72161ef7", "bag of carrots-9f74fd12b417bdc8", "scissors cover-a6205ce6ec86f841", "scissors-b7500debea02c36b", "peeler-ce2dccad611eec54", "plastic cover-e8516b1355630251", "plastic serving spoon-b763996485190f8c", "chopping board3-f3c34af39fab0708", "clove of garlic-86a50ba9146f88ef", "glass lid2-1a1c0412fbdc8f58", "glass lid-c142bd4e12156f19", "wooden spoon-daa08e1f95bbad19", "pan2-ccf19cacfc2943b6", "saucepan2-3c6a47036d8474c2", "packet of minced meat2-18c5e3e040333bea", "packet of minced meat-ac322553d2441535", "salt bottle-36852988b45e6bd3", "chopping board2-6932d696acd5032a", "knife2-ae61b780aa68a4a0", "chopping board-2e3e4f97b1242861", "knife-5a16c68c09537f9f", "onion2-a9afa5b7b76cd1b7", "onion-224e797dfe26cc13", "dirty glass-f0b6b06a86bb7d64", "mesh-e95446965746807f", "pan-4948d2e5651ada57", "saucepan-fbee1e79b449d5e0", "small notebook-b1e44e5112c3b86e", "white towel-c54866de0cab1d79", "blue towel-e98a9f7a908fd99e", "large bowl2-5f736a42eb1057e1", "bottle of oil cap-88edc5df14eaa961", "plastic bag-df3e5dbb5d0f3acf", "bag of flour-b148433d9dfdad75", "water filter jug-ee749c4996f36500", "bowl-5951775fa8b8bace", "spoon-decaa21c7b1c70d5", "can of yeast-4e4be53413d4f697", "bowl of flour-23decab0fbb85e18", "bottle of olive oil-08ebf6ae6cefc83a", "bottle of balsemic vinegar-d463235fcbcd8004", "bottle of vegetable oil-9bf7fc9adb0a25e6", "plate-0ea1342f1b55fbb9", "new onion half-2e97349d356bac9a" ], "reasoning": "1. First, I found how many carrots were peeled; I saw the user pick up the peeler (peeler-ce2dccad611eec54) at 22:39. \n2. At 13:58, I saw a bag of carrots (bag of carrots-9f74fd12b417bdc8) on the counter. I continued watching and noted the times that each individual carrot was peeled. \n3. I saw the user peel the first carrot (carrot1-975ab66b998500ec) from 22:47-22:51 and 23:58-24:18. \n4. I saw the user peel the second carrot (carrot2-532aa17893a6e110) from 24:20-24:34. \n5. I saw the user peel the third carrot (carrot3-c3a43afebd820a86) from 24:35-24:52. \n6. I saw the user peel the fourth carrot (carrot4-42df7816f7f589c5) from 24:55-15:12. \n7. I saw the user peel the fifth carrot (carrot5-c55e1a891787c67a) from 25:19-25:38. \n8. I saw the user peel the sixth carrot (carrot6-975e4a3804a0ba76) from 25:44-26:07. \n9. I saw the user peel the seventh carrot (carrot7-2b7a43a332504a51) from 26:12-26:37. \n10. I saw the user peel the eighth carrot (carrot8-7623c13c407a327b) from 27:41-28:01. \n11. I saw the user peel the ninth carrot (carrot9-786e7f1c6a225c71) from 28:06-28:21.\n12. I saw the user peel the tenth carrot (carrot10-0aae52eddad55523) from 28:26-28:49. \n13. I did not see the user peel any more carrots, therefore I found that they peeled a total of 10 carrots. \n14. I watched the video and looked for when towels were put into the oven. From 07:09-07:19, I saw the user pick up the large bowl (large bowl2-5f736a42eb1057e1) covered in a white towel (white towel-c54866de0cab1d79), put it in the oven, and close the oven door. \n15. I watched the video to see how many towels were on the large bowl (large bowl2-5f736a42eb1057e1) before 07:09. From 06:52-06:53 I saw the user put a blue towel (blue towel-e98a9f7a908fd99e) over a large bowl (large bowl2-5f736a42eb1057e1). \n16. From 07:07-07:08 I saw the user put a white towel (white towel-c54866de0cab1d79) over the blue towel (blue towel-e98a9f7a908fd99e). \n17. Therefore, I counted that there were 2 towels that went into the oven. \n18. I multiplied 10*2=20.", "referenced_object_masks": [ "white towel-c54866de0cab1d79", "carrot1-975ab66b998500ec", "carrot9-786e7f1c6a225c71", "bag of carrots-9f74fd12b417bdc8", "peeler-ce2dccad611eec54", "carrot6-975e4a3804a0ba76", "blue towel-e98a9f7a908fd99e", "carrot5-c55e1a891787c67a", "carrot8-7623c13c407a327b", "carrot10-0aae52eddad55523", "large bowl2-5f736a42eb1057e1", "carrot7-2b7a43a332504a51", "carrot2-532aa17893a6e110", "carrot4-42df7816f7f589c5", "carrot3-c3a43afebd820a86" ] }, { "key": "P01-20240203-123350:c1bfac8ac1878d45cca2de4d5520f2dc0c3b29e7", "video_id": "P01-20240203-123350", "question": "How many cuts do I make to the first onion before I lay it in half on the cutting board?", "answer_choice_0": "6", "answer_choice_1": "3", "answer_choice_2": "4", "answer_choice_3": "2", "answer_choice_4": "1", "answer_id": 2, "answer": "4", "question_type": [ "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counting" ], "objects_of_interest": [ "carrot10-0aae52eddad55523", "carrot9-786e7f1c6a225c71", "carrot8-7623c13c407a327b", "carrot7-2b7a43a332504a51", "carrot6-975e4a3804a0ba76", "carrot5-c55e1a891787c67a", "carrot4-42df7816f7f589c5", "carrot3-c3a43afebd820a86", "carrot2-532aa17893a6e110", "plastic cover of peeler-6075adba6cba9311", "plastic tupperware-dea9178b19800ade", "carrot1-975ab66b998500ec", "mesh of oranges-fb26a0288c9637ea", "spoon2-8558830aa3073276", "plastic package of salad-17d1f6ca0909bf5b", "part of cover2-fbeb40cafec4369b", "chicken stock cube2-13a57f8bebccfb63", "part of cover-8adb9bf79df13865", "chicken stock cube-39ee8c4bc6e4e1cf", "box of chicken stock-1b6260f9d91edf71", "kettle-5b399fca36e15db5", "mug-1c8b74e0e887c6d7", "cover of plastic package-d7dc32e7d938b2aa", "paper-1577fe9bf3a6da7d", "garlic presser-120a303847616d27", "chopping board4-fc0e37ac72161ef7", "bag of carrots-9f74fd12b417bdc8", "scissors cover-a6205ce6ec86f841", "scissors-b7500debea02c36b", "peeler-ce2dccad611eec54", "plastic cover-e8516b1355630251", "plastic serving spoon-b763996485190f8c", "chopping board3-f3c34af39fab0708", "clove of garlic-86a50ba9146f88ef", "glass lid2-1a1c0412fbdc8f58", "glass lid-c142bd4e12156f19", "wooden spoon-daa08e1f95bbad19", "pan2-ccf19cacfc2943b6", "saucepan2-3c6a47036d8474c2", "packet of minced meat2-18c5e3e040333bea", "packet of minced meat-ac322553d2441535", "salt bottle-36852988b45e6bd3", "chopping board2-6932d696acd5032a", "knife2-ae61b780aa68a4a0", "chopping board-2e3e4f97b1242861", "knife-5a16c68c09537f9f", "onion2-a9afa5b7b76cd1b7", "onion-224e797dfe26cc13", "dirty glass-f0b6b06a86bb7d64", "mesh-e95446965746807f", "pan-4948d2e5651ada57", "saucepan-fbee1e79b449d5e0", "small notebook-b1e44e5112c3b86e", "white towel-c54866de0cab1d79", "blue towel-e98a9f7a908fd99e", "large bowl2-5f736a42eb1057e1", "bottle of oil cap-88edc5df14eaa961", "plastic bag-df3e5dbb5d0f3acf", "bag of flour-b148433d9dfdad75", "water filter jug-ee749c4996f36500", "bowl-5951775fa8b8bace", "spoon-decaa21c7b1c70d5", "can of yeast-4e4be53413d4f697", "bowl of flour-23decab0fbb85e18", "bottle of olive oil-08ebf6ae6cefc83a", "bottle of balsemic vinegar-d463235fcbcd8004", "bottle of vegetable oil-9bf7fc9adb0a25e6", "plate-0ea1342f1b55fbb9", "new onion half-2e97349d356bac9a" ], "reasoning": "1. I identify both onions (onion-224e797dfe26cc13) in half on the cutting board (chopping board-2e3e4f97b1242861) at 08:08.\n2. I rewind until the user makes the first cut to the first onion (onion-224e797dfe26cc13) with the knife (knife-5a16c68c09537f9f) at 07:56. The user makes a second cut at 07:58, a third cut at 08:01, and pulls the knife out again at 08:04 to make the fourth cut. \n3. At 08:08, the user splits the first onion (onion-224e797dfe26cc13) in half. \n4. Therefore, the user made 4 cuts before splitting the first onion (onion-224e797dfe26cc13) in half and laying it splayed on the cutting board (chopping board-2e3e4f97b1242861)", "referenced_object_masks": [ "knife-5a16c68c09537f9f", "onion-224e797dfe26cc13", "chopping board-2e3e4f97b1242861" ] }, { "key": "P01-20240203-123350:c8a961dde78740da53d5c30de9adb3b6d516e19b", "video_id": "P01-20240203-123350", "question": "After I place the bottle of olive oil down on the counter for the first time, what two objects are directly behind the bottle of vegetable oil on the shelf to my left?", "answer_choice_0": "Carrots and balsamic vinegar", "answer_choice_1": "Balsamic vinegar and blue towel", "answer_choice_2": "Balsamic vinegar and blue cylinder", "answer_choice_3": "Blue cylinder and blue towel", "answer_choice_4": "Bag of flour and blue cylinder", "answer_id": 2, "answer": "Balsamic vinegar and blue cylinder", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counting" ], "objects_of_interest": [ "carrot10-0aae52eddad55523", "carrot9-786e7f1c6a225c71", "carrot8-7623c13c407a327b", "carrot7-2b7a43a332504a51", "carrot6-975e4a3804a0ba76", "carrot5-c55e1a891787c67a", "carrot4-42df7816f7f589c5", "carrot3-c3a43afebd820a86", "carrot2-532aa17893a6e110", "plastic cover of peeler-6075adba6cba9311", "plastic tupperware-dea9178b19800ade", "carrot1-975ab66b998500ec", "mesh of oranges-fb26a0288c9637ea", "spoon2-8558830aa3073276", "plastic package of salad-17d1f6ca0909bf5b", "part of cover2-fbeb40cafec4369b", "chicken stock cube2-13a57f8bebccfb63", "part of cover-8adb9bf79df13865", "chicken stock cube-39ee8c4bc6e4e1cf", "box of chicken stock-1b6260f9d91edf71", "kettle-5b399fca36e15db5", "mug-1c8b74e0e887c6d7", "cover of plastic package-d7dc32e7d938b2aa", "paper-1577fe9bf3a6da7d", "garlic presser-120a303847616d27", "chopping board4-fc0e37ac72161ef7", "bag of carrots-9f74fd12b417bdc8", "scissors cover-a6205ce6ec86f841", "scissors-b7500debea02c36b", "peeler-ce2dccad611eec54", "plastic cover-e8516b1355630251", "plastic serving spoon-b763996485190f8c", "chopping board3-f3c34af39fab0708", "clove of garlic-86a50ba9146f88ef", "glass lid2-1a1c0412fbdc8f58", "glass lid-c142bd4e12156f19", "wooden spoon-daa08e1f95bbad19", "pan2-ccf19cacfc2943b6", "saucepan2-3c6a47036d8474c2", "packet of minced meat2-18c5e3e040333bea", "packet of minced meat-ac322553d2441535", "salt bottle-36852988b45e6bd3", "chopping board2-6932d696acd5032a", "knife2-ae61b780aa68a4a0", "chopping board-2e3e4f97b1242861", "knife-5a16c68c09537f9f", "onion2-a9afa5b7b76cd1b7", "onion-224e797dfe26cc13", "dirty glass-f0b6b06a86bb7d64", "mesh-e95446965746807f", "pan-4948d2e5651ada57", "saucepan-fbee1e79b449d5e0", "small notebook-b1e44e5112c3b86e", "white towel-c54866de0cab1d79", "blue towel-e98a9f7a908fd99e", "large bowl2-5f736a42eb1057e1", "bottle of oil cap-88edc5df14eaa961", "plastic bag-df3e5dbb5d0f3acf", "bag of flour-b148433d9dfdad75", "water filter jug-ee749c4996f36500", "bowl-5951775fa8b8bace", "spoon-decaa21c7b1c70d5", "can of yeast-4e4be53413d4f697", "bowl of flour-23decab0fbb85e18", "bottle of olive oil-08ebf6ae6cefc83a", "bottle of balsemic vinegar-d463235fcbcd8004", "bottle of vegetable oil-9bf7fc9adb0a25e6", "plate-0ea1342f1b55fbb9", "new onion half-2e97349d356bac9a" ], "reasoning": "1. I identify the olive oil (bottle of olive oil-08ebf6ae6cefc83a) on the shelf in front of the user at 00:15.\n2. I observe the user place the olive oil (bottle of olive oil-08ebf6ae6cefc83a) on the counter to the right of the bowl of flour (bowl of flour-23decab0fbb85e18) at 00:20.\n3. I rewind to observe the user's interaction with the shelf to their left from 00:12 to 00:17.\n4. From 00:13 to 00:14, I observe the user move the balsamic vinegar (bottle of balsemic vinegar-d463235fcbcd8004) to the left directly in front of the blue cylinder .\n5. From 00:15 to 00:17, I observe the user place the vegetable oil (bottle of vegetable oil-9bf7fc9adb0a25e6) in front of the vinegar (bottle of balsemic vinegar-d463235fcbcd8004).\n6. I determine that if the user looked to the shelf on their left when they first placed the olive oil (bottle of olive oil-08ebf6ae6cefc83a), both the balsamic vinegar (bottle of balsemic vinegar-d463235fcbcd8004) and the blue cylinder would be directly behind the vegetable oil (bottle of vegetable oil-9bf7fc9adb0a25e6).", "referenced_object_masks": [ "bottle of balsemic vinegar-d463235fcbcd8004", "bottle of olive oil-08ebf6ae6cefc83a", "bottle of vegetable oil-9bf7fc9adb0a25e6", "bowl of flour-23decab0fbb85e18" ] }, { "key": "P01-20240203-123350:fe29e5f8bd0cec9c32d9ba433c71cb3fc7f492bd", "video_id": "P01-20240203-123350", "question": "Relative to the towel I used to dry a large bowl, where did I find the cutting board used for cutting peeled onions?", "answer_choice_0": "Drawer to the left", "answer_choice_1": "Counter opposite", "answer_choice_2": "Cabinet above", "answer_choice_3": "Counter to the right", "answer_choice_4": "Drawer below", "answer_id": 0, "answer": "Drawer to the left", "question_type": [ "Egocentric & Spatial Perception", "Action Recognition", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "carrot10-0aae52eddad55523", "carrot9-786e7f1c6a225c71", "carrot8-7623c13c407a327b", "carrot7-2b7a43a332504a51", "carrot6-975e4a3804a0ba76", "carrot5-c55e1a891787c67a", "carrot4-42df7816f7f589c5", "carrot3-c3a43afebd820a86", "carrot2-532aa17893a6e110", "plastic cover of peeler-6075adba6cba9311", "plastic tupperware-dea9178b19800ade", "carrot1-975ab66b998500ec", "mesh of oranges-fb26a0288c9637ea", "spoon2-8558830aa3073276", "plastic package of salad-17d1f6ca0909bf5b", "part of cover2-fbeb40cafec4369b", "chicken stock cube2-13a57f8bebccfb63", "part of cover-8adb9bf79df13865", "chicken stock cube-39ee8c4bc6e4e1cf", "box of chicken stock-1b6260f9d91edf71", "kettle-5b399fca36e15db5", "mug-1c8b74e0e887c6d7", "cover of plastic package-d7dc32e7d938b2aa", "paper-1577fe9bf3a6da7d", "garlic presser-120a303847616d27", "chopping board4-fc0e37ac72161ef7", "bag of carrots-9f74fd12b417bdc8", "scissors cover-a6205ce6ec86f841", "scissors-b7500debea02c36b", "peeler-ce2dccad611eec54", "plastic cover-e8516b1355630251", "plastic serving spoon-b763996485190f8c", "chopping board3-f3c34af39fab0708", "clove of garlic-86a50ba9146f88ef", "glass lid2-1a1c0412fbdc8f58", "glass lid-c142bd4e12156f19", "wooden spoon-daa08e1f95bbad19", "pan2-ccf19cacfc2943b6", "saucepan2-3c6a47036d8474c2", "packet of minced meat2-18c5e3e040333bea", "packet of minced meat-ac322553d2441535", "salt bottle-36852988b45e6bd3", "chopping board2-6932d696acd5032a", "knife2-ae61b780aa68a4a0", "chopping board-2e3e4f97b1242861", "knife-5a16c68c09537f9f", "onion2-a9afa5b7b76cd1b7", "onion-224e797dfe26cc13", "dirty glass-f0b6b06a86bb7d64", "mesh-e95446965746807f", "pan-4948d2e5651ada57", "saucepan-fbee1e79b449d5e0", "small notebook-b1e44e5112c3b86e", "white towel-c54866de0cab1d79", "blue towel-e98a9f7a908fd99e", "large bowl2-5f736a42eb1057e1", "bottle of oil cap-88edc5df14eaa961", "plastic bag-df3e5dbb5d0f3acf", "bag of flour-b148433d9dfdad75", "water filter jug-ee749c4996f36500", "bowl-5951775fa8b8bace", "spoon-decaa21c7b1c70d5", "can of yeast-4e4be53413d4f697", "bowl of flour-23decab0fbb85e18", "bottle of olive oil-08ebf6ae6cefc83a", "bottle of balsemic vinegar-d463235fcbcd8004", "bottle of vegetable oil-9bf7fc9adb0a25e6", "plate-0ea1342f1b55fbb9", "new onion half-2e97349d356bac9a" ], "reasoning": "1. I watched the video and saw the user rinse a large bowl (large bowl2-5f736a42eb1057e1) from 05:31-05:34. \n2. I saw the user dry the large bowl (large bowl2-5f736a42eb1057e1) with a towel (white towel-c54866de0cab1d79) from 05:37-05:39, and noted that the towel (white towel-c54866de0cab1d79) was hanging on the oven door . \n3. I found the user slicing a peeled onion (new onion half-2e97349d356bac9a) on a cutting board (chopping board2-6932d696acd5032a) beginning at 09:41. \n4. I went back and found the first time that the cutting board (chopping board2-6932d696acd5032a) was shown and found it in a drawer at 08:27. I noted that the drawer with the cutting board (chopping board2-6932d696acd5032a) was to the left of the oven door .\n5. Therefore, relative to the towel (white towel-c54866de0cab1d79) the user used to dry a large bowl (large bowl2-5f736a42eb1057e1), the cutting board (chopping board2-6932d696acd5032a) is located in the drawer to the left.", "referenced_object_masks": [ "white towel-c54866de0cab1d79", "new onion half-2e97349d356bac9a", "chopping board2-6932d696acd5032a", "large bowl2-5f736a42eb1057e1" ] }, { "key": "P01-20240203-130505:34f79c489c79d892d27abda5b7a592ee3dbfb09b", "video_id": "P01-20240203-130505", "question": "What is the sum total of the number of times I pick something up off the floor and the number of times I stop to write something down before adding the carrot to the pot with the butter?", "answer_choice_0": ", 2", "answer_choice_1": "6", "answer_choice_2": "7", "answer_choice_3": "5", "answer_choice_4": "3", "answer_id": 1, "answer": "6", "question_type": [ "Action Recognition", "Counting", "Object Recognition" ], "objects_of_interest": [ "piece of food trash-cace8c61882e519f", "heat diffuser-fb83bcba1cf521cf", "one piece carrot-b66af41184d5fc79", "carrot10-8ab3e5095f3db85b", "cut pieces-75346e01f8a00393", "carrot9-a4b952ba505a6063", "remaining carrot pieces-d99c94c45604b7d9", "third slice-1670dc6bb2f51d1b", "piece carrot-7b4d177b947664e1", "piece2-cb8a8d30faf33e5b", "one piece of carrot-7ae77b8c164897d4", "wide end of carrot-18990f1f17184913", "carrot8-eee06e5e4d9fd9b5", "carrot7-003c1b56f42eb5ac", "thick end of carrot-ec18bea503e4bb24", "carrot6-6b8cac98b6dacc40", "kettle-8103652c62bad9fd", "piece-b8ce871f9215aa14", "one piece-c71f07fca443333f", "ends of carrot-70249129f7ed5d86", "carrot5-d2fd857a4b991683", "wooden spatula-5f6443a50bfdbd1a", "pan2-0a64c5d983b676c4", "carrot ends-e0e14167414d8aea", "other end-158f9f1bff89c8cf", "carrot end-70fcd976b385ebc5", "plate3-f62815714e2bdd6c", "plate2-cef1ca396fd6593c", "pan-c7434ea99ddb699e", "wooden spoon-ccc33c29aa996e72", "lid-63af0e9f411d7011", "jar of spices-1e904f38ff4ac9b8", "empty chopping board-6716d492a199b2d1", "third half of onion-efc90f22dc7bb2b0", "another half onion-da05fda3d6cabc3c", "onion-14005e5f764bb354", "chopping board-f4d8aec5c9d69cc1", "knife2-eeb5e99455195978", "carrot4-69dc4051fefc14ba", "another carrot-ec25a3583ff5f347", "carrot-8e4f15fdb9559f85", "pile of peeled carrots-2e2d5e54b5c80099", "butter slice-bb198deacbf02eb2", "knife-aa73b8da0b34f746", "remaining butter-e276473a0c04ee48", "jar of chocolate-3fd25f21fdafffcb", "butter-68ff481c410340c3", "bottle of vegetable oil-82c75e65f3154079", "peeler-ef9e62530f7609ee", "empty plastic bag-0033b563cdeec2e0", "scale-7fc001161508cf9a", "pot-a4a3fc0088ddd2d2", "plate-09a17ecfafbf94fe", "pen-d5077b84cad60f3e", "paper notepad-a23f777966ac67d7", "new mug-b627a506ba8d6312" ], "reasoning": "1. First I had to identify the pot (pot-a4a3fc0088ddd2d2) with the butter (remaining butter-e276473a0c04ee48) and butter slices (butter slice-bb198deacbf02eb2). I saw the user put butter (remaining butter-e276473a0c04ee48) in a pot (pot-a4a3fc0088ddd2d2) at 01:04 and add additional butter (butter slice-bb198deacbf02eb2) to the same pot (pot-a4a3fc0088ddd2d2) at 01:29. \n2. Then, I watched the user place that pot (pot-a4a3fc0088ddd2d2) on the stove at 06:09-06:20. \n3. Next, I watched the user add a pile of peeled carrots to the same pot (pot-a4a3fc0088ddd2d2) at 10:44-10:53.\n4. Then, I watched the video from the beginning until 10:44, paying attention to when the user picks something up off the floor. I saw the user pickup a discarded piece of cut produce at 03:14 and a second discarded piece of cut produce (carrot end-70fcd976b385ebc5) at 05:38, for a total of 2 times.\n5. Next, I watched the video from the beginning until 10:44, paying attention to the times the user stops to write something down with a pen (pen-d5077b84cad60f3e) on a notepad (paper notepad-a23f777966ac67d7) . I saw the user do this at 00:05-00:18, 01:33-01:38, 05:22-05:24, and 10:18-10:20, for a total of 4 times. \n6. Finally, I added together the number of times the user stopped to write something down (4) to the number of times the user picked something up off the floor (2) before adding the carrot to the pot (pot-a4a3fc0088ddd2d2) with butter (remaining butter-e276473a0c04ee48) and (butter slice-bb198deacbf02eb2) and got a sum of 6.", "referenced_object_masks": [ "pot-a4a3fc0088ddd2d2", "paper notepad-a23f777966ac67d7", "butter slice-bb198deacbf02eb2", "carrot end-70fcd976b385ebc5", "pen-d5077b84cad60f3e", "remaining butter-e276473a0c04ee48" ] }, { "key": "P01-20240203-130505:42328eaba53833c4f1c6c0295bbe681e80435673", "video_id": "P01-20240203-130505", "question": "If I added 2 additional types of orange produce to the orange produce on the counter throughout the video, then added that number to the number of types of orange produce I put in the pot with butter, what would the final number be?", "answer_choice_0": "8", "answer_choice_1": "3", "answer_choice_2": "5", "answer_choice_3": "6", "answer_choice_4": "2", "answer_id": 2, "answer": "5", "question_type": [ "Counterfactual", "Counting", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "piece of food trash-cace8c61882e519f", "heat diffuser-fb83bcba1cf521cf", "one piece carrot-b66af41184d5fc79", "carrot10-8ab3e5095f3db85b", "cut pieces-75346e01f8a00393", "carrot9-a4b952ba505a6063", "remaining carrot pieces-d99c94c45604b7d9", "third slice-1670dc6bb2f51d1b", "piece carrot-7b4d177b947664e1", "piece2-cb8a8d30faf33e5b", "one piece of carrot-7ae77b8c164897d4", "wide end of carrot-18990f1f17184913", "carrot8-eee06e5e4d9fd9b5", "carrot7-003c1b56f42eb5ac", "thick end of carrot-ec18bea503e4bb24", "carrot6-6b8cac98b6dacc40", "kettle-8103652c62bad9fd", "piece-b8ce871f9215aa14", "one piece-c71f07fca443333f", "ends of carrot-70249129f7ed5d86", "carrot5-d2fd857a4b991683", "wooden spatula-5f6443a50bfdbd1a", "pan2-0a64c5d983b676c4", "carrot ends-e0e14167414d8aea", "other end-158f9f1bff89c8cf", "carrot end-70fcd976b385ebc5", "plate3-f62815714e2bdd6c", "plate2-cef1ca396fd6593c", "pan-c7434ea99ddb699e", "wooden spoon-ccc33c29aa996e72", "lid-63af0e9f411d7011", "jar of spices-1e904f38ff4ac9b8", "empty chopping board-6716d492a199b2d1", "third half of onion-efc90f22dc7bb2b0", "another half onion-da05fda3d6cabc3c", "onion-14005e5f764bb354", "chopping board-f4d8aec5c9d69cc1", "knife2-eeb5e99455195978", "carrot4-69dc4051fefc14ba", "another carrot-ec25a3583ff5f347", "carrot-8e4f15fdb9559f85", "pile of peeled carrots-2e2d5e54b5c80099", "butter slice-bb198deacbf02eb2", "knife-aa73b8da0b34f746", "remaining butter-e276473a0c04ee48", "jar of chocolate-3fd25f21fdafffcb", "butter-68ff481c410340c3", "bottle of vegetable oil-82c75e65f3154079", "peeler-ef9e62530f7609ee", "empty plastic bag-0033b563cdeec2e0", "scale-7fc001161508cf9a", "pot-a4a3fc0088ddd2d2", "plate-09a17ecfafbf94fe", "pen-d5077b84cad60f3e", "paper notepad-a23f777966ac67d7", "new mug-b627a506ba8d6312" ], "reasoning": "1. First, I had to determine how many types of orange produce are on the counter throughout the video. I observed the counter from 00:06-00:36 and determined that oranges and carrots (pile of peeled carrots-2e2d5e54b5c80099) are present. I then watched the rest of the video and determined that no other types of orange produce ever appear on the counter.\n2. Next, I had to determine how many types of orange produce the user put into the pot (pot-a4a3fc0088ddd2d2) with butter (butter slice-bb198deacbf02eb2). I identified the pot (pot-a4a3fc0088ddd2d2) with butter (butter slice-bb198deacbf02eb2) at 01:03-01:30 and saw the user put carrots in this pot at 10:44-10:53. I never saw the user add the oranges to the pot (pot-a4a3fc0088ddd2d2), meaning that only 1 type of orange produce is added to the pot with butter. \n3. Finally, I added 2 to the number of types of orange produce on the counter (2) to the number of types of orange produce (1) the user put in the pot (pot-a4a3fc0088ddd2d2) with butter (butter slice-bb198deacbf02eb2) and got a total of 5.", "referenced_object_masks": [ "pot-a4a3fc0088ddd2d2", "pile of peeled carrots-2e2d5e54b5c80099", "butter slice-bb198deacbf02eb2" ] }, { "key": "P01-20240203-130505:5e251c5006e8da8b2a636982e5b01fb12766ac9d", "video_id": "P01-20240203-130505", "question": "When I put the cut carrot on a plate for the first time, in what relative direction is the cupboard I retrieved the plate from?", "answer_choice_0": "2 o 'clock", "answer_choice_1": "9 o'clock", "answer_choice_2": "11 o'clock", "answer_choice_3": "3 o'clock", "answer_choice_4": "7 o'clock", "answer_id": 4, "answer": "7 o'clock", "question_type": [ "Action Recognition", "Object Recognition", "Egocentric & Spatial Perception", "Situational Awareness" ], "objects_of_interest": [ "piece of food trash-cace8c61882e519f", "heat diffuser-fb83bcba1cf521cf", "one piece carrot-b66af41184d5fc79", "carrot10-8ab3e5095f3db85b", "cut pieces-75346e01f8a00393", "carrot9-a4b952ba505a6063", "remaining carrot pieces-d99c94c45604b7d9", "third slice-1670dc6bb2f51d1b", "piece carrot-7b4d177b947664e1", "piece2-cb8a8d30faf33e5b", "one piece of carrot-7ae77b8c164897d4", "wide end of carrot-18990f1f17184913", "carrot8-eee06e5e4d9fd9b5", "carrot7-003c1b56f42eb5ac", "thick end of carrot-ec18bea503e4bb24", "carrot6-6b8cac98b6dacc40", "kettle-8103652c62bad9fd", "piece-b8ce871f9215aa14", "one piece-c71f07fca443333f", "ends of carrot-70249129f7ed5d86", "carrot5-d2fd857a4b991683", "wooden spatula-5f6443a50bfdbd1a", "pan2-0a64c5d983b676c4", "carrot ends-e0e14167414d8aea", "other end-158f9f1bff89c8cf", "carrot end-70fcd976b385ebc5", "plate3-f62815714e2bdd6c", "plate2-cef1ca396fd6593c", "pan-c7434ea99ddb699e", "wooden spoon-ccc33c29aa996e72", "lid-63af0e9f411d7011", "jar of spices-1e904f38ff4ac9b8", "empty chopping board-6716d492a199b2d1", "third half of onion-efc90f22dc7bb2b0", "another half onion-da05fda3d6cabc3c", "onion-14005e5f764bb354", "chopping board-f4d8aec5c9d69cc1", "knife2-eeb5e99455195978", "carrot4-69dc4051fefc14ba", "another carrot-ec25a3583ff5f347", "carrot-8e4f15fdb9559f85", "pile of peeled carrots-2e2d5e54b5c80099", "butter slice-bb198deacbf02eb2", "knife-aa73b8da0b34f746", "remaining butter-e276473a0c04ee48", "jar of chocolate-3fd25f21fdafffcb", "butter-68ff481c410340c3", "bottle of vegetable oil-82c75e65f3154079", "peeler-ef9e62530f7609ee", "empty plastic bag-0033b563cdeec2e0", "scale-7fc001161508cf9a", "pot-a4a3fc0088ddd2d2", "plate-09a17ecfafbf94fe", "pen-d5077b84cad60f3e", "paper notepad-a23f777966ac67d7", "new mug-b627a506ba8d6312" ], "reasoning": "1. First, I found the moment where the user puts cut carrot on a plate (plate2-cef1ca396fd6593c) for the first time at 08:08-08:14, and noticed the user is directly in front of the microwave at that time. \n2. Next, I found the moment where the user retrieves the plate (plate2-cef1ca396fd6593c) from the cupboard at 04:55-04:57. \n3. At 04:57, I saw a clear view of the cupboard and the microwave . I reasoned about the scene and determined that when the user is in front of the microwave , the cupboard is behind the user on the left, in approximately the 7 o'clock direction.", "referenced_object_masks": [ "plate2-cef1ca396fd6593c" ] }, { "key": "P01-20240203-130505:60ac35300d79e5b3a3c8ac919c21b041c004c3d3", "video_id": "P01-20240203-130505", "question": "When I cut the largest carrot, in what relative direction is the onion?", "answer_choice_0": "8 o'clock", "answer_choice_1": "2 o'clock", "answer_choice_2": "10 o'clock", "answer_choice_3": "6 o 'clock", "answer_choice_4": "4 o'clock", "answer_id": 3, "answer": "6 o 'clock", "question_type": [ "Action Recognition", "Object Recognition", "Egocentric & Spatial Perception", "Situational Awareness" ], "objects_of_interest": [ "piece of food trash-cace8c61882e519f", "heat diffuser-fb83bcba1cf521cf", "one piece carrot-b66af41184d5fc79", "carrot10-8ab3e5095f3db85b", "cut pieces-75346e01f8a00393", "carrot9-a4b952ba505a6063", "remaining carrot pieces-d99c94c45604b7d9", "third slice-1670dc6bb2f51d1b", "piece carrot-7b4d177b947664e1", "piece2-cb8a8d30faf33e5b", "one piece of carrot-7ae77b8c164897d4", "wide end of carrot-18990f1f17184913", "carrot8-eee06e5e4d9fd9b5", "carrot7-003c1b56f42eb5ac", "thick end of carrot-ec18bea503e4bb24", "carrot6-6b8cac98b6dacc40", "kettle-8103652c62bad9fd", "piece-b8ce871f9215aa14", "one piece-c71f07fca443333f", "ends of carrot-70249129f7ed5d86", "carrot5-d2fd857a4b991683", "wooden spatula-5f6443a50bfdbd1a", "pan2-0a64c5d983b676c4", "carrot ends-e0e14167414d8aea", "other end-158f9f1bff89c8cf", "carrot end-70fcd976b385ebc5", "plate3-f62815714e2bdd6c", "plate2-cef1ca396fd6593c", "pan-c7434ea99ddb699e", "wooden spoon-ccc33c29aa996e72", "lid-63af0e9f411d7011", "jar of spices-1e904f38ff4ac9b8", "empty chopping board-6716d492a199b2d1", "third half of onion-efc90f22dc7bb2b0", "another half onion-da05fda3d6cabc3c", "onion-14005e5f764bb354", "chopping board-f4d8aec5c9d69cc1", "knife2-eeb5e99455195978", "carrot4-69dc4051fefc14ba", "another carrot-ec25a3583ff5f347", "carrot-8e4f15fdb9559f85", "pile of peeled carrots-2e2d5e54b5c80099", "butter slice-bb198deacbf02eb2", "knife-aa73b8da0b34f746", "remaining butter-e276473a0c04ee48", "jar of chocolate-3fd25f21fdafffcb", "butter-68ff481c410340c3", "bottle of vegetable oil-82c75e65f3154079", "peeler-ef9e62530f7609ee", "empty plastic bag-0033b563cdeec2e0", "scale-7fc001161508cf9a", "pot-a4a3fc0088ddd2d2", "plate-09a17ecfafbf94fe", "pen-d5077b84cad60f3e", "paper notepad-a23f777966ac67d7", "new mug-b627a506ba8d6312" ], "reasoning": "1. First, I had to identify the largest carrot (carrot8-eee06e5e4d9fd9b5). I found a clear view of all the carrots (pile of peeled carrots-2e2d5e54b5c80099) that the user cuts at 01:38 and noticed that one carrot (carrot8-eee06e5e4d9fd9b5) is larger than the rest. \n2. Next, I had to find when the user cuts this carrot (carrot8-eee06e5e4d9fd9b5) which I located at 08:41-09:24. \n3. Then, I had to locate the onion at 08:41-09:24. and found a clear shot of onion in a pot (pot-a4a3fc0088ddd2d2) on the stove at 08:59-09:09. \n4. I saw the user turn from the stove to the carrot (carrot8-eee06e5e4d9fd9b5) at 09:11, and observed that the carrot (carrot8-eee06e5e4d9fd9b5) is directly opposite the stove. \n5. Reasoning about the scene, I determined that when the user cuts the largest carrot (carrot8-eee06e5e4d9fd9b5), the onion is in approximately the 6 o'clock direction.", "referenced_object_masks": [ "pot-a4a3fc0088ddd2d2", "carrot8-eee06e5e4d9fd9b5", "pile of peeled carrots-2e2d5e54b5c80099" ] }, { "key": "P01-20240203-130505:85e8892cbff091dd2eb0126df24e90afe1deae1e", "video_id": "P01-20240203-130505", "question": "When I season the meat, in what relative direction are the onion skins?", "answer_choice_0": "11 o 'clock", "answer_choice_1": "5 o'clock", "answer_choice_2": "6 o'clock", "answer_choice_3": "9 o'clock", "answer_choice_4": "2 o'clock", "answer_id": 1, "answer": "5 o'clock", "question_type": [ "Action Recognition", "Object Recognition", "Egocentric & Spatial Perception", "Situational Awareness" ], "objects_of_interest": [ "piece of food trash-cace8c61882e519f", "heat diffuser-fb83bcba1cf521cf", "one piece carrot-b66af41184d5fc79", "carrot10-8ab3e5095f3db85b", "cut pieces-75346e01f8a00393", "carrot9-a4b952ba505a6063", "remaining carrot pieces-d99c94c45604b7d9", "third slice-1670dc6bb2f51d1b", "piece carrot-7b4d177b947664e1", "piece2-cb8a8d30faf33e5b", "one piece of carrot-7ae77b8c164897d4", "wide end of carrot-18990f1f17184913", "carrot8-eee06e5e4d9fd9b5", "carrot7-003c1b56f42eb5ac", "thick end of carrot-ec18bea503e4bb24", "carrot6-6b8cac98b6dacc40", "kettle-8103652c62bad9fd", "piece-b8ce871f9215aa14", "one piece-c71f07fca443333f", "ends of carrot-70249129f7ed5d86", "carrot5-d2fd857a4b991683", "wooden spatula-5f6443a50bfdbd1a", "pan2-0a64c5d983b676c4", "carrot ends-e0e14167414d8aea", "other end-158f9f1bff89c8cf", "carrot end-70fcd976b385ebc5", "plate3-f62815714e2bdd6c", "plate2-cef1ca396fd6593c", "pan-c7434ea99ddb699e", "wooden spoon-ccc33c29aa996e72", "lid-63af0e9f411d7011", "jar of spices-1e904f38ff4ac9b8", "empty chopping board-6716d492a199b2d1", "third half of onion-efc90f22dc7bb2b0", "another half onion-da05fda3d6cabc3c", "onion-14005e5f764bb354", "chopping board-f4d8aec5c9d69cc1", "knife2-eeb5e99455195978", "carrot4-69dc4051fefc14ba", "another carrot-ec25a3583ff5f347", "carrot-8e4f15fdb9559f85", "pile of peeled carrots-2e2d5e54b5c80099", "butter slice-bb198deacbf02eb2", "knife-aa73b8da0b34f746", "remaining butter-e276473a0c04ee48", "jar of chocolate-3fd25f21fdafffcb", "butter-68ff481c410340c3", "bottle of vegetable oil-82c75e65f3154079", "peeler-ef9e62530f7609ee", "empty plastic bag-0033b563cdeec2e0", "scale-7fc001161508cf9a", "pot-a4a3fc0088ddd2d2", "plate-09a17ecfafbf94fe", "pen-d5077b84cad60f3e", "paper notepad-a23f777966ac67d7", "new mug-b627a506ba8d6312" ], "reasoning": "1. First, I found the moment where the user seasoned the meat in a pan (pan-c7434ea99ddb699e) with spices from a jar (jar of spices-1e904f38ff4ac9b8) at 03:51-03:59, and noticed that the user was in front of the stove at the center of the long counter against the wall. \n2. Next, I had to go back in time to locate the onion skins . I saw the user throw the onion skins in a trash can at 03:20-03:28, and noticed that the trash can is located on the far left side of the counter opposite the stove. \n3. Reasoning about the scene, I determined that when the user is in front of the stove, the onion skins are in the trash can behind the user on the right, in approximately the 5 o'clock direction.", "referenced_object_masks": [ "pan-c7434ea99ddb699e", "jar of spices-1e904f38ff4ac9b8" ] }, { "key": "P01-20240203-132119:022cfcb57ac57ec9545f7435adf717d3b4338db7", "video_id": "P01-20240203-132119", "question": "How many drinking vessels did I load into the dishwasher?", "answer_choice_0": "1", "answer_choice_1": "3", "answer_choice_2": "2", "answer_choice_3": "5", "answer_choice_4": "4", "answer_id": 2, "answer": "2", "question_type": [ "Counting", "Event Occurence", "Egocentric & Spatial Perception", "Counterfactual", "Object Recognition" ], "objects_of_interest": [ "one piece of biscuit-b5310b516fe1376c", "biscuits3-095cea4424851bb4", "second tube of biscuits-3bb6fd99801fcf24", "glass2-99b48efeedbd26fd", "biscuits2-d8bd5e6bb0674e8c", "biscuits-c900ae96394784a9", "one plastic tube of biscuits-a7036423663732ea", "food processor-e92c427f58c23bf4", "plug of food processor-b6cfc7fb29d2dc74", "towel-59e3239324500237", "centre plastic piece-ed2e5901533cc90f", "box of hand blender-176efa428d3287c7", "plastic piece3-4e3381491fff34a2", "meter inside-55baba28fc0e214b", "thin box-3e3a84fc654849a2", "food processor bowl-1604724a8b814fab", "plastic package2-9306496575fb0acc", "big plastic disk-31e4c540bda6b8cd", "plastic package-5e6c2ce7a37e07d5", "plastic piece2-d6ac269f6cca7a31", "cover of food processer bowl-63fafae74916e4b2", "blade cover2-c72a0652ad013a47", "blade cover1-16c5bb7df8a3b00f", "blade-fbb2475c3352e24a", "plastic piece-2f846fa74cbb90da", "packet of biscuits-e31326679f5a2aaa", "cocoa can1-b0b52dab15606165", "plastic bag-afa37c9a0cc43ec1", "cocoa can2-7562d02d21816f46", "can of condensed milk-a010ead84a02d07c", "butter package2-76405e0a9312d4a5", "butter's package-815655356e7b48a2", "sink hole plug-493a6ca0dc2bf4e8", "second chopping board-a4aac48cfe406ea6", "knife2-3a6e4e7a7ce9e4cb", "chopping board-23a815929202d3f4", "plate2-955b191e808136af", "plate-f91c00bbec7fc1e7", "thermal flask lid-5aae373ee1cd43e6", "mug2-de9142a7caff8a84", "glass-10b22113774eda67", "bowl3-3e3ada444f14a322", "bowl2-b8c4284c204e7938", "spoon2-ea8166aa48f267f5", "knife-f9a0ea882180239f", "spoon-fa8b332ffb422b60", "tupperware-9df836d37764b630", "bowl-e8e38966bc68fb94", "washing up liquids bottle-7e6ffa234d9992bd", "sponge-56a7e30a6aab9f87", "left glove-46c3c38b90003ec3", "right glove-e7ce0cee33ed285f", "thermal flask-8c868c36d1371275", "pan-3b2704d1615e19cc", "strainer-6e6b5c8110f81a55", "ladle-2129c22893ef7a4f", "stack of pots-d92bec3023cfb118", "saucepan-ee52592a1e022b52", "bag of flour-51668af48d1a7842", "lid2-011b4b830a9ed325", "lid-122b8c9de3ba9422", "tupperware lid-411ae9423ab398ff", "carrot piece-3ca360d7506fd255", "empty mug-f9f5c658de80a76c", "pen-4834a5a5778ef81c", "stock cube-d88474d879d24082", "one cube of stock-b3814d3e40f4c501", "box of stock cubes-d5b1a639aef5b172", "kettle-7b4238cb675c8cc9", "spatula-039f7e7efd96f484", "lid3-3db698d3ee6c0541" ], "reasoning": "1. I watched the video to identify when the user begins loading the dishwasher so I could count the drinking vessels. I found that they started doing so at 05:55 when they pulled out a silverware tray in the dishwasher .\n2. I continued watching and noticed the user placed a spoon (spoon-fa8b332ffb422b60), a knife (knife-f9a0ea882180239f) and a second spoon (spoon2-ea8166aa48f267f5) into the dishwasher from 05:55-06:00. Then, they placed a bowl (bowl2-b8c4284c204e7938) in it at 06:10, a second bowl (bowl3-3e3ada444f14a322) in it at 06:18, and a glass (glass-10b22113774eda67) in it at 06:32. I identified this glass (glass-10b22113774eda67) as the first drinking vessel. This was followed by a plate (plate-f91c00bbec7fc1e7) at 06:59, a second plate (plate2-955b191e808136af) at 07:08, a cutting board (chopping board-23a815929202d3f4) at 07:14, and a second cutting board (second chopping board-a4aac48cfe406ea6) at 07:18. The last item placed into the dishwasher was a mug (empty mug-f9f5c658de80a76c) at 07:38, and I counted this as the second drinking vessel.\n3. I watched the rest of the video and did not see the user place any other objects into the dishwasher .\n4. Counting up the drinking vessels gave me a total of 2.", "referenced_object_masks": [ "knife-f9a0ea882180239f", "bowl3-3e3ada444f14a322", "spoon2-ea8166aa48f267f5", "plate2-955b191e808136af", "bowl2-b8c4284c204e7938", "chopping board-23a815929202d3f4", "glass-10b22113774eda67", "empty mug-f9f5c658de80a76c", "second chopping board-a4aac48cfe406ea6", "plate-f91c00bbec7fc1e7", "spoon-fa8b332ffb422b60" ] }, { "key": "P01-20240203-132119:19b70333a8de30b3ef7d2dca7c9a5dae224ec327", "video_id": "P01-20240203-132119", "question": "How many times do I push the dirty water around in the sink before I cause the sink to drain?", "answer_choice_0": "8 times", "answer_choice_1": "4 times", "answer_choice_2": "3 times", "answer_choice_3": "6 times", "answer_choice_4": "10 times", "answer_id": 3, "answer": "6 times", "question_type": [ "Counting", "Action Recognition", "Temporal Reasoning", "Situational Awareness" ], "objects_of_interest": [ "one piece of biscuit-b5310b516fe1376c", "biscuits3-095cea4424851bb4", "second tube of biscuits-3bb6fd99801fcf24", "glass2-99b48efeedbd26fd", "biscuits2-d8bd5e6bb0674e8c", "biscuits-c900ae96394784a9", "one plastic tube of biscuits-a7036423663732ea", "food processor-e92c427f58c23bf4", "plug of food processor-b6cfc7fb29d2dc74", "towel-59e3239324500237", "centre plastic piece-ed2e5901533cc90f", "box of hand blender-176efa428d3287c7", "plastic piece3-4e3381491fff34a2", "meter inside-55baba28fc0e214b", "thin box-3e3a84fc654849a2", "food processor bowl-1604724a8b814fab", "plastic package2-9306496575fb0acc", "big plastic disk-31e4c540bda6b8cd", "plastic package-5e6c2ce7a37e07d5", "plastic piece2-d6ac269f6cca7a31", "cover of food processer bowl-63fafae74916e4b2", "blade cover2-c72a0652ad013a47", "blade cover1-16c5bb7df8a3b00f", "blade-fbb2475c3352e24a", "plastic piece-2f846fa74cbb90da", "packet of biscuits-e31326679f5a2aaa", "cocoa can1-b0b52dab15606165", "plastic bag-afa37c9a0cc43ec1", "cocoa can2-7562d02d21816f46", "can of condensed milk-a010ead84a02d07c", "butter package2-76405e0a9312d4a5", "butter's package-815655356e7b48a2", "sink hole plug-493a6ca0dc2bf4e8", "second chopping board-a4aac48cfe406ea6", "knife2-3a6e4e7a7ce9e4cb", "chopping board-23a815929202d3f4", "plate2-955b191e808136af", "plate-f91c00bbec7fc1e7", "thermal flask lid-5aae373ee1cd43e6", "mug2-de9142a7caff8a84", "glass-10b22113774eda67", "bowl3-3e3ada444f14a322", "bowl2-b8c4284c204e7938", "spoon2-ea8166aa48f267f5", "knife-f9a0ea882180239f", "spoon-fa8b332ffb422b60", "tupperware-9df836d37764b630", "bowl-e8e38966bc68fb94", "washing up liquids bottle-7e6ffa234d9992bd", "sponge-56a7e30a6aab9f87", "left glove-46c3c38b90003ec3", "right glove-e7ce0cee33ed285f", "thermal flask-8c868c36d1371275", "pan-3b2704d1615e19cc", "strainer-6e6b5c8110f81a55", "ladle-2129c22893ef7a4f", "stack of pots-d92bec3023cfb118", "saucepan-ee52592a1e022b52", "bag of flour-51668af48d1a7842", "lid2-011b4b830a9ed325", "lid-122b8c9de3ba9422", "tupperware lid-411ae9423ab398ff", "carrot piece-3ca360d7506fd255", "empty mug-f9f5c658de80a76c", "pen-4834a5a5778ef81c", "stock cube-d88474d879d24082", "one cube of stock-b3814d3e40f4c501", "box of stock cubes-d5b1a639aef5b172", "kettle-7b4238cb675c8cc9", "spatula-039f7e7efd96f484", "lid3-3db698d3ee6c0541" ], "reasoning": "1. I look for the time the user pushes dirty water around in the sink. I notice the user do this from 07:46-07:52, at which point the user unclogs the drain by manipulating the plug (sink hole plug-493a6ca0dc2bf4e8).\n2. I count the number of times the user pushes water around from 07:46-07:52, and I come to a total of 6 times. \n3. Therefore, the answer is 6 times.", "referenced_object_masks": [ "sink hole plug-493a6ca0dc2bf4e8" ] }, { "key": "P01-20240203-132119:8eaf9664be2679dcfe5a834c73c81c3af03ac87a", "video_id": "P01-20240203-132119", "question": "What does the sixth thing put into the dishwasher have in common with the first thing I try to pour my biscuit crumbs into?", "answer_choice_0": "They are both glass", "answer_choice_1": "They are both plates", "answer_choice_2": "They are both white", "answer_choice_3": "They are both bowls", "answer_choice_4": "They are both metal", "answer_id": 0, "answer": "They are both glass", "question_type": [ "Counting", "Object Recognition", "Situational Awareness", "Action Recognition" ], "objects_of_interest": [ "one piece of biscuit-b5310b516fe1376c", "biscuits3-095cea4424851bb4", "second tube of biscuits-3bb6fd99801fcf24", "glass2-99b48efeedbd26fd", "biscuits2-d8bd5e6bb0674e8c", "biscuits-c900ae96394784a9", "one plastic tube of biscuits-a7036423663732ea", "food processor-e92c427f58c23bf4", "plug of food processor-b6cfc7fb29d2dc74", "towel-59e3239324500237", "centre plastic piece-ed2e5901533cc90f", "box of hand blender-176efa428d3287c7", "plastic piece3-4e3381491fff34a2", "meter inside-55baba28fc0e214b", "thin box-3e3a84fc654849a2", "food processor bowl-1604724a8b814fab", "plastic package2-9306496575fb0acc", "big plastic disk-31e4c540bda6b8cd", "plastic package-5e6c2ce7a37e07d5", "plastic piece2-d6ac269f6cca7a31", "cover of food processer bowl-63fafae74916e4b2", "blade cover2-c72a0652ad013a47", "blade cover1-16c5bb7df8a3b00f", "blade-fbb2475c3352e24a", "plastic piece-2f846fa74cbb90da", "packet of biscuits-e31326679f5a2aaa", "cocoa can1-b0b52dab15606165", "plastic bag-afa37c9a0cc43ec1", "cocoa can2-7562d02d21816f46", "can of condensed milk-a010ead84a02d07c", "butter package2-76405e0a9312d4a5", "butter's package-815655356e7b48a2", "sink hole plug-493a6ca0dc2bf4e8", "second chopping board-a4aac48cfe406ea6", "knife2-3a6e4e7a7ce9e4cb", "chopping board-23a815929202d3f4", "plate2-955b191e808136af", "plate-f91c00bbec7fc1e7", "thermal flask lid-5aae373ee1cd43e6", "mug2-de9142a7caff8a84", "glass-10b22113774eda67", "bowl3-3e3ada444f14a322", "bowl2-b8c4284c204e7938", "spoon2-ea8166aa48f267f5", "knife-f9a0ea882180239f", "spoon-fa8b332ffb422b60", "tupperware-9df836d37764b630", "bowl-e8e38966bc68fb94", "washing up liquids bottle-7e6ffa234d9992bd", "sponge-56a7e30a6aab9f87", "left glove-46c3c38b90003ec3", "right glove-e7ce0cee33ed285f", "thermal flask-8c868c36d1371275", "pan-3b2704d1615e19cc", "strainer-6e6b5c8110f81a55", "ladle-2129c22893ef7a4f", "stack of pots-d92bec3023cfb118", "saucepan-ee52592a1e022b52", "bag of flour-51668af48d1a7842", "lid2-011b4b830a9ed325", "lid-122b8c9de3ba9422", "tupperware lid-411ae9423ab398ff", "carrot piece-3ca360d7506fd255", "empty mug-f9f5c658de80a76c", "pen-4834a5a5778ef81c", "stock cube-d88474d879d24082", "one cube of stock-b3814d3e40f4c501", "box of stock cubes-d5b1a639aef5b172", "kettle-7b4238cb675c8cc9", "spatula-039f7e7efd96f484", "lid3-3db698d3ee6c0541" ], "reasoning": "1. I look for the time in which the user begins to wash dishes and load them into the dishwasher, finding it at 05:37.\n2. The first 3 items the user loads into the dishwasher are three utensils: a spoon (spoon2-ea8166aa48f267f5) at 05:57, a knife (knife-f9a0ea882180239f) at 05:58, and another spoon (spoon2-ea8166aa48f267f5) at 05:59.\n3. The fourth item the user loads is a bowl (bowl2-b8c4284c204e7938) at 06:09.\n4. The fifth item the user loads is another bowl (bowl3-3e3ada444f14a322) at 06:18.\n5. The sixth item the user puts into the dishwasher is a glass cup (glass-10b22113774eda67) from 06:30-06:31.\n6. I look for the time the user pours biscuit crumbs.\n7. The user pours biscuit crumbs from a food processing bowl (food processor bowl-1604724a8b814fab) at 2 separate times in the video: from 15:46-15:55, and from 17:15-17:28.\n8. At each time, the user tries to pour the crumbs into a drinking glass (glass2-99b48efeedbd26fd) first, before pouring the glass (glass2-99b48efeedbd26fd) into a white bowl (bowl-e8e38966bc68fb94).\n9. Since the sixth object loaded into the dishwasher and the thing into which biscuit crumbs are poured are both drinking glasses (glass-10b22113774eda67, glass2-99b48efeedbd26fd), I can conclude that they are both made of glass.", "referenced_object_masks": [ "bowl-e8e38966bc68fb94", "glass-10b22113774eda67", "bowl2-b8c4284c204e7938", "spoon2-ea8166aa48f267f5", "glass2-99b48efeedbd26fd", "knife-f9a0ea882180239f", "food processor bowl-1604724a8b814fab", "bowl3-3e3ada444f14a322" ] }, { "key": "P01-20240203-132119:9564baf95fc21184895a453a4f6e40e29a385bf3", "video_id": "P01-20240203-132119", "question": "During which of the first three rounds of putting biscuits into the food processor from the first biscuit tube do I pour the most biscuits?", "answer_choice_0": "Second round", "answer_choice_1": "The user puts in equal amounts of biscuits", "answer_choice_2": "The user pours all the biscuits in only one round", "answer_choice_3": "First round", "answer_choice_4": "Third round", "answer_id": 3, "answer": "First round", "question_type": [ "Counting", "Object Recognition" ], "objects_of_interest": [ "one piece of biscuit-b5310b516fe1376c", "biscuits3-095cea4424851bb4", "second tube of biscuits-3bb6fd99801fcf24", "glass2-99b48efeedbd26fd", "biscuits2-d8bd5e6bb0674e8c", "biscuits-c900ae96394784a9", "one plastic tube of biscuits-a7036423663732ea", "food processor-e92c427f58c23bf4", "plug of food processor-b6cfc7fb29d2dc74", "towel-59e3239324500237", "centre plastic piece-ed2e5901533cc90f", "box of hand blender-176efa428d3287c7", "plastic piece3-4e3381491fff34a2", "meter inside-55baba28fc0e214b", "thin box-3e3a84fc654849a2", "food processor bowl-1604724a8b814fab", "plastic package2-9306496575fb0acc", "big plastic disk-31e4c540bda6b8cd", "plastic package-5e6c2ce7a37e07d5", "plastic piece2-d6ac269f6cca7a31", "cover of food processer bowl-63fafae74916e4b2", "blade cover2-c72a0652ad013a47", "blade cover1-16c5bb7df8a3b00f", "blade-fbb2475c3352e24a", "plastic piece-2f846fa74cbb90da", "packet of biscuits-e31326679f5a2aaa", "cocoa can1-b0b52dab15606165", "plastic bag-afa37c9a0cc43ec1", "cocoa can2-7562d02d21816f46", "can of condensed milk-a010ead84a02d07c", "butter package2-76405e0a9312d4a5", "butter's package-815655356e7b48a2", "sink hole plug-493a6ca0dc2bf4e8", "second chopping board-a4aac48cfe406ea6", "knife2-3a6e4e7a7ce9e4cb", "chopping board-23a815929202d3f4", "plate2-955b191e808136af", "plate-f91c00bbec7fc1e7", "thermal flask lid-5aae373ee1cd43e6", "mug2-de9142a7caff8a84", "glass-10b22113774eda67", "bowl3-3e3ada444f14a322", "bowl2-b8c4284c204e7938", "spoon2-ea8166aa48f267f5", "knife-f9a0ea882180239f", "spoon-fa8b332ffb422b60", "tupperware-9df836d37764b630", "bowl-e8e38966bc68fb94", "washing up liquids bottle-7e6ffa234d9992bd", "sponge-56a7e30a6aab9f87", "left glove-46c3c38b90003ec3", "right glove-e7ce0cee33ed285f", "thermal flask-8c868c36d1371275", "pan-3b2704d1615e19cc", "strainer-6e6b5c8110f81a55", "ladle-2129c22893ef7a4f", "stack of pots-d92bec3023cfb118", "saucepan-ee52592a1e022b52", "bag of flour-51668af48d1a7842", "lid2-011b4b830a9ed325", "lid-122b8c9de3ba9422", "tupperware lid-411ae9423ab398ff", "carrot piece-3ca360d7506fd255", "empty mug-f9f5c658de80a76c", "pen-4834a5a5778ef81c", "stock cube-d88474d879d24082", "one cube of stock-b3814d3e40f4c501", "box of stock cubes-d5b1a639aef5b172", "kettle-7b4238cb675c8cc9", "spatula-039f7e7efd96f484", "lid3-3db698d3ee6c0541" ], "reasoning": "1. I look to find the times the user pours biscuits from the first biscuit tube (one plastic tube of biscuits-a7036423663732ea) into the food processor (food processor bowl-1604724a8b814fab), finding 4 separate times: 13:59-14:00, 14:27-14:31, 14:38-14:40, and 16:10-16:15. I count the number of biscuits poured in the first 3 times.\n2. From 13:59-14:00, I count that the user pours 9 biscuits into the food processor bowl (food processor bowl-1604724a8b814fab).\n3. From 14:27-14:31, I count that the user puts 6 biscuits (biscuits-c900ae96394784a9) into the food processor bowl (food processor bowl-1604724a8b814fab).\n4. From 14:38-14:40, I count that the user puts 5 biscuits (biscuits2-d8bd5e6bb0674e8c) into the bowl (food processor bowl-1604724a8b814fab).\n5. The user puts the most biscuits into the food processor (food processor bowl-1604724a8b814fab) from 13:59-14:00, the first round of pouring, when the user puts the 9 biscuits into the bowl (food processor bowl-1604724a8b814fab).\n6. Therefore, the answer is the first round.", "referenced_object_masks": [ "biscuits2-d8bd5e6bb0674e8c", "food processor bowl-1604724a8b814fab", "biscuits-c900ae96394784a9", "one plastic tube of biscuits-a7036423663732ea" ] }, { "key": "P01-20240203-132119:975dbf77489918d7aa3beac2177b56a3527fc2af", "video_id": "P01-20240203-132119", "question": "What sort of places in the kitchen do I go while holding the small knife with the black handle?", "answer_choice_0": "Countertop, compost bin, refrigerator, sink", "answer_choice_1": "Sink, countertop, stovetop, refrigerator", "answer_choice_2": "Dishwasher, sink, counter top, stovetop", "answer_choice_3": "Countertop, refrigerator, stovetop, compost bin", "answer_choice_4": "Dishwasher, countertop, compost bin, sink", "answer_id": 4, "answer": "Dishwasher, countertop, compost bin, sink", "question_type": [ "Egocentric & Spatial Perception", "Situational Awareness", "Object Recognition" ], "objects_of_interest": [ "one piece of biscuit-b5310b516fe1376c", "biscuits3-095cea4424851bb4", "second tube of biscuits-3bb6fd99801fcf24", "glass2-99b48efeedbd26fd", "biscuits2-d8bd5e6bb0674e8c", "biscuits-c900ae96394784a9", "one plastic tube of biscuits-a7036423663732ea", "food processor-e92c427f58c23bf4", "plug of food processor-b6cfc7fb29d2dc74", "towel-59e3239324500237", "centre plastic piece-ed2e5901533cc90f", "box of hand blender-176efa428d3287c7", "plastic piece3-4e3381491fff34a2", "meter inside-55baba28fc0e214b", "thin box-3e3a84fc654849a2", "food processor bowl-1604724a8b814fab", "plastic package2-9306496575fb0acc", "big plastic disk-31e4c540bda6b8cd", "plastic package-5e6c2ce7a37e07d5", "plastic piece2-d6ac269f6cca7a31", "cover of food processer bowl-63fafae74916e4b2", "blade cover2-c72a0652ad013a47", "blade cover1-16c5bb7df8a3b00f", "blade-fbb2475c3352e24a", "plastic piece-2f846fa74cbb90da", "packet of biscuits-e31326679f5a2aaa", "cocoa can1-b0b52dab15606165", "plastic bag-afa37c9a0cc43ec1", "cocoa can2-7562d02d21816f46", "can of condensed milk-a010ead84a02d07c", "butter package2-76405e0a9312d4a5", "butter's package-815655356e7b48a2", "sink hole plug-493a6ca0dc2bf4e8", "second chopping board-a4aac48cfe406ea6", "knife2-3a6e4e7a7ce9e4cb", "chopping board-23a815929202d3f4", "plate2-955b191e808136af", "plate-f91c00bbec7fc1e7", "thermal flask lid-5aae373ee1cd43e6", "mug2-de9142a7caff8a84", "glass-10b22113774eda67", "bowl3-3e3ada444f14a322", "bowl2-b8c4284c204e7938", "spoon2-ea8166aa48f267f5", "knife-f9a0ea882180239f", "spoon-fa8b332ffb422b60", "tupperware-9df836d37764b630", "bowl-e8e38966bc68fb94", "washing up liquids bottle-7e6ffa234d9992bd", "sponge-56a7e30a6aab9f87", "left glove-46c3c38b90003ec3", "right glove-e7ce0cee33ed285f", "thermal flask-8c868c36d1371275", "pan-3b2704d1615e19cc", "strainer-6e6b5c8110f81a55", "ladle-2129c22893ef7a4f", "stack of pots-d92bec3023cfb118", "saucepan-ee52592a1e022b52", "bag of flour-51668af48d1a7842", "lid2-011b4b830a9ed325", "lid-122b8c9de3ba9422", "tupperware lid-411ae9423ab398ff", "carrot piece-3ca360d7506fd255", "empty mug-f9f5c658de80a76c", "pen-4834a5a5778ef81c", "stock cube-d88474d879d24082", "one cube of stock-b3814d3e40f4c501", "box of stock cubes-d5b1a639aef5b172", "kettle-7b4238cb675c8cc9", "spatula-039f7e7efd96f484", "lid3-3db698d3ee6c0541" ], "reasoning": "1. I look for the time the user picks up a small knife with a black handle (knife2-3a6e4e7a7ce9e4cb). I find the time at 07:15. Then, I keep watching post 07:15 to see where the user goes.\n2. I see the user travel to the dishwasher at 07:17 while holding the knife (knife2-3a6e4e7a7ce9e4cb).\n3. At 07:23, I see the user travel to the countertop.\n4. From 07:28-07:29, I see the user move and reach to the compost bin .\n5. Finally, I see the user put the knife (knife2-3a6e4e7a7ce9e4cb) down by the sink at 07:31.\n8. The user does pick the knife (knife2-3a6e4e7a7ce9e4cb) up again at 07:40, but only to put the knife (knife2-3a6e4e7a7ce9e4cb) by the drying rack at 07:46. At this point, the user is still by the sink.\n9. Therefore, the answer is dishwasher, countertop, compost bin, and sink.", "referenced_object_masks": [ "knife2-3a6e4e7a7ce9e4cb" ] }, { "key": "P01-20240203-132119:984e9322eec8b4d32c1f383cf59f6be19b1263ba", "video_id": "P01-20240203-132119", "question": "What is the fifth item I removed from the dishwashing rack after I closed the bag of flour?", "answer_choice_0": "A plate", "answer_choice_1": "A thermal flask", "answer_choice_2": "A strainer", "answer_choice_3": "A saucepan", "answer_choice_4": "A mug", "answer_id": 1, "answer": "A thermal flask", "question_type": [ "Counting", "Object Recognition", "Egocentric & Spatial Perception", "Action Recognition" ], "objects_of_interest": [ "one piece of biscuit-b5310b516fe1376c", "biscuits3-095cea4424851bb4", "second tube of biscuits-3bb6fd99801fcf24", "glass2-99b48efeedbd26fd", "biscuits2-d8bd5e6bb0674e8c", "biscuits-c900ae96394784a9", "one plastic tube of biscuits-a7036423663732ea", "food processor-e92c427f58c23bf4", "plug of food processor-b6cfc7fb29d2dc74", "towel-59e3239324500237", "centre plastic piece-ed2e5901533cc90f", "box of hand blender-176efa428d3287c7", "plastic piece3-4e3381491fff34a2", "meter inside-55baba28fc0e214b", "thin box-3e3a84fc654849a2", "food processor bowl-1604724a8b814fab", "plastic package2-9306496575fb0acc", "big plastic disk-31e4c540bda6b8cd", "plastic package-5e6c2ce7a37e07d5", "plastic piece2-d6ac269f6cca7a31", "cover of food processer bowl-63fafae74916e4b2", "blade cover2-c72a0652ad013a47", "blade cover1-16c5bb7df8a3b00f", "blade-fbb2475c3352e24a", "plastic piece-2f846fa74cbb90da", "packet of biscuits-e31326679f5a2aaa", "cocoa can1-b0b52dab15606165", "plastic bag-afa37c9a0cc43ec1", "cocoa can2-7562d02d21816f46", "can of condensed milk-a010ead84a02d07c", "butter package2-76405e0a9312d4a5", "butter's package-815655356e7b48a2", "sink hole plug-493a6ca0dc2bf4e8", "second chopping board-a4aac48cfe406ea6", "knife2-3a6e4e7a7ce9e4cb", "chopping board-23a815929202d3f4", "plate2-955b191e808136af", "plate-f91c00bbec7fc1e7", "thermal flask lid-5aae373ee1cd43e6", "mug2-de9142a7caff8a84", "glass-10b22113774eda67", "bowl3-3e3ada444f14a322", "bowl2-b8c4284c204e7938", "spoon2-ea8166aa48f267f5", "knife-f9a0ea882180239f", "spoon-fa8b332ffb422b60", "tupperware-9df836d37764b630", "bowl-e8e38966bc68fb94", "washing up liquids bottle-7e6ffa234d9992bd", "sponge-56a7e30a6aab9f87", "left glove-46c3c38b90003ec3", "right glove-e7ce0cee33ed285f", "thermal flask-8c868c36d1371275", "pan-3b2704d1615e19cc", "strainer-6e6b5c8110f81a55", "ladle-2129c22893ef7a4f", "stack of pots-d92bec3023cfb118", "saucepan-ee52592a1e022b52", "bag of flour-51668af48d1a7842", "lid2-011b4b830a9ed325", "lid-122b8c9de3ba9422", "tupperware lid-411ae9423ab398ff", "carrot piece-3ca360d7506fd255", "empty mug-f9f5c658de80a76c", "pen-4834a5a5778ef81c", "stock cube-d88474d879d24082", "one cube of stock-b3814d3e40f4c501", "box of stock cubes-d5b1a639aef5b172", "kettle-7b4238cb675c8cc9", "spatula-039f7e7efd96f484", "lid3-3db698d3ee6c0541" ], "reasoning": "1. I watched the video to see when the user closed a bag of flour (bag of flour-51668af48d1a7842), and I found that it occurred at 03:30.\n2. Then, I watched to see when the user would remove items from the dishwashing rack , preparing to count to the fifth occurrence. I found they removed a saucepan (saucepan-ee52592a1e022b52) at 03:34, a ladle (ladle-2129c22893ef7a4f) and a strainer (strainer-6e6b5c8110f81a55) at 03:44, a pan (pan-3b2704d1615e19cc) at 04:00, and then a thermal flask (thermal flask-8c868c36d1371275) at 04:08.\n3. Counting up, I determined that the thermal flask (thermal flask-8c868c36d1371275) was the fifth item to be removed from the dishwashing rack after the user closed the bag of flour (bag of flour-51668af48d1a7842).", "referenced_object_masks": [ "pan-3b2704d1615e19cc", "bag of flour-51668af48d1a7842", "thermal flask-8c868c36d1371275", "saucepan-ee52592a1e022b52", "ladle-2129c22893ef7a4f", "strainer-6e6b5c8110f81a55" ] }, { "key": "P01-20240203-132119:9893317e4e1f5173591583f30258256a93ac0cf2", "video_id": "P01-20240203-132119", "question": "How many times did I put biscuits inside the food processor?", "answer_choice_0": "6 times", "answer_choice_1": "3 times", "answer_choice_2": "4 times", "answer_choice_3": "2 times", "answer_choice_4": "5 times", "answer_id": 4, "answer": "5 times", "question_type": [ "Action Recognition", "Counting", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "one piece of biscuit-b5310b516fe1376c", "biscuits3-095cea4424851bb4", "second tube of biscuits-3bb6fd99801fcf24", "glass2-99b48efeedbd26fd", "biscuits2-d8bd5e6bb0674e8c", "biscuits-c900ae96394784a9", "one plastic tube of biscuits-a7036423663732ea", "food processor-e92c427f58c23bf4", "plug of food processor-b6cfc7fb29d2dc74", "towel-59e3239324500237", "centre plastic piece-ed2e5901533cc90f", "box of hand blender-176efa428d3287c7", "plastic piece3-4e3381491fff34a2", "meter inside-55baba28fc0e214b", "thin box-3e3a84fc654849a2", "food processor bowl-1604724a8b814fab", "plastic package2-9306496575fb0acc", "big plastic disk-31e4c540bda6b8cd", "plastic package-5e6c2ce7a37e07d5", "plastic piece2-d6ac269f6cca7a31", "cover of food processer bowl-63fafae74916e4b2", "blade cover2-c72a0652ad013a47", "blade cover1-16c5bb7df8a3b00f", "blade-fbb2475c3352e24a", "plastic piece-2f846fa74cbb90da", "packet of biscuits-e31326679f5a2aaa", "cocoa can1-b0b52dab15606165", "plastic bag-afa37c9a0cc43ec1", "cocoa can2-7562d02d21816f46", "can of condensed milk-a010ead84a02d07c", "butter package2-76405e0a9312d4a5", "butter's package-815655356e7b48a2", "sink hole plug-493a6ca0dc2bf4e8", "second chopping board-a4aac48cfe406ea6", "knife2-3a6e4e7a7ce9e4cb", "chopping board-23a815929202d3f4", "plate2-955b191e808136af", "plate-f91c00bbec7fc1e7", "thermal flask lid-5aae373ee1cd43e6", "mug2-de9142a7caff8a84", "glass-10b22113774eda67", "bowl3-3e3ada444f14a322", "bowl2-b8c4284c204e7938", "spoon2-ea8166aa48f267f5", "knife-f9a0ea882180239f", "spoon-fa8b332ffb422b60", "tupperware-9df836d37764b630", "bowl-e8e38966bc68fb94", "washing up liquids bottle-7e6ffa234d9992bd", "sponge-56a7e30a6aab9f87", "left glove-46c3c38b90003ec3", "right glove-e7ce0cee33ed285f", "thermal flask-8c868c36d1371275", "pan-3b2704d1615e19cc", "strainer-6e6b5c8110f81a55", "ladle-2129c22893ef7a4f", "stack of pots-d92bec3023cfb118", "saucepan-ee52592a1e022b52", "bag of flour-51668af48d1a7842", "lid2-011b4b830a9ed325", "lid-122b8c9de3ba9422", "tupperware lid-411ae9423ab398ff", "carrot piece-3ca360d7506fd255", "empty mug-f9f5c658de80a76c", "pen-4834a5a5778ef81c", "stock cube-d88474d879d24082", "one cube of stock-b3814d3e40f4c501", "box of stock cubes-d5b1a639aef5b172", "kettle-7b4238cb675c8cc9", "spatula-039f7e7efd96f484", "lid3-3db698d3ee6c0541" ], "reasoning": "1. I watched the video and saw the user place the food processor (food processor-e92c427f58c23bf4) on the counter at 13:20. \n2. I then saw the user pick up a package of biscuits at 13:44 and then open it, pulling out a single plastic tube of biscuits (one plastic tube of biscuits-a7036423663732ea).\n3. Then, I continued watching to find the times when the user places biscuits (biscuits-c900ae96394784a9) into the food processor (food processor-e92c427f58c23bf4). From the tube of biscuits (one plastic tube of biscuits-a7036423663732ea), the user places biscuits (biscuits-c900ae96394784a9) into the food processor (food processor-e92c427f58c23bf4) at 13:59, 14:30, 14:40, and from 16:11-16:15.\n4. At 17:37, I noticed the user picked up a second plastic tube of biscuits (second tube of biscuits-3bb6fd99801fcf24) and opened it.\n5. Then, from 17:46-17:55, I noticed the user dumped the entirety of the biscuits (biscuits3-095cea4424851bb4) from the second plastic tube of biscuits (second tube of biscuits-3bb6fd99801fcf24) into the food processor (food processor-e92c427f58c23bf4).\n6. Counting up all the times, I determined that the answer is 5 times.", "referenced_object_masks": [ "biscuits3-095cea4424851bb4", "one plastic tube of biscuits-a7036423663732ea", "food processor-e92c427f58c23bf4", "second tube of biscuits-3bb6fd99801fcf24", "biscuits-c900ae96394784a9" ] }, { "key": "P01-20240203-132119:a691229a9a2aa9848ec06df2a90b0a845dea8b04", "video_id": "P01-20240203-132119", "question": "If I were facing the sink, in what directions, using clock bearing, do I find the seventh and eighth item I load into the dishwasher?", "answer_choice_0": "1 o'clock and 4 o'clock", "answer_choice_1": "9 o'clock and 1 o'clock", "answer_choice_2": "12 o'clock and 3 o'clock", "answer_choice_3": "12 o'clock and 8 o'clock", "answer_choice_4": "11 o'clock and 1 o'clock", "answer_id": 0, "answer": "1 o'clock and 4 o'clock", "question_type": [ "Egocentric & Spatial Perception", "Counting", "Object Recognition", "Situational Awareness", "Action Recognition" ], "objects_of_interest": [ "one piece of biscuit-b5310b516fe1376c", "biscuits3-095cea4424851bb4", "second tube of biscuits-3bb6fd99801fcf24", "glass2-99b48efeedbd26fd", "biscuits2-d8bd5e6bb0674e8c", "biscuits-c900ae96394784a9", "one plastic tube of biscuits-a7036423663732ea", "food processor-e92c427f58c23bf4", "plug of food processor-b6cfc7fb29d2dc74", "towel-59e3239324500237", "centre plastic piece-ed2e5901533cc90f", "box of hand blender-176efa428d3287c7", "plastic piece3-4e3381491fff34a2", "meter inside-55baba28fc0e214b", "thin box-3e3a84fc654849a2", "food processor bowl-1604724a8b814fab", "plastic package2-9306496575fb0acc", "big plastic disk-31e4c540bda6b8cd", "plastic package-5e6c2ce7a37e07d5", "plastic piece2-d6ac269f6cca7a31", "cover of food processer bowl-63fafae74916e4b2", "blade cover2-c72a0652ad013a47", "blade cover1-16c5bb7df8a3b00f", "blade-fbb2475c3352e24a", "plastic piece-2f846fa74cbb90da", "packet of biscuits-e31326679f5a2aaa", "cocoa can1-b0b52dab15606165", "plastic bag-afa37c9a0cc43ec1", "cocoa can2-7562d02d21816f46", "can of condensed milk-a010ead84a02d07c", "butter package2-76405e0a9312d4a5", "butter's package-815655356e7b48a2", "sink hole plug-493a6ca0dc2bf4e8", "second chopping board-a4aac48cfe406ea6", "knife2-3a6e4e7a7ce9e4cb", "chopping board-23a815929202d3f4", "plate2-955b191e808136af", "plate-f91c00bbec7fc1e7", "thermal flask lid-5aae373ee1cd43e6", "mug2-de9142a7caff8a84", "glass-10b22113774eda67", "bowl3-3e3ada444f14a322", "bowl2-b8c4284c204e7938", "spoon2-ea8166aa48f267f5", "knife-f9a0ea882180239f", "spoon-fa8b332ffb422b60", "tupperware-9df836d37764b630", "bowl-e8e38966bc68fb94", "washing up liquids bottle-7e6ffa234d9992bd", "sponge-56a7e30a6aab9f87", "left glove-46c3c38b90003ec3", "right glove-e7ce0cee33ed285f", "thermal flask-8c868c36d1371275", "pan-3b2704d1615e19cc", "strainer-6e6b5c8110f81a55", "ladle-2129c22893ef7a4f", "stack of pots-d92bec3023cfb118", "saucepan-ee52592a1e022b52", "bag of flour-51668af48d1a7842", "lid2-011b4b830a9ed325", "lid-122b8c9de3ba9422", "tupperware lid-411ae9423ab398ff", "carrot piece-3ca360d7506fd255", "empty mug-f9f5c658de80a76c", "pen-4834a5a5778ef81c", "stock cube-d88474d879d24082", "one cube of stock-b3814d3e40f4c501", "box of stock cubes-d5b1a639aef5b172", "kettle-7b4238cb675c8cc9", "spatula-039f7e7efd96f484", "lid3-3db698d3ee6c0541" ], "reasoning": "1. I look for the time the user starts loading items into the dishwasher. I begin to do so at 05:37.\n2. The first 3 items the user loads into the dishwasher are three utensils: a spoon (spoon2-ea8166aa48f267f5) at 05:57, a knife (knife-f9a0ea882180239f) at 05:58, and another spoon (spoon2-ea8166aa48f267f5) at 05:59.\n3. The fourth item the user loads is a bowl (bowl2-b8c4284c204e7938) at 06:09.\n4. The fifth item the user loads is another bowl (bowl3-3e3ada444f14a322) at 06:18.\n5. The sixth item the user puts into the dishwasher is a glass cup (glass-10b22113774eda67) from 06:30-06:31.\n6. The seventh item the user puts into the dishwasher is a plate (plate-f91c00bbec7fc1e7) from 06:57-06:59.\n7. The eighth item the user puts into the dishwasher is another plate (plate2-955b191e808136af) from 07:07-07:08.\n8. I look to see where these two plates (plate-f91c00bbec7fc1e7, plate2-955b191e808136af) came from, and I realize that the first plate (plate-f91c00bbec7fc1e7) came from just to the right of the sink and the second plate (plate2-955b191e808136af) from on top of the microwave .\n9. I realize that the right of the sink corresponds to about 1 o'clock in clock bearing, and the microwave is just a little further past 3 o'clock if the user were standing at the sink facing forward.\n10. Therefore, the answer is 1 o'clock and 4 o'clock.", "referenced_object_masks": [ "glass-10b22113774eda67", "plate-f91c00bbec7fc1e7", "bowl2-b8c4284c204e7938", "spoon2-ea8166aa48f267f5", "plate2-955b191e808136af", "knife-f9a0ea882180239f", "bowl3-3e3ada444f14a322" ] }, { "key": "P01-20240203-132119:b39810b3543f0e7fd4c05d69da3fcac455b6edbe", "video_id": "P01-20240203-132119", "question": "How much time passes from when I start stirring the pot of vegetables until I finish stirring it the final time?", "answer_choice_0": "1 minute and 15 seconds", "answer_choice_1": "1 minute and 48 seconds", "answer_choice_2": "2 minutes and 5 seconds", "answer_choice_3": "3 minutes and 2 seconds", "answer_choice_4": "2 minutes and 36 seconds", "answer_id": 4, "answer": "2 minutes and 36 seconds", "question_type": [ "Action Recognition", "Numerical Reasoning", "Object Recognition", "Event Occurence", "Temporal Reasoning" ], "objects_of_interest": [ "one piece of biscuit-b5310b516fe1376c", "biscuits3-095cea4424851bb4", "second tube of biscuits-3bb6fd99801fcf24", "glass2-99b48efeedbd26fd", "biscuits2-d8bd5e6bb0674e8c", "biscuits-c900ae96394784a9", "one plastic tube of biscuits-a7036423663732ea", "food processor-e92c427f58c23bf4", "plug of food processor-b6cfc7fb29d2dc74", "towel-59e3239324500237", "centre plastic piece-ed2e5901533cc90f", "box of hand blender-176efa428d3287c7", "plastic piece3-4e3381491fff34a2", "meter inside-55baba28fc0e214b", "thin box-3e3a84fc654849a2", "food processor bowl-1604724a8b814fab", "plastic package2-9306496575fb0acc", "big plastic disk-31e4c540bda6b8cd", "plastic package-5e6c2ce7a37e07d5", "plastic piece2-d6ac269f6cca7a31", "cover of food processer bowl-63fafae74916e4b2", "blade cover2-c72a0652ad013a47", "blade cover1-16c5bb7df8a3b00f", "blade-fbb2475c3352e24a", "plastic piece-2f846fa74cbb90da", "packet of biscuits-e31326679f5a2aaa", "cocoa can1-b0b52dab15606165", "plastic bag-afa37c9a0cc43ec1", "cocoa can2-7562d02d21816f46", "can of condensed milk-a010ead84a02d07c", "butter package2-76405e0a9312d4a5", "butter's package-815655356e7b48a2", "sink hole plug-493a6ca0dc2bf4e8", "second chopping board-a4aac48cfe406ea6", "knife2-3a6e4e7a7ce9e4cb", "chopping board-23a815929202d3f4", "plate2-955b191e808136af", "plate-f91c00bbec7fc1e7", "thermal flask lid-5aae373ee1cd43e6", "mug2-de9142a7caff8a84", "glass-10b22113774eda67", "bowl3-3e3ada444f14a322", "bowl2-b8c4284c204e7938", "spoon2-ea8166aa48f267f5", "knife-f9a0ea882180239f", "spoon-fa8b332ffb422b60", "tupperware-9df836d37764b630", "bowl-e8e38966bc68fb94", "washing up liquids bottle-7e6ffa234d9992bd", "sponge-56a7e30a6aab9f87", "left glove-46c3c38b90003ec3", "right glove-e7ce0cee33ed285f", "thermal flask-8c868c36d1371275", "pan-3b2704d1615e19cc", "strainer-6e6b5c8110f81a55", "ladle-2129c22893ef7a4f", "stack of pots-d92bec3023cfb118", "saucepan-ee52592a1e022b52", "bag of flour-51668af48d1a7842", "lid2-011b4b830a9ed325", "lid-122b8c9de3ba9422", "tupperware lid-411ae9423ab398ff", "carrot piece-3ca360d7506fd255", "empty mug-f9f5c658de80a76c", "pen-4834a5a5778ef81c", "stock cube-d88474d879d24082", "one cube of stock-b3814d3e40f4c501", "box of stock cubes-d5b1a639aef5b172", "kettle-7b4238cb675c8cc9", "spatula-039f7e7efd96f484", "lid3-3db698d3ee6c0541" ], "reasoning": "1. I watched the video and noticed the pot of vegetables simmering on the stove at 00:04.\n2. I continued watching and noticed that the user used a wooden spatula (spatula-039f7e7efd96f484) to stir the pot of vegetables from 00:05-00:22, 00:42-00:51, 01:26-02:05, and 02:30-02:41. \n3. Watching the rest of the video, I did not notice any other instances of stirring the pot of vegetables .\n4. Since the user started stirring the pot of vegetables for the first time at 00:05 and finished the last time at 02:41, I subtracted 02:41-00:05 to get a final value of 02:36, making 2 minutes and 36 seconds the final answer.", "referenced_object_masks": [ "spatula-039f7e7efd96f484" ] }, { "key": "P01-20240203-132119:e897f953121a7b821c9d7255a4f612286322c194", "video_id": "P01-20240203-132119", "question": "What is the first item I pick up after throwing something away in the countertop trash bin for the second time?", "answer_choice_0": "A towel", "answer_choice_1": "A tube of biscuits", "answer_choice_2": "A knife", "answer_choice_3": "A sink plug", "answer_choice_4": "A plate", "answer_id": 3, "answer": "A sink plug", "question_type": [ "Counting", "Action Recognition", "Object Recognition" ], "objects_of_interest": [ "one piece of biscuit-b5310b516fe1376c", "biscuits3-095cea4424851bb4", "second tube of biscuits-3bb6fd99801fcf24", "glass2-99b48efeedbd26fd", "biscuits2-d8bd5e6bb0674e8c", "biscuits-c900ae96394784a9", "one plastic tube of biscuits-a7036423663732ea", "food processor-e92c427f58c23bf4", "plug of food processor-b6cfc7fb29d2dc74", "towel-59e3239324500237", "centre plastic piece-ed2e5901533cc90f", "box of hand blender-176efa428d3287c7", "plastic piece3-4e3381491fff34a2", "meter inside-55baba28fc0e214b", "thin box-3e3a84fc654849a2", "food processor bowl-1604724a8b814fab", "plastic package2-9306496575fb0acc", "big plastic disk-31e4c540bda6b8cd", "plastic package-5e6c2ce7a37e07d5", "plastic piece2-d6ac269f6cca7a31", "cover of food processer bowl-63fafae74916e4b2", "blade cover2-c72a0652ad013a47", "blade cover1-16c5bb7df8a3b00f", "blade-fbb2475c3352e24a", "plastic piece-2f846fa74cbb90da", "packet of biscuits-e31326679f5a2aaa", "cocoa can1-b0b52dab15606165", "plastic bag-afa37c9a0cc43ec1", "cocoa can2-7562d02d21816f46", "can of condensed milk-a010ead84a02d07c", "butter package2-76405e0a9312d4a5", "butter's package-815655356e7b48a2", "sink hole plug-493a6ca0dc2bf4e8", "second chopping board-a4aac48cfe406ea6", "knife2-3a6e4e7a7ce9e4cb", "chopping board-23a815929202d3f4", "plate2-955b191e808136af", "plate-f91c00bbec7fc1e7", "thermal flask lid-5aae373ee1cd43e6", "mug2-de9142a7caff8a84", "glass-10b22113774eda67", "bowl3-3e3ada444f14a322", "bowl2-b8c4284c204e7938", "spoon2-ea8166aa48f267f5", "knife-f9a0ea882180239f", "spoon-fa8b332ffb422b60", "tupperware-9df836d37764b630", "bowl-e8e38966bc68fb94", "washing up liquids bottle-7e6ffa234d9992bd", "sponge-56a7e30a6aab9f87", "left glove-46c3c38b90003ec3", "right glove-e7ce0cee33ed285f", "thermal flask-8c868c36d1371275", "pan-3b2704d1615e19cc", "strainer-6e6b5c8110f81a55", "ladle-2129c22893ef7a4f", "stack of pots-d92bec3023cfb118", "saucepan-ee52592a1e022b52", "bag of flour-51668af48d1a7842", "lid2-011b4b830a9ed325", "lid-122b8c9de3ba9422", "tupperware lid-411ae9423ab398ff", "carrot piece-3ca360d7506fd255", "empty mug-f9f5c658de80a76c", "pen-4834a5a5778ef81c", "stock cube-d88474d879d24082", "one cube of stock-b3814d3e40f4c501", "box of stock cubes-d5b1a639aef5b172", "kettle-7b4238cb675c8cc9", "spatula-039f7e7efd96f484", "lid3-3db698d3ee6c0541" ], "reasoning": "1. I watched the video to find a countertop trash bin and I noticed it for the first time at 02:16 on the counter to the right of the sink.\n2. Then, I looked for the moments when the user threw something away into the countertop trash bin , preparing to count to the second occurrence. I found the first occurrence at 02:25, when the user threw away a carrot piece (carrot piece-3ca360d7506fd255).\n3. I continued to watch the video, looking for a second occurrence of the user throwing something away into the countertop trash bin . I found this at 08:20, when the user dumped the contents of the sink plug (sink hole plug-493a6ca0dc2bf4e8) into the countertop trash bin .\n4. Then, I looked to see what item the user picks up after this moment. At 08:25, the user puts the sink plug (sink hole plug-493a6ca0dc2bf4e8) back into the sink, then picks it back up again at 08:40, making the sink plug (sink hole plug-493a6ca0dc2bf4e8) the first thing the user picks up after throwing something away in the countertop trashbin for the second time.", "referenced_object_masks": [ "sink hole plug-493a6ca0dc2bf4e8", "carrot piece-3ca360d7506fd255" ] }, { "key": "P01-20240203-135502:0abea78b4af1bb1f78f697146fd066f93b3ef0d4", "video_id": "P01-20240203-135502", "question": "What is the total duration of the two following actions: stirring the cocoa-butter mixture for the second time, and stirring the condensed milk into the cocoa-butter mixture before putting the empty condensed milk can on the kitchen scale?", "answer_choice_0": "40 seconds", "answer_choice_1": "45 seconds", "answer_choice_2": "34 seconds", "answer_choice_3": "30 seconds", "answer_choice_4": "27 seconds", "answer_id": 2, "answer": "34 seconds", "question_type": [ "Action Recognition", "Object Recognition", "Numerical Reasoning", "Temporal Reasoning" ], "objects_of_interest": [ "spoon6-56d1bd6fa3efbff6", "spoon5-e9b7623ef30f0481", "ladle-2e157ee745fb2769", "bowl3-17ad8761bbf76b51", "knife-98ff5c4d9609a313", "strainer-c29787451611cf9e", "chopping board-8bfb8696e3f09575", "bottle of vegetable oil-945822ab78f564dd", "opened bag of coriander-724ebd15c9c593f4", "towel4-bca834db4478e9df", "teaspoon-ebd18f71ffcc59e5", "jar of spice-2a375bfc9fb9cefa", "blender's piece-82cf565fcf91e56a", "blender-2cbf7e23b8778dc1", "box-e2e5b817c182d16d", "milk frother-3976a1d220a647d2", "plate-2a3d260658b7297a", "spoon4-8740eb7f6177f08b", "jar of sumac spice-55fdab7d47bd5cdb", "spoon3-c0448a4e5347a656", "jar of tahini lid-b9a0a5ea12514273", "jar of tahini-5af78fda7152b5a8", "pan lid2-1ab40216c2e0231c", "pan-dd1e2a31005ddc4f", "olive oil bottle-f055ed448bb10e7e", "tray2-852114c78033e189", "big tray-cd8e8e454bdd8399", "plastic lid-1e9911428db39782", "pile of three trays-d980e89e88480a86", "two towels-b526393ed7990507", "towel3-2eca46eb45f98aef", "towel2-e28980389a3983a5", "bowl2-a7f10388cbbe891d", "box of foil paper-47353281413f77fe", "garlic presser-c5754e05ed234795", "towel-3e765153ebbf8e8f", "washing up liquid bottle-21b2c5ec30f52e2d", "jar of chocolate-9ca7e56a97201f9f", "yogurt-4ca4015083cb1c5c", "spoon2-e806bb70c4afee5f", "three broken pieces of carrot-0daa4ce0c0eb3175", "large chopping board-42a15da8fcb4c56f", "scale-d8e077019d84450d", "biscuit packet-2b5427f5280c5c79", "box of parchment paper-752cf60b10869373", "plastic spoon-304f4bdfb98a2fbc", "box of foil-33caa5d34fdb8406", "can opener-e10b7ce5b4bdaab1", "tray-d103ce70421e1b3a", "collander-058f36ffab7bf054", "things-5c0bc499982e93c4", "cocoa can lid-e9c2929744417e9a", "can of cocoa-97c15b1ab1811716", "small cup of glass-84e936d047991ae0", "small bowl of glass-d61bb7cc9e349b08", "spoon-a4fc0f681c68164e", "can of condensed milk-8bfb182b085406e6", "butter knife-84a85666d0244e97", "notepad-00e192a8b6e8c541", "pen-c70015127cd3d8b8", "plastic part-c6c1c20faae74969", "peeler-7776de9417366fdc", "plastic cover-9a33e9ccd2b027ed", "plastic tupperware-87625abfc216477e", "metal pot-ea599bb07a3e5ca8", "butter-b3d3296c54f0f89b", "wooden spoon-f5a9ca7515580d65", "top smaller pot-b43849214ce7bce0", "pot2-852a8e528ba115f0", "pot-328ab9c9eed4d5ed", "pan lid-f86d3b28227aa40f", "spatula-4c1069565d3ee9fb", "large bowl-235c137c0a6ccd8f", "glass-a902246ff484d129", "food processor bowl-634a88fe4f3081c7", "blade-7d7926605b41ce2d", "center plastic piece-d65bf7beceed6184", "food processor lid-198194792104bb6c" ], "reasoning": "1. The user began stirring together the cocoa from the can of cocoa (can of cocoa-97c15b1ab1811716) with the butter (butter-b3d3296c54f0f89b) for the first time at 06:38. They stirred the mixture for a second time from 07:38 to 07:48, which is 10 seconds.\n2. The user first stirred the condensed milk from the can of condensed milk (can of condensed milk-8bfb182b085406e6) with the cocoa-butter mixture at 08:37. The user finished stirring for the first time right before putting the empty condensed milk can (can of condensed milk-8bfb182b085406e6) on the kitchen scale (scale-d8e077019d84450d) at 09:04 which is 27 seconds.\n3. However, I also notice that the user stops stirring from 9:00-9:03, before resuming again. This makes a total of 3 seconds spent not stirring.\n4. I calculate 10+27-3=34.\n3. I conclude that the answer is 34.", "referenced_object_masks": [ "can of condensed milk-8bfb182b085406e6", "scale-d8e077019d84450d", "butter-b3d3296c54f0f89b", "can of cocoa-97c15b1ab1811716" ] }, { "key": "P01-20240203-135502:0d0f4dcf8c08ee58fae1fabad70325962ca07b94", "video_id": "P01-20240203-135502", "question": "What is the difference between the state of the leftover butter and the used butter (other than being leftover and used)?", "answer_choice_0": "One was eaten, one was thrown away", "answer_choice_1": "One was thrown away, one was left on the counter", "answer_choice_2": "One was frozen, one was melted", "answer_choice_3": "Half of the butter was spoiled, the other half was not", "answer_choice_4": "One was refrigerated, one was melted", "answer_id": 4, "answer": "One was refrigerated, one was melted", "question_type": [ "Action Recognition", "Object Recognition", "State Changes" ], "objects_of_interest": [ "spoon6-56d1bd6fa3efbff6", "spoon5-e9b7623ef30f0481", "ladle-2e157ee745fb2769", "bowl3-17ad8761bbf76b51", "knife-98ff5c4d9609a313", "strainer-c29787451611cf9e", "chopping board-8bfb8696e3f09575", "bottle of vegetable oil-945822ab78f564dd", "opened bag of coriander-724ebd15c9c593f4", "towel4-bca834db4478e9df", "teaspoon-ebd18f71ffcc59e5", "jar of spice-2a375bfc9fb9cefa", "blender's piece-82cf565fcf91e56a", "blender-2cbf7e23b8778dc1", "box-e2e5b817c182d16d", "milk frother-3976a1d220a647d2", "plate-2a3d260658b7297a", "spoon4-8740eb7f6177f08b", "jar of sumac spice-55fdab7d47bd5cdb", "spoon3-c0448a4e5347a656", "jar of tahini lid-b9a0a5ea12514273", "jar of tahini-5af78fda7152b5a8", "pan lid2-1ab40216c2e0231c", "pan-dd1e2a31005ddc4f", "olive oil bottle-f055ed448bb10e7e", "tray2-852114c78033e189", "big tray-cd8e8e454bdd8399", "plastic lid-1e9911428db39782", "pile of three trays-d980e89e88480a86", "two towels-b526393ed7990507", "towel3-2eca46eb45f98aef", "towel2-e28980389a3983a5", "bowl2-a7f10388cbbe891d", "box of foil paper-47353281413f77fe", "garlic presser-c5754e05ed234795", "towel-3e765153ebbf8e8f", "washing up liquid bottle-21b2c5ec30f52e2d", "jar of chocolate-9ca7e56a97201f9f", "yogurt-4ca4015083cb1c5c", "spoon2-e806bb70c4afee5f", "three broken pieces of carrot-0daa4ce0c0eb3175", "large chopping board-42a15da8fcb4c56f", "scale-d8e077019d84450d", "biscuit packet-2b5427f5280c5c79", "box of parchment paper-752cf60b10869373", "plastic spoon-304f4bdfb98a2fbc", "box of foil-33caa5d34fdb8406", "can opener-e10b7ce5b4bdaab1", "tray-d103ce70421e1b3a", "collander-058f36ffab7bf054", "things-5c0bc499982e93c4", "cocoa can lid-e9c2929744417e9a", "can of cocoa-97c15b1ab1811716", "small cup of glass-84e936d047991ae0", "small bowl of glass-d61bb7cc9e349b08", "spoon-a4fc0f681c68164e", "can of condensed milk-8bfb182b085406e6", "butter knife-84a85666d0244e97", "notepad-00e192a8b6e8c541", "pen-c70015127cd3d8b8", "plastic part-c6c1c20faae74969", "peeler-7776de9417366fdc", "plastic cover-9a33e9ccd2b027ed", "plastic tupperware-87625abfc216477e", "metal pot-ea599bb07a3e5ca8", "butter-b3d3296c54f0f89b", "wooden spoon-f5a9ca7515580d65", "top smaller pot-b43849214ce7bce0", "pot2-852a8e528ba115f0", "pot-328ab9c9eed4d5ed", "pan lid-f86d3b28227aa40f", "spatula-4c1069565d3ee9fb", "large bowl-235c137c0a6ccd8f", "glass-a902246ff484d129", "food processor bowl-634a88fe4f3081c7", "blade-7d7926605b41ce2d", "center plastic piece-d65bf7beceed6184", "food processor lid-198194792104bb6c" ], "reasoning": "1. I watched the video to establish what the \"leftover\" butter (butter-b3d3296c54f0f89b) is and what the \"used\" butter (butter-b3d3296c54f0f89b) is, and saw that the user puts half of the butter (butter-b3d3296c54f0f89b) in the pot at 04:16, then the \"leftover\" half of it in the fridge at 04:21.\n2. The user then puts the butter (butter-b3d3296c54f0f89b) on the stove and proceeds to melt it between 04:26-06:33. \n3. Therefore, the user refrigerates the leftover butter (butter-b3d3296c54f0f89b) and melts the other half of the butter.", "referenced_object_masks": [ "butter-b3d3296c54f0f89b" ] }, { "key": "P01-20240203-135502:1a3b609256fce3067b12cb16b4dc1526031f3256", "video_id": "P01-20240203-135502", "question": "What happened to the largest chunk of butter 30 seconds after I dropped it from its wrapping?", "answer_choice_0": "It was melted on the stove", "answer_choice_1": "It moved from the countertop to the stovetop", "answer_choice_2": "It moved from the countertop to the sink", "answer_choice_3": "It was mashed into smaller pieces", "answer_choice_4": "It was weighed and then dumped", "answer_id": 1, "answer": "It moved from the countertop to the stovetop", "question_type": [ "Action Recognition", "Temporal Reasoning", "Object Recognition", "State Changes", "Situational Awareness" ], "objects_of_interest": [ "spoon6-56d1bd6fa3efbff6", "spoon5-e9b7623ef30f0481", "ladle-2e157ee745fb2769", "bowl3-17ad8761bbf76b51", "knife-98ff5c4d9609a313", "strainer-c29787451611cf9e", "chopping board-8bfb8696e3f09575", "bottle of vegetable oil-945822ab78f564dd", "opened bag of coriander-724ebd15c9c593f4", "towel4-bca834db4478e9df", "teaspoon-ebd18f71ffcc59e5", "jar of spice-2a375bfc9fb9cefa", "blender's piece-82cf565fcf91e56a", "blender-2cbf7e23b8778dc1", "box-e2e5b817c182d16d", "milk frother-3976a1d220a647d2", "plate-2a3d260658b7297a", "spoon4-8740eb7f6177f08b", "jar of sumac spice-55fdab7d47bd5cdb", "spoon3-c0448a4e5347a656", "jar of tahini lid-b9a0a5ea12514273", "jar of tahini-5af78fda7152b5a8", "pan lid2-1ab40216c2e0231c", "pan-dd1e2a31005ddc4f", "olive oil bottle-f055ed448bb10e7e", "tray2-852114c78033e189", "big tray-cd8e8e454bdd8399", "plastic lid-1e9911428db39782", "pile of three trays-d980e89e88480a86", "two towels-b526393ed7990507", "towel3-2eca46eb45f98aef", "towel2-e28980389a3983a5", "bowl2-a7f10388cbbe891d", "box of foil paper-47353281413f77fe", "garlic presser-c5754e05ed234795", "towel-3e765153ebbf8e8f", "washing up liquid bottle-21b2c5ec30f52e2d", "jar of chocolate-9ca7e56a97201f9f", "yogurt-4ca4015083cb1c5c", "spoon2-e806bb70c4afee5f", "three broken pieces of carrot-0daa4ce0c0eb3175", "large chopping board-42a15da8fcb4c56f", "scale-d8e077019d84450d", "biscuit packet-2b5427f5280c5c79", "box of parchment paper-752cf60b10869373", "plastic spoon-304f4bdfb98a2fbc", "box of foil-33caa5d34fdb8406", "can opener-e10b7ce5b4bdaab1", "tray-d103ce70421e1b3a", "collander-058f36ffab7bf054", "things-5c0bc499982e93c4", "cocoa can lid-e9c2929744417e9a", "can of cocoa-97c15b1ab1811716", "small cup of glass-84e936d047991ae0", "small bowl of glass-d61bb7cc9e349b08", "spoon-a4fc0f681c68164e", "can of condensed milk-8bfb182b085406e6", "butter knife-84a85666d0244e97", "notepad-00e192a8b6e8c541", "pen-c70015127cd3d8b8", "plastic part-c6c1c20faae74969", "peeler-7776de9417366fdc", "plastic cover-9a33e9ccd2b027ed", "plastic tupperware-87625abfc216477e", "metal pot-ea599bb07a3e5ca8", "butter-b3d3296c54f0f89b", "wooden spoon-f5a9ca7515580d65", "top smaller pot-b43849214ce7bce0", "pot2-852a8e528ba115f0", "pot-328ab9c9eed4d5ed", "pan lid-f86d3b28227aa40f", "spatula-4c1069565d3ee9fb", "large bowl-235c137c0a6ccd8f", "glass-a902246ff484d129", "food processor bowl-634a88fe4f3081c7", "blade-7d7926605b41ce2d", "center plastic piece-d65bf7beceed6184", "food processor lid-198194792104bb6c" ], "reasoning": "1. I watched until the user began handling the butter (butter-b3d3296c54f0f89b), which began at 03:58. The user cuts the butter (butter-b3d3296c54f0f89b) with a butter knife (butter knife-84a85666d0244e97) at 04:08 into two, unevenly-sized chunks.\n2. At 04:15, the largest chunk of butter , prodded by the user, peels off the wrapping and drops into the mixing pot (pot2-852a8e528ba115f0).\n3. I track everything that happens to this chunk of butter till 04:35.\n4. Nothing happens to it until 04:24, when the user picks up the mixing pot (pot2-852a8e528ba115f0) and, at 04:26, places it on the stovetop.\n5. Nothing else happens to the butter by 04:35.\n5. Therefore, the largest chunk of butter moved from the countertop to the stovetop.", "referenced_object_masks": [ "butter knife-84a85666d0244e97", "butter-b3d3296c54f0f89b", "pot2-852a8e528ba115f0" ] }, { "key": "P01-20240203-135502:54dc05c9f65d4422ce5cc103cad1e0a3e5707a25", "video_id": "P01-20240203-135502", "question": "At the beginning of the video, were I to walk from the sink to the refrigerator, what are the first three objects, in order, that I would pass on the left hand side?", "answer_choice_0": "Box of coffee pods, coffee machine, milk frother", "answer_choice_1": "Compost bin, bag of sugar, food processor", "answer_choice_2": "Oil bottle, box with blender, kettle", "answer_choice_3": "Food processor, bag of sugar, compost bin", "answer_choice_4": "Kettle, box with a blender, oil bottle", "answer_id": 1, "answer": "Compost bin, bag of sugar, food processor", "question_type": [ "Egocentric & Spatial Perception", "Counterfactual", "Object Recognition", "Situational Awareness", "Temporal Reasoning" ], "objects_of_interest": [ "spoon6-56d1bd6fa3efbff6", "spoon5-e9b7623ef30f0481", "ladle-2e157ee745fb2769", "bowl3-17ad8761bbf76b51", "knife-98ff5c4d9609a313", "strainer-c29787451611cf9e", "chopping board-8bfb8696e3f09575", "bottle of vegetable oil-945822ab78f564dd", "opened bag of coriander-724ebd15c9c593f4", "towel4-bca834db4478e9df", "teaspoon-ebd18f71ffcc59e5", "jar of spice-2a375bfc9fb9cefa", "blender's piece-82cf565fcf91e56a", "blender-2cbf7e23b8778dc1", "box-e2e5b817c182d16d", "milk frother-3976a1d220a647d2", "plate-2a3d260658b7297a", "spoon4-8740eb7f6177f08b", "jar of sumac spice-55fdab7d47bd5cdb", "spoon3-c0448a4e5347a656", "jar of tahini lid-b9a0a5ea12514273", "jar of tahini-5af78fda7152b5a8", "pan lid2-1ab40216c2e0231c", "pan-dd1e2a31005ddc4f", "olive oil bottle-f055ed448bb10e7e", "tray2-852114c78033e189", "big tray-cd8e8e454bdd8399", "plastic lid-1e9911428db39782", "pile of three trays-d980e89e88480a86", "two towels-b526393ed7990507", "towel3-2eca46eb45f98aef", "towel2-e28980389a3983a5", "bowl2-a7f10388cbbe891d", "box of foil paper-47353281413f77fe", "garlic presser-c5754e05ed234795", "towel-3e765153ebbf8e8f", "washing up liquid bottle-21b2c5ec30f52e2d", "jar of chocolate-9ca7e56a97201f9f", "yogurt-4ca4015083cb1c5c", "spoon2-e806bb70c4afee5f", "three broken pieces of carrot-0daa4ce0c0eb3175", "large chopping board-42a15da8fcb4c56f", "scale-d8e077019d84450d", "biscuit packet-2b5427f5280c5c79", "box of parchment paper-752cf60b10869373", "plastic spoon-304f4bdfb98a2fbc", "box of foil-33caa5d34fdb8406", "can opener-e10b7ce5b4bdaab1", "tray-d103ce70421e1b3a", "collander-058f36ffab7bf054", "things-5c0bc499982e93c4", "cocoa can lid-e9c2929744417e9a", "can of cocoa-97c15b1ab1811716", "small cup of glass-84e936d047991ae0", "small bowl of glass-d61bb7cc9e349b08", "spoon-a4fc0f681c68164e", "can of condensed milk-8bfb182b085406e6", "butter knife-84a85666d0244e97", "notepad-00e192a8b6e8c541", "pen-c70015127cd3d8b8", "plastic part-c6c1c20faae74969", "peeler-7776de9417366fdc", "plastic cover-9a33e9ccd2b027ed", "plastic tupperware-87625abfc216477e", "metal pot-ea599bb07a3e5ca8", "butter-b3d3296c54f0f89b", "wooden spoon-f5a9ca7515580d65", "top smaller pot-b43849214ce7bce0", "pot2-852a8e528ba115f0", "pot-328ab9c9eed4d5ed", "pan lid-f86d3b28227aa40f", "spatula-4c1069565d3ee9fb", "large bowl-235c137c0a6ccd8f", "glass-a902246ff484d129", "food processor bowl-634a88fe4f3081c7", "blade-7d7926605b41ce2d", "center plastic piece-d65bf7beceed6184", "food processor lid-198194792104bb6c" ], "reasoning": "1. I look at the beginning of the video from 00:00 until about 00:06, when I get a good view of the countertop by the sink . In this segment, the user travels past the fridge at 00:04 and toward a food processor bowl (food processor bowl-634a88fe4f3081c7) with its lid (food processor lid-198194792104bb6c). This appliance is near the sink .\n2. I see 3 objects right away: a compost bin , a red paper bag of sugar , and a food processor bowl (food processor bowl-634a88fe4f3081c7) with its lid (food processor lid-198194792104bb6c).\n3. I realize that, were I to turn around from the sink and face away from it toward the refrigerator , these 3 items would be on my left.\n4. In order, the objects I would pass would be the compost bin , the bag of sugar , and the food processor bowl (food processor bowl-634a88fe4f3081c7) with its lid (food processor lid-198194792104bb6c).", "referenced_object_masks": [ "food processor lid-198194792104bb6c", "food processor bowl-634a88fe4f3081c7" ] }, { "key": "P01-20240203-135502:7ae317cfca426176c29463282ae2704b5765e00b", "video_id": "P01-20240203-135502", "question": "What item did I not put back into the drawer while searching for the can opener?", "answer_choice_0": "Box of foil", "answer_choice_1": ", colander", "answer_choice_2": "Tray", "answer_choice_3": "Plastic spoon", "answer_choice_4": ", Parchment paper", "answer_id": 0, "answer": "Box of foil", "question_type": [ "Action Recognition", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "spoon6-56d1bd6fa3efbff6", "spoon5-e9b7623ef30f0481", "ladle-2e157ee745fb2769", "bowl3-17ad8761bbf76b51", "knife-98ff5c4d9609a313", "strainer-c29787451611cf9e", "chopping board-8bfb8696e3f09575", "bottle of vegetable oil-945822ab78f564dd", "opened bag of coriander-724ebd15c9c593f4", "towel4-bca834db4478e9df", "teaspoon-ebd18f71ffcc59e5", "jar of spice-2a375bfc9fb9cefa", "blender's piece-82cf565fcf91e56a", "blender-2cbf7e23b8778dc1", "box-e2e5b817c182d16d", "milk frother-3976a1d220a647d2", "plate-2a3d260658b7297a", "spoon4-8740eb7f6177f08b", "jar of sumac spice-55fdab7d47bd5cdb", "spoon3-c0448a4e5347a656", "jar of tahini lid-b9a0a5ea12514273", "jar of tahini-5af78fda7152b5a8", "pan lid2-1ab40216c2e0231c", "pan-dd1e2a31005ddc4f", "olive oil bottle-f055ed448bb10e7e", "tray2-852114c78033e189", "big tray-cd8e8e454bdd8399", "plastic lid-1e9911428db39782", "pile of three trays-d980e89e88480a86", "two towels-b526393ed7990507", "towel3-2eca46eb45f98aef", "towel2-e28980389a3983a5", "bowl2-a7f10388cbbe891d", "box of foil paper-47353281413f77fe", "garlic presser-c5754e05ed234795", "towel-3e765153ebbf8e8f", "washing up liquid bottle-21b2c5ec30f52e2d", "jar of chocolate-9ca7e56a97201f9f", "yogurt-4ca4015083cb1c5c", "spoon2-e806bb70c4afee5f", "three broken pieces of carrot-0daa4ce0c0eb3175", "large chopping board-42a15da8fcb4c56f", "scale-d8e077019d84450d", "biscuit packet-2b5427f5280c5c79", "box of parchment paper-752cf60b10869373", "plastic spoon-304f4bdfb98a2fbc", "box of foil-33caa5d34fdb8406", "can opener-e10b7ce5b4bdaab1", "tray-d103ce70421e1b3a", "collander-058f36ffab7bf054", "things-5c0bc499982e93c4", "cocoa can lid-e9c2929744417e9a", "can of cocoa-97c15b1ab1811716", "small cup of glass-84e936d047991ae0", "small bowl of glass-d61bb7cc9e349b08", "spoon-a4fc0f681c68164e", "can of condensed milk-8bfb182b085406e6", "butter knife-84a85666d0244e97", "notepad-00e192a8b6e8c541", "pen-c70015127cd3d8b8", "plastic part-c6c1c20faae74969", "peeler-7776de9417366fdc", "plastic cover-9a33e9ccd2b027ed", "plastic tupperware-87625abfc216477e", "metal pot-ea599bb07a3e5ca8", "butter-b3d3296c54f0f89b", "wooden spoon-f5a9ca7515580d65", "top smaller pot-b43849214ce7bce0", "pot2-852a8e528ba115f0", "pot-328ab9c9eed4d5ed", "pan lid-f86d3b28227aa40f", "spatula-4c1069565d3ee9fb", "large bowl-235c137c0a6ccd8f", "glass-a902246ff484d129", "food processor bowl-634a88fe4f3081c7", "blade-7d7926605b41ce2d", "center plastic piece-d65bf7beceed6184", "food processor lid-198194792104bb6c" ], "reasoning": "1. The user began looking for the can opener (can opener-e10b7ce5b4bdaab1) at 05:32 when they opened the drawers.\n2. The user took out the tray (tray-d103ce70421e1b3a) and colander (colander-058f36ffab7bf054) at 05:46 but put it back at 05:42.\n3. The user took out the plastic spoon at 05:51 but put it back at 05:57.\n4. The user put the box of parchment paper (box of parchment paper-752cf60b10869373) back in the drawer at 05:59.\n5. The user took out the box of foil (box of foil-33caa5d34fdb8406) at 05:54 and did not put it back into the drawer when they closed it at 06:02.", "referenced_object_masks": [ "box of foil-33caa5d34fdb8406", "tray-d103ce70421e1b3a", "box of parchment paper-752cf60b10869373", "can opener-e10b7ce5b4bdaab1", "colander-058f36ffab7bf054" ] }, { "key": "P01-20240203-135502:b886df8a9e25abf96d25a0b430cc925815201c17", "video_id": "P01-20240203-135502", "question": "How many times did I shake both pieces of the food processor lid before putting it down?", "answer_choice_0": "5 times", "answer_choice_1": "9 times", "answer_choice_2": "10 times", "answer_choice_3": "14 times", "answer_choice_4": "25 times", "answer_id": 3, "answer": "14 times", "question_type": [ "Counting", "Object Recognition", "Action Recognition", "Numerical Reasoning", "Audio Reasoning" ], "objects_of_interest": [ "spoon6-56d1bd6fa3efbff6", "spoon5-e9b7623ef30f0481", "ladle-2e157ee745fb2769", "bowl3-17ad8761bbf76b51", "knife-98ff5c4d9609a313", "strainer-c29787451611cf9e", "chopping board-8bfb8696e3f09575", "bottle of vegetable oil-945822ab78f564dd", "opened bag of coriander-724ebd15c9c593f4", "towel4-bca834db4478e9df", "teaspoon-ebd18f71ffcc59e5", "jar of spice-2a375bfc9fb9cefa", "blender's piece-82cf565fcf91e56a", "blender-2cbf7e23b8778dc1", "box-e2e5b817c182d16d", "milk frother-3976a1d220a647d2", "plate-2a3d260658b7297a", "spoon4-8740eb7f6177f08b", "jar of sumac spice-55fdab7d47bd5cdb", "spoon3-c0448a4e5347a656", "jar of tahini lid-b9a0a5ea12514273", "jar of tahini-5af78fda7152b5a8", "pan lid2-1ab40216c2e0231c", "pan-dd1e2a31005ddc4f", "olive oil bottle-f055ed448bb10e7e", "tray2-852114c78033e189", "big tray-cd8e8e454bdd8399", "plastic lid-1e9911428db39782", "pile of three trays-d980e89e88480a86", "two towels-b526393ed7990507", "towel3-2eca46eb45f98aef", "towel2-e28980389a3983a5", "bowl2-a7f10388cbbe891d", "box of foil paper-47353281413f77fe", "garlic presser-c5754e05ed234795", "towel-3e765153ebbf8e8f", "washing up liquid bottle-21b2c5ec30f52e2d", "jar of chocolate-9ca7e56a97201f9f", "yogurt-4ca4015083cb1c5c", "spoon2-e806bb70c4afee5f", "three broken pieces of carrot-0daa4ce0c0eb3175", "large chopping board-42a15da8fcb4c56f", "scale-d8e077019d84450d", "biscuit packet-2b5427f5280c5c79", "box of parchment paper-752cf60b10869373", "plastic spoon-304f4bdfb98a2fbc", "box of foil-33caa5d34fdb8406", "can opener-e10b7ce5b4bdaab1", "tray-d103ce70421e1b3a", "collander-058f36ffab7bf054", "things-5c0bc499982e93c4", "cocoa can lid-e9c2929744417e9a", "can of cocoa-97c15b1ab1811716", "small cup of glass-84e936d047991ae0", "small bowl of glass-d61bb7cc9e349b08", "spoon-a4fc0f681c68164e", "can of condensed milk-8bfb182b085406e6", "butter knife-84a85666d0244e97", "notepad-00e192a8b6e8c541", "pen-c70015127cd3d8b8", "plastic part-c6c1c20faae74969", "peeler-7776de9417366fdc", "plastic cover-9a33e9ccd2b027ed", "plastic tupperware-87625abfc216477e", "metal pot-ea599bb07a3e5ca8", "butter-b3d3296c54f0f89b", "wooden spoon-f5a9ca7515580d65", "top smaller pot-b43849214ce7bce0", "pot2-852a8e528ba115f0", "pot-328ab9c9eed4d5ed", "pan lid-f86d3b28227aa40f", "spatula-4c1069565d3ee9fb", "large bowl-235c137c0a6ccd8f", "glass-a902246ff484d129", "food processor bowl-634a88fe4f3081c7", "blade-7d7926605b41ce2d", "center plastic piece-d65bf7beceed6184", "food processor lid-198194792104bb6c" ], "reasoning": "1. I watched the video to find when the user shook the food processor lid (food processor lid-198194792104bb6c) and saw this happen between 00:08-00:11.\n2. Shortly afterwards, I observed the user shake a plastic piece that's attached to the lid for a brief moment at 00:12.\n3. Other than visually seeing how many shakes are made, a plastic clicking noise is made each time the user shakes these pieces, which serves as a good backup confirmation of how many shakes take place.\n4. Between 00:08-00:11, the user visually and audibly shakes the lid (food processor lid-198194792104bb6c) 11 times.\n5. During 00:12, the user shakes the plastic piece 3 times in quick succession before putting the lid (food processor lid-198194792104bb6c) down.\n6. Adding up the number of times the user shook both pieces comes to 14.", "referenced_object_masks": [ "food processor lid-198194792104bb6c" ] }, { "key": "P01-20240203-135502:bd19987fc032fbdf0c33fc342d4f91d9be3c6a3b", "video_id": "P01-20240203-135502", "question": "How many times did the user move the coriander with their hands when it is in the strainer and near the sink?", "answer_choice_0": "19 times", "answer_choice_1": "35 times", "answer_choice_2": "29 times", "answer_choice_3": "25 times", "answer_choice_4": "31 times", "answer_id": 2, "answer": "29 times", "question_type": [ "Object Recognition", "Action Recognition", "Counting" ], "objects_of_interest": [ "spoon6-56d1bd6fa3efbff6", "spoon5-e9b7623ef30f0481", "ladle-2e157ee745fb2769", "bowl3-17ad8761bbf76b51", "knife-98ff5c4d9609a313", "strainer-c29787451611cf9e", "chopping board-8bfb8696e3f09575", "bottle of vegetable oil-945822ab78f564dd", "opened bag of coriander-724ebd15c9c593f4", "towel4-bca834db4478e9df", "teaspoon-ebd18f71ffcc59e5", "jar of spice-2a375bfc9fb9cefa", "blender's piece-82cf565fcf91e56a", "blender-2cbf7e23b8778dc1", "box-e2e5b817c182d16d", "milk frother-3976a1d220a647d2", "plate-2a3d260658b7297a", "spoon4-8740eb7f6177f08b", "jar of sumac spice-55fdab7d47bd5cdb", "spoon3-c0448a4e5347a656", "jar of tahini lid-b9a0a5ea12514273", "jar of tahini-5af78fda7152b5a8", "pan lid2-1ab40216c2e0231c", "pan-dd1e2a31005ddc4f", "olive oil bottle-f055ed448bb10e7e", "tray2-852114c78033e189", "big tray-cd8e8e454bdd8399", "plastic lid-1e9911428db39782", "pile of three trays-d980e89e88480a86", "two towels-b526393ed7990507", "towel3-2eca46eb45f98aef", "towel2-e28980389a3983a5", "bowl2-a7f10388cbbe891d", "box of foil paper-47353281413f77fe", "garlic presser-c5754e05ed234795", "towel-3e765153ebbf8e8f", "washing up liquid bottle-21b2c5ec30f52e2d", "jar of chocolate-9ca7e56a97201f9f", "yogurt-4ca4015083cb1c5c", "spoon2-e806bb70c4afee5f", "three broken pieces of carrot-0daa4ce0c0eb3175", "large chopping board-42a15da8fcb4c56f", "scale-d8e077019d84450d", "biscuit packet-2b5427f5280c5c79", "box of parchment paper-752cf60b10869373", "plastic spoon-304f4bdfb98a2fbc", "box of foil-33caa5d34fdb8406", "can opener-e10b7ce5b4bdaab1", "tray-d103ce70421e1b3a", "collander-058f36ffab7bf054", "things-5c0bc499982e93c4", "cocoa can lid-e9c2929744417e9a", "can of cocoa-97c15b1ab1811716", "small cup of glass-84e936d047991ae0", "small bowl of glass-d61bb7cc9e349b08", "spoon-a4fc0f681c68164e", "can of condensed milk-8bfb182b085406e6", "butter knife-84a85666d0244e97", "notepad-00e192a8b6e8c541", "pen-c70015127cd3d8b8", "plastic part-c6c1c20faae74969", "peeler-7776de9417366fdc", "plastic cover-9a33e9ccd2b027ed", "plastic tupperware-87625abfc216477e", "metal pot-ea599bb07a3e5ca8", "butter-b3d3296c54f0f89b", "wooden spoon-f5a9ca7515580d65", "top smaller pot-b43849214ce7bce0", "pot2-852a8e528ba115f0", "pot-328ab9c9eed4d5ed", "pan lid-f86d3b28227aa40f", "spatula-4c1069565d3ee9fb", "large bowl-235c137c0a6ccd8f", "glass-a902246ff484d129", "food processor bowl-634a88fe4f3081c7", "blade-7d7926605b41ce2d", "center plastic piece-d65bf7beceed6184", "food processor lid-198194792104bb6c" ], "reasoning": "1. I watched the video to find when the user moves coriander (opened bag of coriander-724ebd15c9c593f4) with their hands while it is in a strainer (strainer-c29787451611cf9e) and near the sink and saw this happening between 38:28-38:43.\n2. Using the user's thumb movements as the beginning and ending point of a \"move\", between 38:28-38:39, the user moves the coriander (opened bag of coriander-724ebd15c9c593f4) 26 times.\n3. Then, in a separate set of moves involving picking up the coriander (opened bag of coriander-724ebd15c9c593f4) and dropping it back inside the strainer (strainer-c29787451611cf9e), the user moved the coriander (opened bag of coriander-724ebd15c9c593f4) 3 more times between 38:40-38:43.\n4. Therefore, the user moved the coriander (opened bag of coriander-724ebd15c9c593f4) with their hands 29 times when it was in the strainer (strainer-c29787451611cf9e) and near the sink.", "referenced_object_masks": [ "strainer-c29787451611cf9e", "opened bag of coriander-724ebd15c9c593f4" ] }, { "key": "P01-20240203-135502:ccf7d21b42458d4f75dd23065cd7c1296f3a0d24", "video_id": "P01-20240203-135502", "question": "How many trays are behind me on the counter when I close the lid of the oven?", "answer_choice_0": "2", "answer_choice_1": "3", "answer_choice_2": "1", "answer_choice_3": "4", "answer_choice_4": "5", "answer_id": 0, "answer": "2", "question_type": [ "Egocentric & Spatial Perception", "Counting", "Object Recognition" ], "objects_of_interest": [ "spoon6-56d1bd6fa3efbff6", "spoon5-e9b7623ef30f0481", "ladle-2e157ee745fb2769", "bowl3-17ad8761bbf76b51", "knife-98ff5c4d9609a313", "strainer-c29787451611cf9e", "chopping board-8bfb8696e3f09575", "bottle of vegetable oil-945822ab78f564dd", "opened bag of coriander-724ebd15c9c593f4", "towel4-bca834db4478e9df", "teaspoon-ebd18f71ffcc59e5", "jar of spice-2a375bfc9fb9cefa", "blender's piece-82cf565fcf91e56a", "blender-2cbf7e23b8778dc1", "box-e2e5b817c182d16d", "milk frother-3976a1d220a647d2", "plate-2a3d260658b7297a", "spoon4-8740eb7f6177f08b", "jar of sumac spice-55fdab7d47bd5cdb", "spoon3-c0448a4e5347a656", "jar of tahini lid-b9a0a5ea12514273", "jar of tahini-5af78fda7152b5a8", "pan lid2-1ab40216c2e0231c", "pan-dd1e2a31005ddc4f", "olive oil bottle-f055ed448bb10e7e", "tray2-852114c78033e189", "big tray-cd8e8e454bdd8399", "plastic lid-1e9911428db39782", "pile of three trays-d980e89e88480a86", "two towels-b526393ed7990507", "towel3-2eca46eb45f98aef", "towel2-e28980389a3983a5", "bowl2-a7f10388cbbe891d", "box of foil paper-47353281413f77fe", "garlic presser-c5754e05ed234795", "towel-3e765153ebbf8e8f", "washing up liquid bottle-21b2c5ec30f52e2d", "jar of chocolate-9ca7e56a97201f9f", "yogurt-4ca4015083cb1c5c", "spoon2-e806bb70c4afee5f", "three broken pieces of carrot-0daa4ce0c0eb3175", "large chopping board-42a15da8fcb4c56f", "scale-d8e077019d84450d", "biscuit packet-2b5427f5280c5c79", "box of parchment paper-752cf60b10869373", "plastic spoon-304f4bdfb98a2fbc", "box of foil-33caa5d34fdb8406", "can opener-e10b7ce5b4bdaab1", "tray-d103ce70421e1b3a", "collander-058f36ffab7bf054", "things-5c0bc499982e93c4", "cocoa can lid-e9c2929744417e9a", "can of cocoa-97c15b1ab1811716", "small cup of glass-84e936d047991ae0", "small bowl of glass-d61bb7cc9e349b08", "spoon-a4fc0f681c68164e", "can of condensed milk-8bfb182b085406e6", "butter knife-84a85666d0244e97", "notepad-00e192a8b6e8c541", "pen-c70015127cd3d8b8", "plastic part-c6c1c20faae74969", "peeler-7776de9417366fdc", "plastic cover-9a33e9ccd2b027ed", "plastic tupperware-87625abfc216477e", "metal pot-ea599bb07a3e5ca8", "butter-b3d3296c54f0f89b", "wooden spoon-f5a9ca7515580d65", "top smaller pot-b43849214ce7bce0", "pot2-852a8e528ba115f0", "pot-328ab9c9eed4d5ed", "pan lid-f86d3b28227aa40f", "spatula-4c1069565d3ee9fb", "large bowl-235c137c0a6ccd8f", "glass-a902246ff484d129", "food processor bowl-634a88fe4f3081c7", "blade-7d7926605b41ce2d", "center plastic piece-d65bf7beceed6184", "food processor lid-198194792104bb6c" ], "reasoning": "1. I observed that the user left the oven door open at 19:26 when they retrieved the covered bowl (bowl2-a7f10388cbbe891d).\n2. I continued watching and saw that the user closed the oven door at 20:01. I watched the moments before the user closed the door to observe the counter behind them. At 19:56, the user set a big tray (big tray-cd8e8e454bdd8399) on the counter space. This shows that there is at least 1 tray behind the user when they close the oven door .\n3. I continued watching until the user interacted with the big tray (big tray-cd8e8e454bdd8399) after they closed the oven door at 20:01. I noticed, at 20:15, the user placed the big tray (big tray-cd8e8e454bdd8399) on the left and revealed a smaller tray (tray2-852114c78033e189) underneath. \n4. This means that there were 2 stacked trays when the user closed the oven door .", "referenced_object_masks": [ "big tray-cd8e8e454bdd8399", "tray2-852114c78033e189", "bowl2-a7f10388cbbe891d" ] }, { "key": "P01-20240203-135502:dc91d79482f017d066326fda872c591fe2c5cbd2", "video_id": "P01-20240203-135502", "question": "How does the shape of the brown dough mixture in the large white bowl change over time?", "answer_choice_0": "The dough is shapeless at first, then is poured into a mold", "answer_choice_1": "The dough has a shape at first, then melts", "answer_choice_2": "The dough hardens in the bowl", "answer_choice_3": "The dough freezes", "answer_choice_4": "The dough is shapeless at first, then is formed into a cylinder", "answer_id": 4, "answer": "The dough is shapeless at first, then is formed into a cylinder", "question_type": [ "Action Recognition", "Object Recognition", "State Changes" ], "objects_of_interest": [ "spoon6-56d1bd6fa3efbff6", "spoon5-e9b7623ef30f0481", "ladle-2e157ee745fb2769", "bowl3-17ad8761bbf76b51", "knife-98ff5c4d9609a313", "strainer-c29787451611cf9e", "chopping board-8bfb8696e3f09575", "bottle of vegetable oil-945822ab78f564dd", "opened bag of coriander-724ebd15c9c593f4", "towel4-bca834db4478e9df", "teaspoon-ebd18f71ffcc59e5", "jar of spice-2a375bfc9fb9cefa", "blender's piece-82cf565fcf91e56a", "blender-2cbf7e23b8778dc1", "box-e2e5b817c182d16d", "milk frother-3976a1d220a647d2", "plate-2a3d260658b7297a", "spoon4-8740eb7f6177f08b", "jar of sumac spice-55fdab7d47bd5cdb", "spoon3-c0448a4e5347a656", "jar of tahini lid-b9a0a5ea12514273", "jar of tahini-5af78fda7152b5a8", "pan lid2-1ab40216c2e0231c", "pan-dd1e2a31005ddc4f", "olive oil bottle-f055ed448bb10e7e", "tray2-852114c78033e189", "big tray-cd8e8e454bdd8399", "plastic lid-1e9911428db39782", "pile of three trays-d980e89e88480a86", "two towels-b526393ed7990507", "towel3-2eca46eb45f98aef", "towel2-e28980389a3983a5", "bowl2-a7f10388cbbe891d", "box of foil paper-47353281413f77fe", "garlic presser-c5754e05ed234795", "towel-3e765153ebbf8e8f", "washing up liquid bottle-21b2c5ec30f52e2d", "jar of chocolate-9ca7e56a97201f9f", "yogurt-4ca4015083cb1c5c", "spoon2-e806bb70c4afee5f", "three broken pieces of carrot-0daa4ce0c0eb3175", "large chopping board-42a15da8fcb4c56f", "scale-d8e077019d84450d", "biscuit packet-2b5427f5280c5c79", "box of parchment paper-752cf60b10869373", "plastic spoon-304f4bdfb98a2fbc", "box of foil-33caa5d34fdb8406", "can opener-e10b7ce5b4bdaab1", "tray-d103ce70421e1b3a", "collander-058f36ffab7bf054", "things-5c0bc499982e93c4", "cocoa can lid-e9c2929744417e9a", "can of cocoa-97c15b1ab1811716", "small cup of glass-84e936d047991ae0", "small bowl of glass-d61bb7cc9e349b08", "spoon-a4fc0f681c68164e", "can of condensed milk-8bfb182b085406e6", "butter knife-84a85666d0244e97", "notepad-00e192a8b6e8c541", "pen-c70015127cd3d8b8", "plastic part-c6c1c20faae74969", "peeler-7776de9417366fdc", "plastic cover-9a33e9ccd2b027ed", "plastic tupperware-87625abfc216477e", "metal pot-ea599bb07a3e5ca8", "butter-b3d3296c54f0f89b", "wooden spoon-f5a9ca7515580d65", "top smaller pot-b43849214ce7bce0", "pot2-852a8e528ba115f0", "pot-328ab9c9eed4d5ed", "pan lid-f86d3b28227aa40f", "spatula-4c1069565d3ee9fb", "large bowl-235c137c0a6ccd8f", "glass-a902246ff484d129", "food processor bowl-634a88fe4f3081c7", "blade-7d7926605b41ce2d", "center plastic piece-d65bf7beceed6184", "food processor lid-198194792104bb6c" ], "reasoning": "1. I watched the video to find the dough mixture in the large white bowl (large bowl-235c137c0a6ccd8f) and saw the user making the dough between 13:49-16:29.\n2. During this time, the user mixes all of the ingredients together, continuously altering the shape of the dough over and over again until the mixing is finished, at which point it is still a shapeless mass.\n3. I then observed the user pour out the finished mixture onto a cutting board (large chopping board-42a15da8fcb4c56f) at 16:30.\n4. Using their hands, the user then rolls the mixture into a cylinder, its final shape, between 16:31-17:23.", "referenced_object_masks": [ "large bowl-235c137c0a6ccd8f", "large chopping board-42a15da8fcb4c56f" ] }, { "key": "P01-20240203-135502:e16bcbe5c8c29c5dd0b6f9cdf7d6fd263cf1fe11", "video_id": "P01-20240203-135502", "question": "How many times did I move the pot into which I put the butter around the kitchen?", "answer_choice_0": "Once", "answer_choice_1": "4 times", "answer_choice_2": "5 times", "answer_choice_3": "Twice", "answer_choice_4": "Thrice", "answer_id": 1, "answer": "4 times", "question_type": [ "Action Recognition", "Counting", "Egocentric & Spatial Perception", "Object Recognition" ], "objects_of_interest": [ "spoon6-56d1bd6fa3efbff6", "spoon5-e9b7623ef30f0481", "ladle-2e157ee745fb2769", "bowl3-17ad8761bbf76b51", "knife-98ff5c4d9609a313", "strainer-c29787451611cf9e", "chopping board-8bfb8696e3f09575", "bottle of vegetable oil-945822ab78f564dd", "opened bag of coriander-724ebd15c9c593f4", "towel4-bca834db4478e9df", "teaspoon-ebd18f71ffcc59e5", "jar of spice-2a375bfc9fb9cefa", "blender's piece-82cf565fcf91e56a", "blender-2cbf7e23b8778dc1", "box-e2e5b817c182d16d", "milk frother-3976a1d220a647d2", "plate-2a3d260658b7297a", "spoon4-8740eb7f6177f08b", "jar of sumac spice-55fdab7d47bd5cdb", "spoon3-c0448a4e5347a656", "jar of tahini lid-b9a0a5ea12514273", "jar of tahini-5af78fda7152b5a8", "pan lid2-1ab40216c2e0231c", "pan-dd1e2a31005ddc4f", "olive oil bottle-f055ed448bb10e7e", "tray2-852114c78033e189", "big tray-cd8e8e454bdd8399", "plastic lid-1e9911428db39782", "pile of three trays-d980e89e88480a86", "two towels-b526393ed7990507", "towel3-2eca46eb45f98aef", "towel2-e28980389a3983a5", "bowl2-a7f10388cbbe891d", "box of foil paper-47353281413f77fe", "garlic presser-c5754e05ed234795", "towel-3e765153ebbf8e8f", "washing up liquid bottle-21b2c5ec30f52e2d", "jar of chocolate-9ca7e56a97201f9f", "yogurt-4ca4015083cb1c5c", "spoon2-e806bb70c4afee5f", "three broken pieces of carrot-0daa4ce0c0eb3175", "large chopping board-42a15da8fcb4c56f", "scale-d8e077019d84450d", "biscuit packet-2b5427f5280c5c79", "box of parchment paper-752cf60b10869373", "plastic spoon-304f4bdfb98a2fbc", "box of foil-33caa5d34fdb8406", "can opener-e10b7ce5b4bdaab1", "tray-d103ce70421e1b3a", "collander-058f36ffab7bf054", "things-5c0bc499982e93c4", "cocoa can lid-e9c2929744417e9a", "can of cocoa-97c15b1ab1811716", "small cup of glass-84e936d047991ae0", "small bowl of glass-d61bb7cc9e349b08", "spoon-a4fc0f681c68164e", "can of condensed milk-8bfb182b085406e6", "butter knife-84a85666d0244e97", "notepad-00e192a8b6e8c541", "pen-c70015127cd3d8b8", "plastic part-c6c1c20faae74969", "peeler-7776de9417366fdc", "plastic cover-9a33e9ccd2b027ed", "plastic tupperware-87625abfc216477e", "metal pot-ea599bb07a3e5ca8", "butter-b3d3296c54f0f89b", "wooden spoon-f5a9ca7515580d65", "top smaller pot-b43849214ce7bce0", "pot2-852a8e528ba115f0", "pot-328ab9c9eed4d5ed", "pan lid-f86d3b28227aa40f", "spatula-4c1069565d3ee9fb", "large bowl-235c137c0a6ccd8f", "glass-a902246ff484d129", "food processor bowl-634a88fe4f3081c7", "blade-7d7926605b41ce2d", "center plastic piece-d65bf7beceed6184", "food processor lid-198194792104bb6c" ], "reasoning": "1. I watched the video to find the pot that the user put the butter (butter-b3d3296c54f0f89b) into and found this pot (pot2-852a8e528ba115f0) at 04:15.\n2. I then watched the video to see the location the user got the pot (pot2-852a8e528ba115f0) from and saw this happen between 01:40-01:45 when the user grabbed the pot (pot2-852a8e528ba115f0) from a cabinet on one side of the kitchen, then put the pot (pot2-852a8e528ba115f0) on the stove on the opposite side of the kitchen, directly across from the pot's (pot2-852a8e528ba115f0) origin.\n3. I then observed the user moving the pot (pot2-852a8e528ba115f0) to the other side of the kitchen, on the counter, between 03:53-03:54. Here, the user begins to put the butter (butter-b3d3296c54f0f89b) inside the pot (pot2-852a8e528ba115f0) and weigh it on the kitchen scale (scale-d8e077019d84450d).\n4. I then observed the user moving the pot (pot2-852a8e528ba115f0) to the stove again between 04:24-04:26 to cook the butter (butter-b3d3296c54f0f89b).\n5. Finally, after pouring the pot's (pot2-852a8e528ba115f0) melted chocolate into a bowl (large bowl-235c137c0a6ccd8f), I observed the user put the pot (pot2-852a8e528ba115f0) in its final place in the kitchen sink at the far side of the kitchen at 13:46.\n6. Therefore, from the pot's origin in the cabinet across from the stove , the pot (pot2-852a8e528ba115f0) was placed in different locations in the kitchen 4 times.", "referenced_object_masks": [ "scale-d8e077019d84450d", "butter-b3d3296c54f0f89b", "pot2-852a8e528ba115f0", "large bowl-235c137c0a6ccd8f" ] }, { "key": "P01-20240203-150506:057b737634949aac6683aa6e4df880123d0c0c4c", "video_id": "P01-20240203-150506", "question": "What is the ratio of the number of objects I throw into the trash to the number of times I slap my thigh with my right hand while pondering my next move in the kitchen?", "answer_choice_0": "1:2", "answer_choice_1": "1:5", "answer_choice_2": "2:3", "answer_choice_3": "4:3", "answer_choice_4": "5:7", "answer_id": 2, "answer": "2:3", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Counting", "Temporal Reasoning", "Numerical Reasoning", "Object Recognition" ], "objects_of_interest": [ "plastic container-1cc811bfb9e60750", "wire-7975f3f78980814d", "can of yeast-42cbffffa3909067", "plate blue-2390888a948673e4", "spoon3-606f06e6b7fdc7b6", "pot2-4909febf1a4b2a49", "kitchen towel-f285847b713c33c5", "lid-234dfde80c01fc6a", "empty plastic container-ca0be88e2fffdcc9", "lid of bottle of olive oil-fc8668b6fc2f31b2", "bottle of olive oil-1ac533f74f831ff9", "package of biscuits-43065fdddcadea2f", "plug-5ce3cd051076d16c", "food processor motor-ccd334fff62b1e8e", "cloth-47065b0d60656545", "centerpiece-dd64f49e600be31b", "another plastic piece-297f616ee258d7ca", "blade-c020687e1f80cf34", "plastic piece-559bf8f8eac3d9dd", "mesh-c5e6e94a84c9a191", "tupperware3-d130426e195f5091", "bottle of dishwasher rinser-fab323d4012ee51b", "one tablet-03047efe522bc2c7", "bag of dishwasher tablets-652811e34ff005b3", "glass-4e2a28ac36a96548", "spoon-2512a02e50d677b6", "second spoon-a6fa2c62ff9c2af9", "bowl3-33a66668776cca12", "tupperware2-6004353194130cb3", "plate-0ee2c1c3d229f015", "one bowl-7320bee2148c8850", "chopping board-cca33e4b15ef9e95", "pot-8cd44a185441bd60", "bowl2-9b816d15bca4cc17", "spatula-7a6b109b11ee8eed", "wooden spoon2-8d17389bdc4fdd03", "wooden spoon-ff3dd75e6f11fc72", "bowl-af4ba11ded87ab6d", "knife-9efa2d21c6b59347", "fifth spoon-681111d357a3a4d8", "fourth spoon-78259b5406dd6a5c", "third spoon-12a6c9ef6bd4614a", "another spoon-329d158ecc9de849", "one spoon-59ff062749141a88", "washing up liquid bottle-1069bf7f3ca6493a", "sponge-bf003ca7db6253e9", "blender's piece-61608b71067a3e0d", "pan-460c899ccec6f357", "left glove-76ed7ef3a9920bbe", "plastic tupperware-cd5acf5d9581b903", "right glove-3a333795ed395fa2", "box-7d56ff50afaff34a", "bag of flour-cfbe1eafe04da172", "food processor bowl-d101c1f3d14cba50", "bag of coriander-fe8fd83b0507286b", "second tray-362a850b28fe1b25", "first tray of pies-59af83c284adc72e", "ladle-a3aa5ddead44c15d", "empty bowl-a070b9a0f9c63725", "kitchen towel piece-f6e1176aad6737ca" ], "reasoning": "1. I notice the user throw items into the trash bin a couple times in the video.\n2. From 06:52-06:53, the user throws a bottle of dishwashing fluid (bottle of dishwasher rinser-fab323d4012ee51b) into the trash . Additionally, at 10:49, the user also throws a scrap of paper towel (kitchen towel piece-f6e1176aad6737ca) into the trash .\n3. This makes for a total of 2 items thrown in the trash .\n4. I also notice, from 10:33-10:35, the user slap their thigh with their right hand while deciding what next to do in the kitchen.\n5. I counted the user slapping their thigh with their right hand 3 times.\n6. The ratio of items thrown into the trash to the number of slaps is 2:3.\n7: The answer, therefore, is 2:3.", "referenced_object_masks": [ "bottle of dishwasher rinser-fab323d4012ee51b", "kitchen towel piece-f6e1176aad6737ca" ] }, { "key": "P01-20240203-150506:45a1f15ad98fd1023d59f504e996896aed11ef83", "video_id": "P01-20240203-150506", "question": "If I sliced each orange that I had on the counter into quarters, would I be able to place a slice on each pie that is put in the oven?", "answer_choice_0": "Yes, there would be more oranges slices than pies", "answer_choice_1": "Yes, the number of orange slices and pies are equal", "answer_choice_2": "There are no pies", "answer_choice_3": "No, there would be less orange slices than pies", "answer_choice_4": "There are no oranges", "answer_id": 3, "answer": "No, there would be less orange slices than pies", "question_type": [ "Object Recognition", "Counterfactual", "Counting", "Numerical Reasoning" ], "objects_of_interest": [ "plastic container-1cc811bfb9e60750", "wire-7975f3f78980814d", "can of yeast-42cbffffa3909067", "plate blue-2390888a948673e4", "spoon3-606f06e6b7fdc7b6", "pot2-4909febf1a4b2a49", "kitchen towel-f285847b713c33c5", "lid-234dfde80c01fc6a", "empty plastic container-ca0be88e2fffdcc9", "lid of bottle of olive oil-fc8668b6fc2f31b2", "bottle of olive oil-1ac533f74f831ff9", "package of biscuits-43065fdddcadea2f", "plug-5ce3cd051076d16c", "food processor motor-ccd334fff62b1e8e", "cloth-47065b0d60656545", "centerpiece-dd64f49e600be31b", "another plastic piece-297f616ee258d7ca", "blade-c020687e1f80cf34", "plastic piece-559bf8f8eac3d9dd", "mesh-c5e6e94a84c9a191", "tupperware3-d130426e195f5091", "bottle of dishwasher rinser-fab323d4012ee51b", "one tablet-03047efe522bc2c7", "bag of dishwasher tablets-652811e34ff005b3", "glass-4e2a28ac36a96548", "spoon-2512a02e50d677b6", "second spoon-a6fa2c62ff9c2af9", "bowl3-33a66668776cca12", "tupperware2-6004353194130cb3", "plate-0ee2c1c3d229f015", "one bowl-7320bee2148c8850", "chopping board-cca33e4b15ef9e95", "pot-8cd44a185441bd60", "bowl2-9b816d15bca4cc17", "spatula-7a6b109b11ee8eed", "wooden spoon2-8d17389bdc4fdd03", "wooden spoon-ff3dd75e6f11fc72", "bowl-af4ba11ded87ab6d", "knife-9efa2d21c6b59347", "fifth spoon-681111d357a3a4d8", "fourth spoon-78259b5406dd6a5c", "third spoon-12a6c9ef6bd4614a", "another spoon-329d158ecc9de849", "one spoon-59ff062749141a88", "washing up liquid bottle-1069bf7f3ca6493a", "sponge-bf003ca7db6253e9", "blender's piece-61608b71067a3e0d", "pan-460c899ccec6f357", "left glove-76ed7ef3a9920bbe", "plastic tupperware-cd5acf5d9581b903", "right glove-3a333795ed395fa2", "box-7d56ff50afaff34a", "bag of flour-cfbe1eafe04da172", "food processor bowl-d101c1f3d14cba50", "bag of coriander-fe8fd83b0507286b", "second tray-362a850b28fe1b25", "first tray of pies-59af83c284adc72e", "ladle-a3aa5ddead44c15d", "empty bowl-a070b9a0f9c63725", "kitchen towel piece-f6e1176aad6737ca" ], "reasoning": "1. I look for how many oranges there are on the countertop.\n2. A good view of the oranges in a mesh bag (mesh-c5e6e94a84c9a191) shows up at 07:00. I count 2 oranges .\n3. If I cut 2 oranges into quarters, I would get 8 orange slices, because 2*4=8.\n4. I look for the number of pies put in the oven .\n5. From 00:18-00:20, I see the user put a tray of pies (first tray of pies-59af83c284adc72e) in the oven .\n6. On this tray (first tray of pies-59af83c284adc72e), I count 7 pies . \n7. From 00:26-00:27, I see the user put another tray of pies (second tray-362a850b28fe1b25) into the oven .\n8. On this tray (second tray-362a850b28fe1b25), I count 8 pies . \n9. I add 7 to 8 and I get 15 pies.\n10. There aren't enough orange slices to put on each pie put in the oven , because 8<15.\n11. The answer therefore, is no, there would be less orange slices than pies.", "referenced_object_masks": [ "mesh-c5e6e94a84c9a191", "second tray-362a850b28fe1b25", "first tray of pies-59af83c284adc72e" ] }, { "key": "P01-20240203-150506:97321c35f69258801af5ecc1f15d1a4c06a8f41f", "video_id": "P01-20240203-150506", "question": "From 01:29-06:01, how many times do I turn to the dishwasher?", "answer_choice_0": "4 times", "answer_choice_1": "15 times", "answer_choice_2": "8 times", "answer_choice_3": "5 times", "answer_choice_4": "12 times", "answer_id": 4, "answer": "12 times", "question_type": [ "Egocentric & Spatial Perception", "Counting" ], "objects_of_interest": [ "plastic container-1cc811bfb9e60750", "wire-7975f3f78980814d", "can of yeast-42cbffffa3909067", "plate blue-2390888a948673e4", "spoon3-606f06e6b7fdc7b6", "pot2-4909febf1a4b2a49", "kitchen towel-f285847b713c33c5", "lid-234dfde80c01fc6a", "empty plastic container-ca0be88e2fffdcc9", "lid of bottle of olive oil-fc8668b6fc2f31b2", "bottle of olive oil-1ac533f74f831ff9", "package of biscuits-43065fdddcadea2f", "plug-5ce3cd051076d16c", "food processor motor-ccd334fff62b1e8e", "cloth-47065b0d60656545", "centerpiece-dd64f49e600be31b", "another plastic piece-297f616ee258d7ca", "blade-c020687e1f80cf34", "plastic piece-559bf8f8eac3d9dd", "mesh-c5e6e94a84c9a191", "tupperware3-d130426e195f5091", "bottle of dishwasher rinser-fab323d4012ee51b", "one tablet-03047efe522bc2c7", "bag of dishwasher tablets-652811e34ff005b3", "glass-4e2a28ac36a96548", "spoon-2512a02e50d677b6", "second spoon-a6fa2c62ff9c2af9", "bowl3-33a66668776cca12", "tupperware2-6004353194130cb3", "plate-0ee2c1c3d229f015", "one bowl-7320bee2148c8850", "chopping board-cca33e4b15ef9e95", "pot-8cd44a185441bd60", "bowl2-9b816d15bca4cc17", "spatula-7a6b109b11ee8eed", "wooden spoon2-8d17389bdc4fdd03", "wooden spoon-ff3dd75e6f11fc72", "bowl-af4ba11ded87ab6d", "knife-9efa2d21c6b59347", "fifth spoon-681111d357a3a4d8", "fourth spoon-78259b5406dd6a5c", "third spoon-12a6c9ef6bd4614a", "another spoon-329d158ecc9de849", "one spoon-59ff062749141a88", "washing up liquid bottle-1069bf7f3ca6493a", "sponge-bf003ca7db6253e9", "blender's piece-61608b71067a3e0d", "pan-460c899ccec6f357", "left glove-76ed7ef3a9920bbe", "plastic tupperware-cd5acf5d9581b903", "right glove-3a333795ed395fa2", "box-7d56ff50afaff34a", "bag of flour-cfbe1eafe04da172", "food processor bowl-d101c1f3d14cba50", "bag of coriander-fe8fd83b0507286b", "second tray-362a850b28fe1b25", "first tray of pies-59af83c284adc72e", "ladle-a3aa5ddead44c15d", "empty bowl-a070b9a0f9c63725", "kitchen towel piece-f6e1176aad6737ca" ], "reasoning": "1. I look from 01:29-06:01 to see how many times the user turns to the dishwasher .\n2. In this segment, the user washes dishes, then turns back left to place the dishes into the dishwasher .\n3. The user turns to the dishwasher at 02:06, from 02:26-02:27, 02:50-02:51, 03:06-03:07, 03:24, 03:34-03:35, 03:41-03:42, 03:56, 04:39-04:40, 04:48-04:49, 05:37-05:38, and from 05:47-05:49.\n4. The user doesn't turn to the dishwasher again by the time I get to 06:01.\n5. Adding up all these times, the user turns 12 times to the dishwasher .\n6. Therefore, the answer is 12 times.", "referenced_object_masks": [] }, { "key": "P01-20240203-150506:bc4af1f70e4cba610e589abe83fe8b6dedd5edfa", "video_id": "P01-20240203-150506", "question": "What do the first and last objects put into the dishwasher have in common?", "answer_choice_0": "They are both utensils", "answer_choice_1": "They are both silver", "answer_choice_2": "They are both used in eating cereal", "answer_choice_3": "They are both plates", "answer_choice_4": "They are both used to propose a toast", "answer_id": 4, "answer": "They are both used to propose a toast", "question_type": [ "Object Recognition", "Goal Reasoning", "Counting" ], "objects_of_interest": [ "plastic container-1cc811bfb9e60750", "wire-7975f3f78980814d", "can of yeast-42cbffffa3909067", "plate blue-2390888a948673e4", "spoon3-606f06e6b7fdc7b6", "pot2-4909febf1a4b2a49", "kitchen towel-f285847b713c33c5", "lid-234dfde80c01fc6a", "empty plastic container-ca0be88e2fffdcc9", "lid of bottle of olive oil-fc8668b6fc2f31b2", "bottle of olive oil-1ac533f74f831ff9", "package of biscuits-43065fdddcadea2f", "plug-5ce3cd051076d16c", "food processor motor-ccd334fff62b1e8e", "cloth-47065b0d60656545", "centerpiece-dd64f49e600be31b", "another plastic piece-297f616ee258d7ca", "blade-c020687e1f80cf34", "plastic piece-559bf8f8eac3d9dd", "mesh-c5e6e94a84c9a191", "tupperware3-d130426e195f5091", "bottle of dishwasher rinser-fab323d4012ee51b", "one tablet-03047efe522bc2c7", "bag of dishwasher tablets-652811e34ff005b3", "glass-4e2a28ac36a96548", "spoon-2512a02e50d677b6", "second spoon-a6fa2c62ff9c2af9", "bowl3-33a66668776cca12", "tupperware2-6004353194130cb3", "plate-0ee2c1c3d229f015", "one bowl-7320bee2148c8850", "chopping board-cca33e4b15ef9e95", "pot-8cd44a185441bd60", "bowl2-9b816d15bca4cc17", "spatula-7a6b109b11ee8eed", "wooden spoon2-8d17389bdc4fdd03", "wooden spoon-ff3dd75e6f11fc72", "bowl-af4ba11ded87ab6d", "knife-9efa2d21c6b59347", "fifth spoon-681111d357a3a4d8", "fourth spoon-78259b5406dd6a5c", "third spoon-12a6c9ef6bd4614a", "another spoon-329d158ecc9de849", "one spoon-59ff062749141a88", "washing up liquid bottle-1069bf7f3ca6493a", "sponge-bf003ca7db6253e9", "blender's piece-61608b71067a3e0d", "pan-460c899ccec6f357", "left glove-76ed7ef3a9920bbe", "plastic tupperware-cd5acf5d9581b903", "right glove-3a333795ed395fa2", "box-7d56ff50afaff34a", "bag of flour-cfbe1eafe04da172", "food processor bowl-d101c1f3d14cba50", "bag of coriander-fe8fd83b0507286b", "second tray-362a850b28fe1b25", "first tray of pies-59af83c284adc72e", "ladle-a3aa5ddead44c15d", "empty bowl-a070b9a0f9c63725", "kitchen towel piece-f6e1176aad6737ca" ], "reasoning": "1. I notice the user putting their first utensil (knife-9efa2d21c6b59347) in the dishwasher at 02:10.\n2. I see that it is a silver knife (knife-9efa2d21c6b59347).\n3. I notice the user washing and placing objects in the dishwasher from 02:07-06:01, and from 06:16-06:49, I notice the user loading a soap tablet (one tablet-03047efe522bc2c7), filling it with soap from a bottle (bottle of dishwasher rinser-fab323d4012ee51b), and closing the dishwasher . After 06:01, therefore, no more objects are loaded into the dishwasher .\n4. At 06:01, I notice the user place a transparent drinking glass (glass-4e2a28ac36a96548) in the dishwasher .\n5. A knife (knife-9efa2d21c6b59347) and a drinking glass (glass-4e2a28ac36a96548) are not both plates.\n6. The knife (knife-9efa2d21c6b59347) and the drinking glass (glass-4e2a28ac36a96548) are not both silver.\n7. The knife (knife-9efa2d21c6b59347) and the drinking glass (glass-4e2a28ac36a96548) can be used to propose a toast.\n8. The knife (knife-9efa2d21c6b59347) and the drinking glass (glass-4e2a28ac36a96548) are not both utensils.\n9. The knife (knife-9efa2d21c6b59347) and the drinking glass (glass-4e2a28ac36a96548) are not used in eating cereal.\n10. Therefore, the answer is that they are both used to propose a toast.", "referenced_object_masks": [ "knife-9efa2d21c6b59347", "glass-4e2a28ac36a96548", "bottle of dishwasher rinser-fab323d4012ee51b", "one tablet-03047efe522bc2c7" ] }, { "key": "P01-20240203-150506:fbae797d10dae5c6f3bc74d1be8b51022ec0ad20", "video_id": "P01-20240203-150506", "question": "How many times do I turn the faucet on to clean the second kitchen vessel I put in the dishwasher?", "answer_choice_0": "3 times", "answer_choice_1": "5 times", "answer_choice_2": "2 times", "answer_choice_3": "1 time", "answer_choice_4": "4 times", "answer_id": 0, "answer": "3 times", "question_type": [ "Counting", "Temporal Reasoning", "Action Recognition", "Egocentric & Spatial Perception", "Object Recognition" ], "objects_of_interest": [ "plastic container-1cc811bfb9e60750", "wire-7975f3f78980814d", "can of yeast-42cbffffa3909067", "plate blue-2390888a948673e4", "spoon3-606f06e6b7fdc7b6", "pot2-4909febf1a4b2a49", "kitchen towel-f285847b713c33c5", "lid-234dfde80c01fc6a", "empty plastic container-ca0be88e2fffdcc9", "lid of bottle of olive oil-fc8668b6fc2f31b2", "bottle of olive oil-1ac533f74f831ff9", "package of biscuits-43065fdddcadea2f", "plug-5ce3cd051076d16c", "food processor motor-ccd334fff62b1e8e", "cloth-47065b0d60656545", "centerpiece-dd64f49e600be31b", "another plastic piece-297f616ee258d7ca", "blade-c020687e1f80cf34", "plastic piece-559bf8f8eac3d9dd", "mesh-c5e6e94a84c9a191", "tupperware3-d130426e195f5091", "bottle of dishwasher rinser-fab323d4012ee51b", "one tablet-03047efe522bc2c7", "bag of dishwasher tablets-652811e34ff005b3", "glass-4e2a28ac36a96548", "spoon-2512a02e50d677b6", "second spoon-a6fa2c62ff9c2af9", "bowl3-33a66668776cca12", "tupperware2-6004353194130cb3", "plate-0ee2c1c3d229f015", "one bowl-7320bee2148c8850", "chopping board-cca33e4b15ef9e95", "pot-8cd44a185441bd60", "bowl2-9b816d15bca4cc17", "spatula-7a6b109b11ee8eed", "wooden spoon2-8d17389bdc4fdd03", "wooden spoon-ff3dd75e6f11fc72", "bowl-af4ba11ded87ab6d", "knife-9efa2d21c6b59347", "fifth spoon-681111d357a3a4d8", "fourth spoon-78259b5406dd6a5c", "third spoon-12a6c9ef6bd4614a", "another spoon-329d158ecc9de849", "one spoon-59ff062749141a88", "washing up liquid bottle-1069bf7f3ca6493a", "sponge-bf003ca7db6253e9", "blender's piece-61608b71067a3e0d", "pan-460c899ccec6f357", "left glove-76ed7ef3a9920bbe", "plastic tupperware-cd5acf5d9581b903", "right glove-3a333795ed395fa2", "box-7d56ff50afaff34a", "bag of flour-cfbe1eafe04da172", "food processor bowl-d101c1f3d14cba50", "bag of coriander-fe8fd83b0507286b", "second tray-362a850b28fe1b25", "first tray of pies-59af83c284adc72e", "ladle-a3aa5ddead44c15d", "empty bowl-a070b9a0f9c63725", "kitchen towel piece-f6e1176aad6737ca" ], "reasoning": "1. I look for a time the user cleans the dishes, specifically looking for the second kitchen vessel to be put in the dishwasher .\n2. From 02:21-02:25, the user washes a bowl (empty bowl-a070b9a0f9c63725). This is the first kitchen vessel washed.\n3. From 02:27-02:29, the user puts this bowl (empty bowl-a070b9a0f9c63725) into the dishwasher, making this the first kitchen vessel the user puts into the dishwasher.\n4. From 02:31-02:50, the user washes a pan (pan-460c899ccec6f357). This is the second kitchen vessel washed.\n5. The user turns on the faucet 3 separate times while cleaning the pan (pan-460c899ccec6f357): once at 02:36, another time at 02:44, and a final time at 02:48.\n6. From 02:51-02:59, the user places the pan (pan-460c899ccec6f357) in the dishwasher . This is the second kitchen vessel the user places into the dishwasher.\n7. Since the pan (pan-460c899ccec6f357) is the second kitchen vessel to be placed in the dishwasher , and since the user turned the faucet on 3 times to clean the pan (pan-460c899ccec6f357), the answer is 3 times.", "referenced_object_masks": [ "pan-460c899ccec6f357", "empty bowl-a070b9a0f9c63725" ] }, { "key": "P01-20240203-152323:5a5dc7b36da5f962e0a386e20337dc34d2a40df4", "video_id": "P01-20240203-152323", "question": "What specific tableware do I retrieve after picking up the pot holder for the third time?", "answer_choice_0": "Serving plate", "answer_choice_1": "Plastic spoon", "answer_choice_2": "Spatula", "answer_choice_3": "Fork", "answer_choice_4": "Plate", "answer_id": 4, "answer": "Plate", "question_type": [ "Object Recognition", "Action Recognition", "Event Occurence" ], "objects_of_interest": [ "plate-60c770f6eb2341eb", "second tray-5080843de45706ff", "meat pie3-509ecb71aeddb708", "meat pie2-8b62e0ccf60fd204", "meat pie1-02fea42161507f6c", "serving plate-3d93b52e8c409262", "plastic spoon-0e653c50728967d1", "fork-4269c87a209049d6", "tray of food-e6985fedab409533", "red glove-d55d2fef91a13db3" ], "reasoning": "1. I noticed the user first picks up the red pot holder (red glove-d55d2fef91a13db3) at 00:20, a second time at 00:50, and a third time at 01:50. \n2. Immediately after picking up the red potholder (red glove-d55d2fef91a13db3) for the third time, the user retrieves the first baking sheet (tray of food-e6985fedab409533) at 01:53 and then a second baking sheet (second tray-5080843de45706ff) at 02:01. Neither of these items are tableware. \n3. Finally, the user retrieves a plate (plate-60c770f6eb2341eb) from the cabinet at 02:11, which is considered tableware.\n4. Therefore, the answer is plate (plate-60c770f6eb2341eb).", "referenced_object_masks": [ "tray of food-e6985fedab409533", "red glove-d55d2fef91a13db3", "second tray-5080843de45706ff", "plate-60c770f6eb2341eb" ] }, { "key": "P01-20240203-152323:5ac5483f6159efa538f1c315f1fb2ff2898f9b83", "video_id": "P01-20240203-152323", "question": "In which direction do I gaze while opening the drawer after I touch the pot holder for the second time?", "answer_choice_0": "Right of stove", "answer_choice_1": "Left of fridge", "answer_choice_2": "Up", "answer_choice_3": "Down", "answer_choice_4": "Behind", "answer_id": 3, "answer": "Down", "question_type": [ "Egocentric & Spatial Perception", "Situational Awareness", "Action Recognition", "Event Occurence" ], "objects_of_interest": [ "plate-60c770f6eb2341eb", "second tray-5080843de45706ff", "meat pie3-509ecb71aeddb708", "meat pie2-8b62e0ccf60fd204", "meat pie1-02fea42161507f6c", "serving plate-3d93b52e8c409262", "plastic spoon-0e653c50728967d1", "fork-4269c87a209049d6", "tray of food-e6985fedab409533", "red glove-d55d2fef91a13db3" ], "reasoning": "1. The user touches the red pot holder (red glove-d55d2fef91a13db3) for the first time at 00:20 and a second time at 00:50. \n2. The drawer in question is visible on the user's left, next to the stove. \n3. The user opens the drawer at 00:54, while looking down, searching for a cooking utensil.", "referenced_object_masks": [ "red glove-d55d2fef91a13db3" ] }, { "key": "P01-20240203-152323:6812499d201e41861686afeb26978cb00eb198a0", "video_id": "P01-20240203-152323", "question": "What paper product is on the counter behind me while I am standing at the stove?", "answer_choice_0": "Stack of paper plates", "answer_choice_1": "Heap of mail", "answer_choice_2": "Roll of paper towels", "answer_choice_3": "Pile of napkins", "answer_choice_4": "Box of Kleenex", "answer_id": 2, "answer": "Roll of paper towels", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "plate-60c770f6eb2341eb", "second tray-5080843de45706ff", "meat pie3-509ecb71aeddb708", "meat pie2-8b62e0ccf60fd204", "meat pie1-02fea42161507f6c", "serving plate-3d93b52e8c409262", "plastic spoon-0e653c50728967d1", "fork-4269c87a209049d6", "tray of food-e6985fedab409533", "red glove-d55d2fef91a13db3" ], "reasoning": "1. The user stands at the stove throughout the video, specifically when they are removing meat pies from the oven, starting at 00:32. \n2. The user enters the kitchen at 00:02, where two parallel counters run the length of the kitchen: one on the front and another on the back. \n3. Various items are visible on the counter opposite the stove. This would be the counter behind the user when they are standing at the stove. \n3. A roll of paper towels is visible on the back counter.", "referenced_object_masks": [] }, { "key": "P01-20240203-152323:9509886098c47879cafdceca4c535c33c4d85821", "video_id": "P01-20240203-152323", "question": "If I'd burned the first 3 meat pies from the first sheet and they were inedible, while also burning 5 meat pies from the second sheet, how many pies would I have left to serve?", "answer_choice_0": "7", "answer_choice_1": "6", "answer_choice_2": "8", "answer_choice_3": "5", "answer_choice_4": "10", "answer_id": 0, "answer": "7", "question_type": [ "Counterfactual", "Counting", "Numerical Reasoning", "Object Recognition" ], "objects_of_interest": [ "plate-60c770f6eb2341eb", "second tray-5080843de45706ff", "meat pie3-509ecb71aeddb708", "meat pie2-8b62e0ccf60fd204", "meat pie1-02fea42161507f6c", "serving plate-3d93b52e8c409262", "plastic spoon-0e653c50728967d1", "fork-4269c87a209049d6", "tray of food-e6985fedab409533", "red glove-d55d2fef91a13db3" ], "reasoning": "1. I counted the number of meat pies from the first sheet (tray of food-e6985fedab409533), visible at 00:51. There were 7 meat pies in total. \n2. If the user burned 3, that would leave 4 pies , subtracting 3 from 7. \n3. Next, I counted the number of meat pies on the second sheet (second tray-5080843de45706ff), visible at 02:03. There were 8 meat pies in total. \n4. If the user burned 5, that would leave 3 pies , subtracting 5 from 8.\n5. Adding the 4 pies left over from the first sheet (tray of food-e6985fedab409533) with the 3 pies left over from the second sheet (second tray-5080843de45706ff) would leave 7 pies left over to serve.", "referenced_object_masks": [ "tray of food-e6985fedab409533", "second tray-5080843de45706ff" ] }, { "key": "P01-20240203-152323:e3253aafea1f17be3c4550dfd57ba4e4b2090987", "video_id": "P01-20240203-152323", "question": "Not counting knobs or handles, how many objects do I touch with my left hand between the times I pull the first and second sheet of meat pies out from the oven?", "answer_choice_0": "2", "answer_choice_1": "10", "answer_choice_2": "6", "answer_choice_3": "8", "answer_choice_4": "4", "answer_id": 4, "answer": "4", "question_type": [ "Temporal Reasoning", "Action Recognition", "Event Occurence", "Object Recognition", "Counting" ], "objects_of_interest": [ "plate-60c770f6eb2341eb", "second tray-5080843de45706ff", "meat pie3-509ecb71aeddb708", "meat pie2-8b62e0ccf60fd204", "meat pie1-02fea42161507f6c", "serving plate-3d93b52e8c409262", "plastic spoon-0e653c50728967d1", "fork-4269c87a209049d6", "tray of food-e6985fedab409533", "red glove-d55d2fef91a13db3" ], "reasoning": "1. I determined the user removed the first sheet (tray of food-e6985fedab409533) from the oven at 00:31 and briefly pulled the second sheet (second tray-5080843de45706ff) at 02:03. This gives a timespan of 00:31-02:03.\n2. Next, I determined all the objects the user touched between those times, ignoring knobs and handles. These objects included a black plastic spoon (plastic spoon-0e653c50728967d1) at 00:56, a dark blue serving plate (serving plate-3d93b52e8c409262) at 00:59, a freshly baked meat pie (meat pie1-02fea42161507f6c) at 01:44, and a red potholder (red glove-d55d2fef91a13db3) at 00:35 and 01:51. \n3. Counting the number of objects gives a total of 4.", "referenced_object_masks": [ "meat pie1-02fea42161507f6c", "red glove-d55d2fef91a13db3", "second tray-5080843de45706ff", "tray of food-e6985fedab409533", "serving plate-3d93b52e8c409262", "plastic spoon-0e653c50728967d1" ] }, { "key": "P01-20240203-152956:2f391e5eeeec935ea0add8df5e9df4414aa2e700", "video_id": "P01-20240203-152956", "question": "What is the ratio of sheets of aluminum foil I used to pies I put on the first plate?", "answer_choice_0": "1:4", "answer_choice_1": "1:8", "answer_choice_2": "1:3", "answer_choice_3": "1:2", "answer_choice_4": "1:6", "answer_id": 0, "answer": "1:4", "question_type": [ "Numerical Reasoning", "Counting", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "foil2-bb44361e219f3117", "unrolled foil-cf1d912cff6dee38", "foil wrap-2235546dde63c803", "box of foil wrap-bc24aa33519b5a40", "pie2-7c0e8b08f01b540a", "plate2-2ba459bb82d1644b", "plate-93b520de4380b9e1", "fork-96935f2c226f655a", "pie-3faff8c98a7e48be", "plastic spoon-85e1cf228aa07339", "tray-b425dac2854a391a", "oven glove-bbc50cb48f983958" ], "reasoning": "1. I watched the video and first saw a plate of pies (plate2-2ba459bb82d1644b) at 00:04. I observed that there were 6 individual pies already on the plate of pies (plate2-2ba459bb82d1644b). \n2. From 00:23-00:29 I saw the user move a pie (pie-3faff8c98a7e48be) from the tray (tray-b425dac2854a391a) to the plate of pies (plate2-2ba459bb82d1644b).\n3. From 00:35-00:48, I saw the user move a second pie from the tray (tray-b425dac2854a391a) to the plate of pies (plate2-2ba459bb82d1644b). \n4. At 00:54, I saw the user grab another plate (plate-93b520de4380b9e1) from the cabinet. \n5. After 00:54, no more pies were added to the plate of pies (plate2-2ba459bb82d1644b), therefore I counted that the user had fit a total of 8 pies onto it. \n6. From 02:19-02:23, I saw the user unroll and tear off 1 sheet of aluminum foil (unrolled foil-cf1d912cff6dee38). \n7. From 02:25-02:28, I saw the user use the torn sheet of aluminum foil (unrolled foil-cf1d912cff6dee38) to cover the plate of pies (plate2-2ba459bb82d1644b). \n8. From 02:37-02:41, I saw the user unroll and tear off a second sheet of aluminum foil (foil2-bb44361e219f3117). \n9. I did not see any more sheets of foil, therefore I counted that 2 sheets of foil were used. \n10. I determined the ratio of sheets of foil to pies on the first plate was 2:8. I simplified this to 1:4.", "referenced_object_masks": [ "foil2-bb44361e219f3117", "unrolled foil-cf1d912cff6dee38", "tray-b425dac2854a391a", "pie-3faff8c98a7e48be", "plate2-2ba459bb82d1644b", "plate-93b520de4380b9e1" ] }, { "key": "P01-20240203-152956:4f711d03ee58ad45653ed3a382051d58eb870947", "video_id": "P01-20240203-152956", "question": "Which is greater, and by how much: the time I spent wearing the oven mitt or the duration that elapsed in between times that I was wearing the oven mitt?", "answer_choice_0": "Wearing the mitt--by approximately 1 minute and 20 seconds", "answer_choice_1": "Neither--the times were nearly identical", "answer_choice_2": "Wearing the mitt--by approximately 45 seconds", "answer_choice_3": "Between wearing the mitt--by approximately 1 minute and 20 seconds", "answer_choice_4": "Between wearing the mitt--by approximately 45 seconds", "answer_id": 3, "answer": "Between wearing the mitt--by approximately 1 minute and 20 seconds", "question_type": [ "Numerical Reasoning", "Temporal Reasoning", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "foil2-bb44361e219f3117", "unrolled foil-cf1d912cff6dee38", "foil wrap-2235546dde63c803", "box of foil wrap-bc24aa33519b5a40", "pie2-7c0e8b08f01b540a", "plate2-2ba459bb82d1644b", "plate-93b520de4380b9e1", "fork-96935f2c226f655a", "pie-3faff8c98a7e48be", "plastic spoon-85e1cf228aa07339", "tray-b425dac2854a391a", "oven glove-bbc50cb48f983958" ], "reasoning": "1. I watched the video and found that the user first wore the oven mitt (oven glove-bbc50cb48f983958) from 00:06-00:21. \n2. I calculated 21-6=15 to find that the user wore the oven mitt (oven glove-bbc50cb48f983958) for 15 seconds during the first occurrence. \n3. I watched for the next time the user wore the oven mitt (oven glove-bbc50cb48f983958) and found that it was from 02:05-02:11. \n4. I calculated 11-5=6 to find that the user wore the oven mitt (oven glove-bbc50cb48f983958) for 6 seconds during the second occurrence. \n5. I subtracted 21 seconds from 02:05 to find that the duration between occurrences wearing the oven mitt (oven glove-bbc50cb48f983958) was 1 minute and 44 seconds. \n6. I added 15+6=21 to find the total duration wearing the oven mitt (oven glove-bbc50cb48f983958) was 21 seconds. \n7. I subtracted 21 seconds from 1 minute and 44 seconds to find a difference of 1 minute and 23 seconds. \n8. I concluded that the time between wearing the oven mitt (oven glove-bbc50cb48f983958) was greater than the time spent wearing it by approximately 1 minute and 20 seconds.", "referenced_object_masks": [ "oven glove-bbc50cb48f983958" ] }, { "key": "P01-20240203-152956:62494534cb47e1487cad2b790022e33fa87781aa", "video_id": "P01-20240203-152956", "question": "How many pies did I fit onto the first plate?", "answer_choice_0": "8", "answer_choice_1": "12", "answer_choice_2": "14", "answer_choice_3": "10", "answer_choice_4": "6", "answer_id": 0, "answer": "8", "question_type": [ "Action Recognition", "Object Recognition", "Counting", "Temporal Reasoning" ], "objects_of_interest": [ "foil2-bb44361e219f3117", "unrolled foil-cf1d912cff6dee38", "foil wrap-2235546dde63c803", "box of foil wrap-bc24aa33519b5a40", "pie2-7c0e8b08f01b540a", "plate2-2ba459bb82d1644b", "plate-93b520de4380b9e1", "fork-96935f2c226f655a", "pie-3faff8c98a7e48be", "plastic spoon-85e1cf228aa07339", "tray-b425dac2854a391a", "oven glove-bbc50cb48f983958" ], "reasoning": "1. I watched the video and first saw a plate of pies (plate2-2ba459bb82d1644b) at 00:04. I observed that there were 6 individual pies already on the plate of pies (plate2-2ba459bb82d1644b). \n2. From 00:15-00:19 I saw the user lift and place a tray (tray-b425dac2854a391a) of individual pies onto the stovetop. \n3. From 00:23-00:29 I saw the user move a pie (pie-3faff8c98a7e48be) from the tray (tray-b425dac2854a391a) to the plate of pies (plate2-2ba459bb82d1644b).\n4. From 00:35-00:48, I saw the user move a second pie from the tray (tray-b425dac2854a391a) to the plate of pies (plate2-2ba459bb82d1644b). \n5. At 00:54, I saw the user grab another plate (plate-93b520de4380b9e1) from the cabinet. \n6. After 00:54, no more pies were added to the plate of pies (plate2-2ba459bb82d1644b), therefore I counted that the user had fit a total of 8 pies onto it.", "referenced_object_masks": [ "tray-b425dac2854a391a", "pie-3faff8c98a7e48be", "plate2-2ba459bb82d1644b", "plate-93b520de4380b9e1" ] }, { "key": "P01-20240203-152956:84aee95b0b8178016988460e64e4e5c4dc39fab9", "video_id": "P01-20240203-152956", "question": "Relative to the tray, where is the second plate when I added most of the pies to it?", "answer_choice_0": "Above and in front", "answer_choice_1": "Level and to the right", "answer_choice_2": "Level and to the left", "answer_choice_3": "Level and in front", "answer_choice_4": "Below and in front", "answer_id": 0, "answer": "Above and in front", "question_type": [ "Counting", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning", "Action Recognition", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "foil2-bb44361e219f3117", "unrolled foil-cf1d912cff6dee38", "foil wrap-2235546dde63c803", "box of foil wrap-bc24aa33519b5a40", "pie2-7c0e8b08f01b540a", "plate2-2ba459bb82d1644b", "plate-93b520de4380b9e1", "fork-96935f2c226f655a", "pie-3faff8c98a7e48be", "plastic spoon-85e1cf228aa07339", "tray-b425dac2854a391a", "oven glove-bbc50cb48f983958" ], "reasoning": "1. I watched the video and saw the user lift and place a tray (tray-b425dac2854a391a) of individual pies onto the stovetop from 00:15-00:19. \n3. At 00:19, I observed that there was already 1 plate of pies (plate2-2ba459bb82d1644b) on the counter to the right of the stovetop. \n4. At 00:54, I saw the user grab another plate (plate-93b520de4380b9e1) from the cabinet. \n5. At 00:59, I saw the user place the second plate (plate-93b520de4380b9e1) on the counter, to the right of the stovetop and to the left of the first plate (plate2-2ba459bb82d1644b). \n6. At 00:59, I counted that there were 6 individual pies remaining on the tray (tray-b425dac2854a391a). I continued watching, noting the location of the second plate (plate-93b520de4380b9e1) relative to the tray (tray-b425dac2854a391a) when each individual pie was added to it. \n7. From 01:00-01:07, I saw the user move 2 pies from the tray (tray-b425dac2854a391a) to the second plate (plate-93b520de4380b9e1) while it was on the counter right of the stovetop. \n9. At 01:32, I saw the user pick up the second plate (plate-93b520de4380b9e1). And from 01:34-02:00 I saw the user hold the second plate (plate-93b520de4380b9e1) above and in front of the tray (tray-b425dac2854a391a). The user put down the second plate (plate-93b520de4380b9e1) at 02:02. \n10. I saw that the remaining 5 pies were added one by one from 01:35-02:00 to the second plate (plate-93b520de4380b9e1) while it was above and in front of the tray (tray-b425dac2854a391a).\n11. Since 5 is greater than 1, I determined that the second plate (plate-93b520de4380b9e1) was above and in front of the tray (tray-b425dac2854a391a) when most pies were added.", "referenced_object_masks": [ "plate2-2ba459bb82d1644b", "plate-93b520de4380b9e1", "tray-b425dac2854a391a" ] }, { "key": "P01-20240203-152956:8cfa6d506473d612a6de978f1636251cba7a2b1f", "video_id": "P01-20240203-152956", "question": "What percentage of pies on the tray was I able to lift from the tray without a fork?", "answer_choice_0": "50%", "answer_choice_1": "100%", "answer_choice_2": "87.5%", "answer_choice_3": "75%", "answer_choice_4": "62.5%", "answer_id": 2, "answer": "87.5%", "question_type": [ "Numerical Reasoning", "Counting", "Action Recognition", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "foil2-bb44361e219f3117", "unrolled foil-cf1d912cff6dee38", "foil wrap-2235546dde63c803", "box of foil wrap-bc24aa33519b5a40", "pie2-7c0e8b08f01b540a", "plate2-2ba459bb82d1644b", "plate-93b520de4380b9e1", "fork-96935f2c226f655a", "pie-3faff8c98a7e48be", "plastic spoon-85e1cf228aa07339", "tray-b425dac2854a391a", "oven glove-bbc50cb48f983958" ], "reasoning": "1. I watched the video and saw the user lift and place a tray (tray-b425dac2854a391a) of individual pies onto the stovetop from 00:15-00:19. \n2. I noted the times that each individual pie was moved off of the tray (tray-b425dac2854a391a) and whether a fork (fork-96935f2c226f655a) was used. \n3. From 00:23-00:29, I saw the first pie (pie-3faff8c98a7e48be) moved from the tray (tray-b425dac2854a391a) with just a plastic spoon (plastic spoon-85e1cf228aa07339). \n4. From 00:35-00:48, I saw the user use a fork (fork-96935f2c226f655a) and plastic spoon (plastic spoon-85e1cf228aa07339) to move a second pie from the tray (tray-b425dac2854a391a) to the plate of pies (plate2-2ba459bb82d1644b). \n5. From 01:00-01:07, I saw the user use just a plastic spoon (plastic spoon-85e1cf228aa07339) to move 2 pies from the tray (tray-b425dac2854a391a) to a new plate (plate-93b520de4380b9e1). \n6. From 01:16-01:44, I saw the user use just a plastic spoon (plastic spoon-85e1cf228aa07339) and their hand to move a fifth pie (pie2-7c0e8b08f01b540a) from the tray (tray-b425dac2854a391a) to a new plate (plate-93b520de4380b9e1). \n7. From 01:45-01:47 I saw the user use just a plastic spoon (plastic spoon-85e1cf228aa07339) to move a sixth pie from the tray (tray-b425dac2854a391a) to a new plate (plate-93b520de4380b9e1). \n8. From 01:47-01:57, I saw the user use just a plastic spoon (plastic spoon-85e1cf228aa07339) to move a seventh pie from the tray (tray-b425dac2854a391a) to a new plate (plate-93b520de4380b9e1). \n9. From 01:57-02:00, I saw the user use just a plastic spoon (plastic spoon-85e1cf228aa07339) to move an eighth pie from the tray (tray-b425dac2854a391a) to a new plate (plate-93b520de4380b9e1). \n10. There were no more pies moved, so I counted that 1 out of 8 pies from the tray (tray-b425dac2854a391a) were moved with a fork (fork-96935f2c226f655a). \n11. To find the percentage that didn't use a fork, I divided 7/8 and multiplied by 100 to convert the decimal into a percent. I calculated the answer to be 87.5%.", "referenced_object_masks": [ "fork-96935f2c226f655a", "pie2-7c0e8b08f01b540a", "tray-b425dac2854a391a", "pie-3faff8c98a7e48be", "plastic spoon-85e1cf228aa07339", "plate2-2ba459bb82d1644b", "plate-93b520de4380b9e1" ] }, { "key": "P01-20240203-161757:1a71e112a1b12058bba8c81e9fb1188752410686", "video_id": "P01-20240203-161757", "question": "When I put the foil that was originally around the roll in its final location, where am I in relation to the place I was when I put the second tupperware container in its final location?", "answer_choice_0": "Diagonally to the left", "answer_choice_1": "Directly left", "answer_choice_2": "Directly right", "answer_choice_3": "Diagonally to the right", "answer_choice_4": "Directly in front", "answer_id": 4, "answer": "Directly in front", "question_type": [ "Action Recognition", "Counting", "Egocentric & Spatial Perception", "Event Occurence", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "scale-7e0034310be994a3", "foil paper roll-5212109bb8d42a57", "empty bag-320a09564e1c66e3", "pie2pie2-abeadeca2462de3c", "pie-2aa6164113f62137", "tupperware3-8dfa6509acd22704", "slice20-de81edc3d6d56122", "plastic lid-e20e3dbea18715a4", "tupperware2-a0fb3db3ff346ccd", "slice19-7adf9f025b3d6b59", "slice18-dbbee25d6c16d2e4", "slice17-46d0e52b3709446f", "slice16-8e422edddf6a6820", "slice15-acb1852f63be87aa", "slice14-b58a073b2674f35f", "slice13-ecbf630c5963662f", "slice12-a4654453f6effd5a", "slice11-bb6d37d2632b26d6", "slice10-2b0f119ef97d85fc", "slice9-cf3c43c810245018", "slice8-61a36cb20c92dec5", "slice7-7a175ec917ee580a", "slice6-53a16161fef1ed27", "slice5-d1b5b7e30cd2a2e0", "slice4-174f446987562733", "slice3-6d1f36b77afea1d2", "second slice of roll-96e8f8d28e82e36c", "cut slice-97f1eea8ce9b1ae5", "roll of sweet-354a69cd22d1f4b2", "bigger knife-24cc37ff248c8eee", "plastic tupperware-d190ba8e69287123", "foil paper-3d2467ca729ddaeb", "cardboard box-97bffb5f2e9163f0", "plastic piece-2ec3c718ddb20074", "chopping board-f6dca058eee9d3b7", "knife-ad2a6a83f75d9ff4", "trash2-2a4e9a10ffaad4f8", "trash-6dbfe2ff40cb0eb0", "plate2-2259b736fc5a94b5", "plate-423f0c0894855f32" ], "reasoning": "1. First, I had to locate the second tupperware container (tupperware3-8dfa6509acd22704) the user filled with food . I saw the user fill the first tupperware container (plastic tupperware-d190ba8e69287123) full with food from 00:57-02:13 and the second (tupperware3-8dfa6509acd22704) at 03:04-03:15. \n2. Next, having located this container (tupperware3-8dfa6509acd22704), I watched to see its final location, and observed the user putting it (tupperware3-8dfa6509acd22704) into the refrigerator at 03:29. It does not appear again in the video. \n3. Then, I looked for a foil (foil paper-3d2467ca729ddaeb) wrapped roll (roll of sweet-354a69cd22d1f4b2) and found that the user takes one out of the refrigerator at 00:28 and unwraps it at 00:40-00:47. \n4. Next, I looked for the final location where the user puts the foil (foil paper-3d2467ca729ddaeb) that was originally around the roll (roll of sweet-354a69cd22d1f4b2), and saw the user place it (foil paper-3d2467ca729ddaeb) in a trashcan at 02:50. \n5. Finally, I observed where the user is at 02:50 in relation to the refrigerator , and deduced the user is directly in front of it.", "referenced_object_masks": [ "roll of sweet-354a69cd22d1f4b2", "foil paper-3d2467ca729ddaeb", "tupperware3-8dfa6509acd22704", "plastic tupperware-d190ba8e69287123" ] }, { "key": "P01-20240203-161757:1b39f3dfed4c429aee3c469a8c331192043db26a", "video_id": "P01-20240203-161757", "question": "If I walked from where I am facing when I put foil somewhere for the final time to the elliptical machine, how many times would I turn left?", "answer_choice_0": "1", "answer_choice_1": "0", "answer_choice_2": "7", "answer_choice_3": "3", "answer_choice_4": "5", "answer_id": 3, "answer": "3", "question_type": [ "Counterfactual", "Situational Awareness", "Egocentric & Spatial Perception", "Action Recognition", "Object Recognition" ], "objects_of_interest": [ "scale-7e0034310be994a3", "foil paper roll-5212109bb8d42a57", "empty bag-320a09564e1c66e3", "pie2pie2-abeadeca2462de3c", "pie-2aa6164113f62137", "tupperware3-8dfa6509acd22704", "slice20-de81edc3d6d56122", "plastic lid-e20e3dbea18715a4", "tupperware2-a0fb3db3ff346ccd", "slice19-7adf9f025b3d6b59", "slice18-dbbee25d6c16d2e4", "slice17-46d0e52b3709446f", "slice16-8e422edddf6a6820", "slice15-acb1852f63be87aa", "slice14-b58a073b2674f35f", "slice13-ecbf630c5963662f", "slice12-a4654453f6effd5a", "slice11-bb6d37d2632b26d6", "slice10-2b0f119ef97d85fc", "slice9-cf3c43c810245018", "slice8-61a36cb20c92dec5", "slice7-7a175ec917ee580a", "slice6-53a16161fef1ed27", "slice5-d1b5b7e30cd2a2e0", "slice4-174f446987562733", "slice3-6d1f36b77afea1d2", "second slice of roll-96e8f8d28e82e36c", "cut slice-97f1eea8ce9b1ae5", "roll of sweet-354a69cd22d1f4b2", "bigger knife-24cc37ff248c8eee", "plastic tupperware-d190ba8e69287123", "foil paper-3d2467ca729ddaeb", "cardboard box-97bffb5f2e9163f0", "plastic piece-2ec3c718ddb20074", "chopping board-f6dca058eee9d3b7", "knife-ad2a6a83f75d9ff4", "trash2-2a4e9a10ffaad4f8", "trash-6dbfe2ff40cb0eb0", "plate2-2259b736fc5a94b5", "plate-423f0c0894855f32" ], "reasoning": "1. First, I had to locate the last place where the user puts foil. At 03:50, I saw the user place a roll of foil (foil paper roll-5212109bb8d42a57) in a drawer to the right of the refrigerator . I watched the rest of the video and confirmed this was the final time the user does something with a piece of foil in the video. \n2. Next, I had to locate the elliptical machine . Its handlebars are visible at the very beginning of the video, 00:00, and at 00:21 I saw a clearer view of it. I noticed that the elliptical machine is straight across from the drawer where the user puts foil (foil paper roll-5212109bb8d42a57) for the final time, but that a kitchen counter is in between these two places. \n3. Reasoning about the scene, I realized that if I was standing in front of the drawer where I put the foil (foil paper roll-5212109bb8d42a57), the elliptical machine would be behind me, and that I would need to walk around the kitchen counter to reach it. The first direction I would need to turn would be left, then I would walk straight, then turn left again, then walk straight, then turn left a final time before walking straight to reach the elliptical machine .", "referenced_object_masks": [ "foil paper roll-5212109bb8d42a57" ] }, { "key": "P01-20240203-161757:50e92eb8b1b6e91380c68c355d497511bd02292a", "video_id": "P01-20240203-161757", "question": "In what two consecutive directions would I have to walk if I was standing in front of and facing away from the blue bicycle and wanted to get to the last location that I carried a piece of roll?", "answer_choice_0": "Forward and right", "answer_choice_1": "Left and right", "answer_choice_2": "Forward and left", "answer_choice_3": "Left and left", "answer_choice_4": "Right and right", "answer_id": 0, "answer": "Forward and right", "question_type": [ "Egocentric & Spatial Perception", "Counterfactual", "Object Recognition", "Temporal Reasoning", "Situational Awareness" ], "objects_of_interest": [ "scale-7e0034310be994a3", "foil paper roll-5212109bb8d42a57", "empty bag-320a09564e1c66e3", "pie2pie2-abeadeca2462de3c", "pie-2aa6164113f62137", "tupperware3-8dfa6509acd22704", "slice20-de81edc3d6d56122", "plastic lid-e20e3dbea18715a4", "tupperware2-a0fb3db3ff346ccd", "slice19-7adf9f025b3d6b59", "slice18-dbbee25d6c16d2e4", "slice17-46d0e52b3709446f", "slice16-8e422edddf6a6820", "slice15-acb1852f63be87aa", "slice14-b58a073b2674f35f", "slice13-ecbf630c5963662f", "slice12-a4654453f6effd5a", "slice11-bb6d37d2632b26d6", "slice10-2b0f119ef97d85fc", "slice9-cf3c43c810245018", "slice8-61a36cb20c92dec5", "slice7-7a175ec917ee580a", "slice6-53a16161fef1ed27", "slice5-d1b5b7e30cd2a2e0", "slice4-174f446987562733", "slice3-6d1f36b77afea1d2", "second slice of roll-96e8f8d28e82e36c", "cut slice-97f1eea8ce9b1ae5", "roll of sweet-354a69cd22d1f4b2", "bigger knife-24cc37ff248c8eee", "plastic tupperware-d190ba8e69287123", "foil paper-3d2467ca729ddaeb", "cardboard box-97bffb5f2e9163f0", "plastic piece-2ec3c718ddb20074", "chopping board-f6dca058eee9d3b7", "knife-ad2a6a83f75d9ff4", "trash2-2a4e9a10ffaad4f8", "trash-6dbfe2ff40cb0eb0", "plate2-2259b736fc5a94b5", "plate-423f0c0894855f32" ], "reasoning": "1. First, I located the blue bicycle . I saw a clear visual of the bicycle at 00:20 and noticed that it is against the wall on the right in the room directly behind the kitchen counter. \n2. Next, I located the last location where the user is seen carrying a piece of roll (slice18-dbbee25d6c16d2e4). I found this at 04:01, the user's last location in the video, when the user is carrying a slice of roll (slice18-dbbee25d6c16d2e4) on a plate (plate-423f0c0894855f32). I realized that the user is leaving the kitchen, walking away from the room with the blue bicycle at this moment. \n3. I realized that if the user was standing in front of the blue bicycle and facing away from it, the user would have to walk forward and then right to get to their last location at 04:01.", "referenced_object_masks": [ "plate-423f0c0894855f32", "slice18-dbbee25d6c16d2e4" ] }, { "key": "P01-20240203-161757:84516f9a96f7f03e769744ec6577ad15d6974cab", "video_id": "P01-20240203-161757", "question": "If I added the number of roll slices that I put in the refrigerator to the number of pies that I put into the refrigerator, what would be the sum?", "answer_choice_0": "12", "answer_choice_1": "15", "answer_choice_2": "17", "answer_choice_3": "20", "answer_choice_4": "19", "answer_id": 3, "answer": "20", "question_type": [ "Action Recognition", "Counting", "Numerical Reasoning", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "scale-7e0034310be994a3", "foil paper roll-5212109bb8d42a57", "empty bag-320a09564e1c66e3", "pie2pie2-abeadeca2462de3c", "pie-2aa6164113f62137", "tupperware3-8dfa6509acd22704", "slice20-de81edc3d6d56122", "plastic lid-e20e3dbea18715a4", "tupperware2-a0fb3db3ff346ccd", "slice19-7adf9f025b3d6b59", "slice18-dbbee25d6c16d2e4", "slice17-46d0e52b3709446f", "slice16-8e422edddf6a6820", "slice15-acb1852f63be87aa", "slice14-b58a073b2674f35f", "slice13-ecbf630c5963662f", "slice12-a4654453f6effd5a", "slice11-bb6d37d2632b26d6", "slice10-2b0f119ef97d85fc", "slice9-cf3c43c810245018", "slice8-61a36cb20c92dec5", "slice7-7a175ec917ee580a", "slice6-53a16161fef1ed27", "slice5-d1b5b7e30cd2a2e0", "slice4-174f446987562733", "slice3-6d1f36b77afea1d2", "second slice of roll-96e8f8d28e82e36c", "cut slice-97f1eea8ce9b1ae5", "roll of sweet-354a69cd22d1f4b2", "bigger knife-24cc37ff248c8eee", "plastic tupperware-d190ba8e69287123", "foil paper-3d2467ca729ddaeb", "cardboard box-97bffb5f2e9163f0", "plastic piece-2ec3c718ddb20074", "chopping board-f6dca058eee9d3b7", "knife-ad2a6a83f75d9ff4", "trash2-2a4e9a10ffaad4f8", "trash-6dbfe2ff40cb0eb0", "plate2-2259b736fc5a94b5", "plate-423f0c0894855f32" ], "reasoning": "1. First, I had to determine the number of roll slices the user put into the refrigerator . At 02:30, I saw the user put a tupperware container (plastic tupperware-d190ba8e69287123) full of roll slices into the refrigerator . \n2. Next, I went back to count how many roll slices the user put into the tupperware container (plastic tupperware-d190ba8e69287123). I saw the user place individual slices into the container (plastic tupperware-d190ba8e69287123). The user puts the first slice (cut slice-97f1eea8ce9b1ae5) in at 00:57, the second (second slice of roll-96e8f8d28e82e36c) at 01:03, the third (slice3-6d1f36b77afea1d2) at 01:07, the fourth (slice4-174f446987562733) at 01:11, the fifth (slice5-d1b5b7e30cd2a2e0) at 01:14, the sixth (slice6-53a16161fef1ed27) at 01:17, the seventh (slice7-7a175ec917ee580a) at 01:21, the eighth (slice8-61a36cb20c92dec5) at 01:24, the ninth (slice9-cf3c43c810245018) at 01:28, the tenth (slice10-2b0f119ef97d85fc) at 01:33, the eleventh (slice12-a4654453f6effd5a) at 01:39, the twelfth (slice13-ecbf630c5963662f) at 01:43, the thirteenth (slice14-b58a073b2674f35f) at 01:48, the fourteenth (slice15-acb1852f63be87aa) at 01:54, the fifteenth (slice18-dbbee25d6c16d2e4) at 02:09, and the sixteenth (slice19-7adf9f025b3d6b59) 02:13. I added up all these instances and got a total of 16. \n3. Then, I looked for the moment where the user puts pies into the refrigerator and saw the user place a tupperware container (tupperware3-8dfa6509acd22704) full of pies into the refrigerator at 03:30. \n4. I counted the number of pies the user put into the tupperware container (tupperware3-8dfa6509acd22704) and saw the user place one pie into the container (tupperware3-8dfa6509acd22704) at 03:04 (pie-2aa6164113f62137) and 03:06 (pie2pie2-abeadeca2462de3c), and two pies at 03:15, for a total of 4. \n5. Finally, I added the total number (16) of roll slices that went into the refrigerator to the total number (4) of pies that went into the refrigerator and got a total of 21.", "referenced_object_masks": [ "slice7-7a175ec917ee580a", "slice14-b58a073b2674f35f", "slice9-cf3c43c810245018", "pie2pie2-abeadeca2462de3c", "slice6-53a16161fef1ed27", "slice3-6d1f36b77afea1d2", "slice12-a4654453f6effd5a", "slice19-7adf9f025b3d6b59", "slice13-ecbf630c5963662f", "slice5-d1b5b7e30cd2a2e0", "pie-2aa6164113f62137", "second slice of roll-96e8f8d28e82e36c", "slice4-174f446987562733", "plastic tupperware-d190ba8e69287123", "slice15-acb1852f63be87aa", "cut slice-97f1eea8ce9b1ae5", "slice10-2b0f119ef97d85fc", "slice18-dbbee25d6c16d2e4", "slice8-61a36cb20c92dec5", "tupperware3-8dfa6509acd22704" ] }, { "key": "P01-20240203-161757:c96a0da1a79f03981e02b61c6944f92cfcd14f8b", "video_id": "P01-20240203-161757", "question": "When I put the first thing I pick up off the floor into a trashcan, where is the place where I retrieve the foil wrapped roll in relation to me?", "answer_choice_0": "3 o'clock", "answer_choice_1": "11 o 'clock", "answer_choice_2": "7 o'clock", "answer_choice_3": "9 o'clock", "answer_choice_4": "2 o'clock", "answer_id": 2, "answer": "7 o'clock", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "scale-7e0034310be994a3", "foil paper roll-5212109bb8d42a57", "empty bag-320a09564e1c66e3", "pie2pie2-abeadeca2462de3c", "pie-2aa6164113f62137", "tupperware3-8dfa6509acd22704", "slice20-de81edc3d6d56122", "plastic lid-e20e3dbea18715a4", "tupperware2-a0fb3db3ff346ccd", "slice19-7adf9f025b3d6b59", "slice18-dbbee25d6c16d2e4", "slice17-46d0e52b3709446f", "slice16-8e422edddf6a6820", "slice15-acb1852f63be87aa", "slice14-b58a073b2674f35f", "slice13-ecbf630c5963662f", "slice12-a4654453f6effd5a", "slice11-bb6d37d2632b26d6", "slice10-2b0f119ef97d85fc", "slice9-cf3c43c810245018", "slice8-61a36cb20c92dec5", "slice7-7a175ec917ee580a", "slice6-53a16161fef1ed27", "slice5-d1b5b7e30cd2a2e0", "slice4-174f446987562733", "slice3-6d1f36b77afea1d2", "second slice of roll-96e8f8d28e82e36c", "cut slice-97f1eea8ce9b1ae5", "roll of sweet-354a69cd22d1f4b2", "bigger knife-24cc37ff248c8eee", "plastic tupperware-d190ba8e69287123", "foil paper-3d2467ca729ddaeb", "cardboard box-97bffb5f2e9163f0", "plastic piece-2ec3c718ddb20074", "chopping board-f6dca058eee9d3b7", "knife-ad2a6a83f75d9ff4", "trash2-2a4e9a10ffaad4f8", "trash-6dbfe2ff40cb0eb0", "plate2-2259b736fc5a94b5", "plate-423f0c0894855f32" ], "reasoning": "1. First, I found the moment where the user puts the first piece of trash (trash2-2a4e9a10ffaad4f8) they pick up off the floor into a trashcan. I saw the user pick a piece of trash (trash2-2a4e9a10ffaad4f8) up off the floor at 00:07 and put it (trash2-2a4e9a10ffaad4f8) in the small trash can on the kitchen counter at 00:09. I noticed that the user is facing the corner of the kitchen that is opposite the refrigerator at this moment. \n2. Next, I looked for the moment where the user retrieves a foil (foil paper-3d2467ca729ddaeb) wrapped roll (roll of sweet-354a69cd22d1f4b2). I found this at 00:28, when the user retrieves the foil (foil paper-3d2467ca729ddaeb) wrapped roll (roll of sweet-354a69cd22d1f4b2) from the refrigerator . \n3. Reasoning about the scene and considering the user's angle (facing the corner of the kitchen counter) when they put trash (trash2-2a4e9a10ffaad4f8) in the trash can at 00:09, I concluded that the refrigerator is at the user's 7 o'clock at that moment.", "referenced_object_masks": [ "trash2-2a4e9a10ffaad4f8", "roll of sweet-354a69cd22d1f4b2", "foil paper-3d2467ca729ddaeb" ] }, { "key": "P01-20240203-184045:1019f001d2ef45b3fd7bcaefab348aedde422b7f", "video_id": "P01-20240203-184045", "question": "When I first pull an item out from underneath something else in the video, how many degrees to the right would I need to turn to face the second item I eventually pull out from underneath something else?", "answer_choice_0": "270 degrees", "answer_choice_1": "30 degrees", "answer_choice_2": "180 degrees", "answer_choice_3": "90 degrees", "answer_choice_4": "45 degrees", "answer_id": 2, "answer": "180 degrees", "question_type": [ "Egocentric & Spatial Perception", "Temporal Reasoning", "Action Recognition", "Situational Awareness" ], "objects_of_interest": [ "heat diffuser-c31909c9f1c68a4a", "pan-837efd3382aaf75a", "spatula-559856ff0e6b7833", "lid-8507a62c7b718116", "pot-3e374f58097c50ca", "plastic container-64474a3579aa4aa5", "chicken-9df3ae3b12d0aef0" ], "reasoning": "1. I look for the time the user first pulls an item out from underneath something else.\n2. This happens at 00:08, when the user pulls out a package of chicken (chicken-9df3ae3b12d0aef0) from underneath another package.\n3. I look for the second item the user pulls out from underneath something else.\n4. This happens from 00:45-00:46, when the user pulls a heat diffuser (heat diffuser-c31909c9f1c68a4a) out from underneath the pan (pan-837efd3382aaf75a).\n5. I recall where the user was standing at 00:08, and realize that it was in front of the refrigerator .\n6. I recall that the heat diffuser (heat diffuser-c31909c9f1c68a4a) was located directly behind the refrigerator .\n7. If the user, after grabbing the chicken (chicken-9df3ae3b12d0aef0), wanted to turn to the right by a certain amount of degrees to face the diffuser (heat diffuser-c31909c9f1c68a4a), the user would have to turn completely around: 180 degrees.\n8. Therefore, the answer is 180 degrees.", "referenced_object_masks": [ "pan-837efd3382aaf75a", "heat diffuser-c31909c9f1c68a4a", "chicken-9df3ae3b12d0aef0" ] }, { "key": "P01-20240203-184045:2f7504d22b3be5be7594b091ffb0f93fd210d409", "video_id": "P01-20240203-184045", "question": "From 00:27-00:40, what is the ratio of the number of knives to my right and the number of lids on the stove?", "answer_choice_0": "2:3", "answer_choice_1": "3:1", "answer_choice_2": "3:2", "answer_choice_3": "2:1", "answer_choice_4": "1:1", "answer_id": 4, "answer": "1:1", "question_type": [ "Egocentric & Spatial Perception", "Numerical Reasoning", "Object Recognition", "Counting" ], "objects_of_interest": [ "heat diffuser-c31909c9f1c68a4a", "pan-837efd3382aaf75a", "spatula-559856ff0e6b7833", "lid-8507a62c7b718116", "pot-3e374f58097c50ca", "plastic container-64474a3579aa4aa5", "chicken-9df3ae3b12d0aef0" ], "reasoning": "1. I looked at 00:27-00:40 to see what is happening.\n2. I notice that there are 3 knives to the right of the user during this timeframe.\n3. During this timeframe, I also see on the stove one lid (lid-8507a62c7b718116), and two other lids on the back of the stove .\n4. The ratio of knives to lids is 3:3, which reduces to 1:1.\n5. Therefore, 1:1 is the answer.", "referenced_object_masks": [ "lid-8507a62c7b718116" ] }, { "key": "P01-20240203-184045:568cb65d6f16f2f532fdbabb387264232c56366f", "video_id": "P01-20240203-184045", "question": "Rounded to the nearest whole number, what is the total approximate percentage of time in the video I spend standing in front of an open fridge and the time I spend stirring rice in a pot?", "answer_choice_0": "72%", "answer_choice_1": "61%", "answer_choice_2": "38%", "answer_choice_3": "19%", "answer_choice_4": "23%", "answer_id": 2, "answer": "38%", "question_type": [ "Action Recognition", "Temporal Reasoning", "Numerical Reasoning", "Counting" ], "objects_of_interest": [ "heat diffuser-c31909c9f1c68a4a", "pan-837efd3382aaf75a", "spatula-559856ff0e6b7833", "lid-8507a62c7b718116", "pot-3e374f58097c50ca", "plastic container-64474a3579aa4aa5", "chicken-9df3ae3b12d0aef0" ], "reasoning": "1. I watched the user approach the fridge , and calculated the amount of time they spent standing in front of it while it was open, which happened from 00:03-00:13 for a total of 10 seconds.\n2. Next, I looked at the time the user stirred the pot (pot-3e374f58097c50ca) of rice which occurred from 00:28-00:38 for a total of 10 seconds.\n3. I added these numbers together to get 20 seconds.\n4. I noticed the video is 53 seconds long.\n5. Next, I did the equation 20/53 which equaled 0.377.\n6. Then, I converted 0.377 into a percentage which equaled 37.7%.\n7. Finally, I rounded 37.7% to the nearest whole number which is 38%.", "referenced_object_masks": [ "pot-3e374f58097c50ca" ] }, { "key": "P01-20240203-184045:7b81e6fc7bb7a95795ce903f2749558199df3fef", "video_id": "P01-20240203-184045", "question": "What letters are shared by the first food item I pull from the fridge and letters on the tote bag above the pan?", "answer_choice_0": "H, I, and E", "answer_choice_1": "C, K and E", "answer_choice_2": "C, I, and E", "answer_choice_3": "K, H, and I", "answer_choice_4": "H, I and N", "answer_id": 0, "answer": "H, I, and E", "question_type": [ "Object Recognition", "Temporal Reasoning", "Reading" ], "objects_of_interest": [ "heat diffuser-c31909c9f1c68a4a", "pan-837efd3382aaf75a", "spatula-559856ff0e6b7833", "lid-8507a62c7b718116", "pot-3e374f58097c50ca", "plastic container-64474a3579aa4aa5", "chicken-9df3ae3b12d0aef0" ], "reasoning": "1. I look for the first food item pulled out of the fridge .\n2. I find it at the 00:08 timestamp to be a package of chicken (chicken-9df3ae3b12d0aef0).\n3. I look for the tote bag above the pan (pan-837efd3382aaf75a) and find a good view at 00:45.\n4. The tote bag says \"Phase Eight\" on it in pink cursive writing.\n5. \"Chicken\" and \"Phase Eight\" share three letters: H, I, and E.\n6. Therefore, H, I, and E is the answer.", "referenced_object_masks": [ "pan-837efd3382aaf75a", "chicken-9df3ae3b12d0aef0" ] }, { "key": "P01-20240203-184045:88c48a64c5f6ad8d9b904f2c7419744f2447cf7f", "video_id": "P01-20240203-184045", "question": "When I am picking up the heat diffuser, which ingredient pulled from the fridge is closest to me?", "answer_choice_0": "Rice", "answer_choice_1": "Milk", "answer_choice_2": "Ketchup", "answer_choice_3": "Soda", "answer_choice_4": "Chicken", "answer_id": 4, "answer": "Chicken", "question_type": [ "Egocentric & Spatial Perception", "Temporal Reasoning", "Object Recognition", "Situational Awareness", "Action Recognition" ], "objects_of_interest": [ "heat diffuser-c31909c9f1c68a4a", "pan-837efd3382aaf75a", "spatula-559856ff0e6b7833", "lid-8507a62c7b718116", "pot-3e374f58097c50ca", "plastic container-64474a3579aa4aa5", "chicken-9df3ae3b12d0aef0" ], "reasoning": "1. I notice the user pull 2 ingredients out of the fridge from 00:07-00:12: chicken (chicken-9df3ae3b12d0aef0) and a pot of rice (pot-3e374f58097c50ca).\n2. I notice the user place the chicken (chicken-9df3ae3b12d0aef0) to the right of the fridge at 00:16.\n3. I notice the user place the pot of rice (pot-3e374f58097c50ca) on the stove, to the right of the chicken (chicken-9df3ae3b12d0aef0), at 00:20.\n4. I notice the user pick up the heat diffuser (heat diffuser-c31909c9f1c68a4a) from 00:45-00:46.\n5. I realize that the heat diffuser (heat diffuser-c31909c9f1c68a4a) is just across from the fridge , with the chicken (chicken-9df3ae3b12d0aef0) right beside it. When the user picks up the heat diffuser (heat diffuser-c31909c9f1c68a4a), the pot of rice (pot-3e374f58097c50ca) is further back on the stove .\n6. Therefore, when the user picks up the heat diffuser (heat diffuser-c31909c9f1c68a4a), the chicken (chicken-9df3ae3b12d0aef0) is closest to the user.", "referenced_object_masks": [ "pot-3e374f58097c50ca", "chicken-9df3ae3b12d0aef0", "heat diffuser-c31909c9f1c68a4a" ] }, { "key": "P01-20240203-184214:00234ad8b51abe91cc4d81f59b249db75f2f510c", "video_id": "P01-20240203-184214", "question": "After adding garlic cloves, what is the next item I added to the bowl of chicken?", "answer_choice_0": "Herbs", "answer_choice_1": "Seasoning", "answer_choice_2": "Soy sauce", "answer_choice_3": "Water", "answer_choice_4": "More chicken", "answer_id": 3, "answer": "Water", "question_type": [ "Action Recognition", "Object Recognition", "Event Occurence" ], "objects_of_interest": [ "spoon6-33c8bfff1918aace", "fork-45a33725fa74a496", "strainer-c529a71bec097785", "water jug-12ad6038b16867c4", "yogurt tub-f67ff2b38fe2ac89", "oven glove-dbda442d3fcea7a9", "chopping board3-c1372540249ae684", "plate-44c08c86e9c3ea1a", "sink plug strainer-e1bd4587a3b15f61", "brown bowl3-dbf0ca015a75b19d", "utensil-5387e7df63d67ea5", "another spoon-d59e594e9f351f0b", "spoon5-a34483d6809a3eba", "peeler-1eeedcae304d1d22", "brush-7b9681be579c37ab", "garlic presser-cae077c2b9d154b9", "ladle-d4de839ce4831a2d", "spoon4-365b88cd9aa7b189", "pot lid-0bfdf38fce111555", "pot2-e5180d7c0497147a", "spoon3-2cb966a90c8b8255", "tray2-5a353aac35bab6f7", "tray-e66ec5449927e615", "washing up liquid bottle-827b331264949a3f", "sponge-3f2901c19c2cebc9", "left glove-db436861532d57b9", "right glove-bbad1d72146236df", "forks-43a638d22cccd234", "spoons-1787989e2bb2d9bd", "knives-51920bb35853d658", "spoon-833b72c093ab1847", "chopsticks-42ea5f4fc3505ecd", "another wooden spoon-a417baff305bdf7e", "spatula-3a8138906926877c", "two wooden spoons-77faf60e708a52a4", "wooden spoon-33f63db213143c2c", "small spoon-adfed6bd5d907172", "three spoons-9c434f4bb59013a8", "small saucepan-4b8eae2680ee23b0", "glass5-bd953e6d0d25e823", "glass4-b11bc4b06ff9941c", "mug6-11f8110fff32e10e", "mug5-f068d4e6d909ed77", "mug4-d2f5bb286ebcfdab", "mug3-0d37de48b0707a7f", "glass3-f78f94beabc9be54", "mug2-c078941b1c4cd5f9", "glass2-d61d1f1b0efc1d11", "mug-5dcfb59e918a8462", "bowl6-b09afc58215f9a7c", "tupperware-3c937a774f6335a5", "small plate2-a2e8c4448544dc78", "small plate1-6f565b0087ceb29e", "white bowl2-28daf47d63508aca", "white bowl-b9d2d5a55d01145e", "brown bowl2-c23b65d3d78d26a9", "brown bowl-92933582a073bdee", "plate7-845db95c51accf52", "plate6-20944720e4193d28", "plate5-0206cd126c7a5896", "plate4-fb52d61590bc7c9c", "plate3-147dca4c0dd72f8a", "plate2-6d017a4282405e76", "plate1-7ab8af1ed048b21a", "glass-488b336709e4872c", "bowl5-8748ce040bf79fa1", "bowl4-dfbbc9ec54966246", "bowl3-60ee87fd7bc0690d", "saucepan2-c2a19dda0fc3781c", "saucepan-906fe51394744e1f", "frying pan-2e77364cd5a75f6e", "bowl2-b8c577c4759013d1", "chopping board2-cdb1fbee4bb1634e", "chopping board-35bd00cb47bf6f04", "jars-eea006517abed7ce", "container of soup-43020600da90c96f", "cling film-3016ee5ec69afe2f", "cling film tube-43d8d388fb908930", "box of cling film-654a61f021904c84", "garlic clove2-3814360e1e87f24d", "one clove of garlic-6a7a9cda174cdbc1", "garlic cloves-3499d69f0de1cae0", "garlic head-5c91b75e7c7e193f", "garlic peel-b4e149844b17da3b", "jar of pepper-5e1e8f7779d96664", "jar of spice2-a031c86baaa63bcc", "scissors-ffdbd4f4e4440590", "plastic spoon-03f718efd8bc3de9", "lid of pot-eb5a0e4afb621ccf", "lemon half4-b1d92110fb79d30b", "lemon half3-6d32d471b8151a64", "knife2-0bdd948148543acc", "lemon squeezer-4dab8760b1cee85a", "bottle of soya sauce-3e87e92b074230f1", "lemon-01419c851f20d863", "bag of rosemary-49c8aac5dbea17ae", "bag-d25cbcf923d808c8", "second half of lemon-b56f15fa47435854", "half of lemon-ef066479f2b81109", "vinegar bottle-eb7c811ce88c6dea", "small ladder-6e30d1da29527a83", "stack of sauce packets-50f6ca05efb7a3fc", "packet-3008399ee8d4b052", "jar of spice-2923409ed1946c76", "chicken pieces3-30dfb93c005e5f3a", "two chicken legs-2699488fc25b183c", "chicken piece1-ba4f890a06630751", "packaging of chicken-118b72d19313ec0d", "knife-aacf63abf1fdb2f6", "bowl-22c376cd58a97c85", "heat diffuser-34678f4c7aa297d8", "pot-6d571bb446bb1206", "lid-f4f7ea1749546bf3", "chicken pieces4-f69f6f181316e90d", "new chicken thigh-6d5a2a025452cd9a", "top brown bowl-59608aab8e97a782" ], "reasoning": "1. I watched the video to find out when the user added garlic cloves to the bowl of chicken. I first noticed the user added chicken (chicken pieces4-f69f6f181316e90d) to a white bowl (bowl-22c376cd58a97c85) at 00:55. \n2. Since this was the bowl (bowl-22c376cd58a97c85) in question, I continued watching to find out when they added garlic to it. I saw the user grab garlic (garlic head-5c91b75e7c7e193f) at 07:47.\n3. From 08:17, I watched the user cut the garlic (one clove of garlic-6a7a9cda174cdbc1) into smaller pieces and add it to the bowl (bowl-22c376cd58a97c85). The user started cutting a second clove (garlic clove2-3814360e1e87f24d) at 09:17, finishing at 09:30.\n4. I then watched to see what the user did next. At 09:32, I saw them grab the bowl (bowl-22c376cd58a97c85) and bring it to the sink. At the sink, the user added water to the bowl at 09:38. Since this was the next ingredient, I determined that to be the answer.", "referenced_object_masks": [ "chicken pieces4-f69f6f181316e90d", "bowl-22c376cd58a97c85", "garlic clove2-3814360e1e87f24d", "one clove of garlic-6a7a9cda174cdbc1", "garlic head-5c91b75e7c7e193f" ] }, { "key": "P01-20240203-184214:141503323584b18de809a0c1e49c35baddd826d2", "video_id": "P01-20240203-184214", "question": "If I had removed silverware out of the top dishwasher tray twice as many times, how many times would I have removed silverware out of the tray?", "answer_choice_0": "20", "answer_choice_1": "14", "answer_choice_2": "18", "answer_choice_3": "16", "answer_choice_4": "12", "answer_id": 1, "answer": "14", "question_type": [ "Action Recognition", "Counting", "Egocentric & Spatial Perception", "Event Occurence", "Object Recognition", "Counterfactual" ], "objects_of_interest": [ "spoon6-33c8bfff1918aace", "fork-45a33725fa74a496", "strainer-c529a71bec097785", "water jug-12ad6038b16867c4", "yogurt tub-f67ff2b38fe2ac89", "oven glove-dbda442d3fcea7a9", "chopping board3-c1372540249ae684", "plate-44c08c86e9c3ea1a", "sink plug strainer-e1bd4587a3b15f61", "brown bowl3-dbf0ca015a75b19d", "utensil-5387e7df63d67ea5", "another spoon-d59e594e9f351f0b", "spoon5-a34483d6809a3eba", "peeler-1eeedcae304d1d22", "brush-7b9681be579c37ab", "garlic presser-cae077c2b9d154b9", "ladle-d4de839ce4831a2d", "spoon4-365b88cd9aa7b189", "pot lid-0bfdf38fce111555", "pot2-e5180d7c0497147a", "spoon3-2cb966a90c8b8255", "tray2-5a353aac35bab6f7", "tray-e66ec5449927e615", "washing up liquid bottle-827b331264949a3f", "sponge-3f2901c19c2cebc9", "left glove-db436861532d57b9", "right glove-bbad1d72146236df", "forks-43a638d22cccd234", "spoons-1787989e2bb2d9bd", "knives-51920bb35853d658", "spoon-833b72c093ab1847", "chopsticks-42ea5f4fc3505ecd", "another wooden spoon-a417baff305bdf7e", "spatula-3a8138906926877c", "two wooden spoons-77faf60e708a52a4", "wooden spoon-33f63db213143c2c", "small spoon-adfed6bd5d907172", "three spoons-9c434f4bb59013a8", "small saucepan-4b8eae2680ee23b0", "glass5-bd953e6d0d25e823", "glass4-b11bc4b06ff9941c", "mug6-11f8110fff32e10e", "mug5-f068d4e6d909ed77", "mug4-d2f5bb286ebcfdab", "mug3-0d37de48b0707a7f", "glass3-f78f94beabc9be54", "mug2-c078941b1c4cd5f9", "glass2-d61d1f1b0efc1d11", "mug-5dcfb59e918a8462", "bowl6-b09afc58215f9a7c", "tupperware-3c937a774f6335a5", "small plate2-a2e8c4448544dc78", "small plate1-6f565b0087ceb29e", "white bowl2-28daf47d63508aca", "white bowl-b9d2d5a55d01145e", "brown bowl2-c23b65d3d78d26a9", "brown bowl-92933582a073bdee", "plate7-845db95c51accf52", "plate6-20944720e4193d28", "plate5-0206cd126c7a5896", "plate4-fb52d61590bc7c9c", "plate3-147dca4c0dd72f8a", "plate2-6d017a4282405e76", "plate1-7ab8af1ed048b21a", "glass-488b336709e4872c", "bowl5-8748ce040bf79fa1", "bowl4-dfbbc9ec54966246", "bowl3-60ee87fd7bc0690d", "saucepan2-c2a19dda0fc3781c", "saucepan-906fe51394744e1f", "frying pan-2e77364cd5a75f6e", "bowl2-b8c577c4759013d1", "chopping board2-cdb1fbee4bb1634e", "chopping board-35bd00cb47bf6f04", "jars-eea006517abed7ce", "container of soup-43020600da90c96f", "cling film-3016ee5ec69afe2f", "cling film tube-43d8d388fb908930", "box of cling film-654a61f021904c84", "garlic clove2-3814360e1e87f24d", "one clove of garlic-6a7a9cda174cdbc1", "garlic cloves-3499d69f0de1cae0", "garlic head-5c91b75e7c7e193f", "garlic peel-b4e149844b17da3b", "jar of pepper-5e1e8f7779d96664", "jar of spice2-a031c86baaa63bcc", "scissors-ffdbd4f4e4440590", "plastic spoon-03f718efd8bc3de9", "lid of pot-eb5a0e4afb621ccf", "lemon half4-b1d92110fb79d30b", "lemon half3-6d32d471b8151a64", "knife2-0bdd948148543acc", "lemon squeezer-4dab8760b1cee85a", "bottle of soya sauce-3e87e92b074230f1", "lemon-01419c851f20d863", "bag of rosemary-49c8aac5dbea17ae", "bag-d25cbcf923d808c8", "second half of lemon-b56f15fa47435854", "half of lemon-ef066479f2b81109", "vinegar bottle-eb7c811ce88c6dea", "small ladder-6e30d1da29527a83", "stack of sauce packets-50f6ca05efb7a3fc", "packet-3008399ee8d4b052", "jar of spice-2923409ed1946c76", "chicken pieces3-30dfb93c005e5f3a", "two chicken legs-2699488fc25b183c", "chicken piece1-ba4f890a06630751", "packaging of chicken-118b72d19313ec0d", "knife-aacf63abf1fdb2f6", "bowl-22c376cd58a97c85", "heat diffuser-34678f4c7aa297d8", "pot-6d571bb446bb1206", "lid-f4f7ea1749546bf3", "chicken pieces4-f69f6f181316e90d", "new chicken thigh-6d5a2a025452cd9a", "top brown bowl-59608aab8e97a782" ], "reasoning": "1. I watched the video to find out when the user removed silverware out of the top dishwasher tray . I noticed the user pulled out the top dishwasher tray at 14:30.\n2. Then, I counted the number of times the user removed silverware. The first occurrence was at 14:33, when the user grabbed three spoons (three spoons-9c434f4bb59013a8) at once. Then, at 14:35, the user grabbed an additional spoon (small spoon-adfed6bd5d907172).\n3. The next silverware grab occurred at 14:44, when the user grabbed silver chopsticks (chopsticks-42ea5f4fc3505ecd). Then, the user grabbed a spoon (spoon-833b72c093ab1847) at 14:47, some knives (knives-51920bb35853d658) at 14:50, more spoons (spoons-1787989e2bb2d9bd) at 14:54, and forks (forks-43a638d22cccd234) at 14:57.\n4. Counting these all up, I determined the user removed silverware 7 times. I doubled this (7x2) to get a final answer of 14.", "referenced_object_masks": [ "forks-43a638d22cccd234", "chopsticks-42ea5f4fc3505ecd", "spoon-833b72c093ab1847", "small spoon-adfed6bd5d907172", "three spoons-9c434f4bb59013a8", "spoons-1787989e2bb2d9bd", "knives-51920bb35853d658" ] }, { "key": "P01-20240203-184214:41fa78b45c4d38ab569003bc534c8d923ed657d0", "video_id": "P01-20240203-184214", "question": "What is the fourth dish I take out of the dishwasher?", "answer_choice_0": "Pan", "answer_choice_1": "Glass", "answer_choice_2": "Plate", "answer_choice_3": "Knife", "answer_choice_4": "Bowl", "answer_id": 0, "answer": "Pan", "question_type": [ "Action Recognition", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "spoon6-33c8bfff1918aace", "fork-45a33725fa74a496", "strainer-c529a71bec097785", "water jug-12ad6038b16867c4", "yogurt tub-f67ff2b38fe2ac89", "oven glove-dbda442d3fcea7a9", "chopping board3-c1372540249ae684", "plate-44c08c86e9c3ea1a", "sink plug strainer-e1bd4587a3b15f61", "brown bowl3-dbf0ca015a75b19d", "utensil-5387e7df63d67ea5", "another spoon-d59e594e9f351f0b", "spoon5-a34483d6809a3eba", "peeler-1eeedcae304d1d22", "brush-7b9681be579c37ab", "garlic presser-cae077c2b9d154b9", "ladle-d4de839ce4831a2d", "spoon4-365b88cd9aa7b189", "pot lid-0bfdf38fce111555", "pot2-e5180d7c0497147a", "spoon3-2cb966a90c8b8255", "tray2-5a353aac35bab6f7", "tray-e66ec5449927e615", "washing up liquid bottle-827b331264949a3f", "sponge-3f2901c19c2cebc9", "left glove-db436861532d57b9", "right glove-bbad1d72146236df", "forks-43a638d22cccd234", "spoons-1787989e2bb2d9bd", "knives-51920bb35853d658", "spoon-833b72c093ab1847", "chopsticks-42ea5f4fc3505ecd", "another wooden spoon-a417baff305bdf7e", "spatula-3a8138906926877c", "two wooden spoons-77faf60e708a52a4", "wooden spoon-33f63db213143c2c", "small spoon-adfed6bd5d907172", "three spoons-9c434f4bb59013a8", "small saucepan-4b8eae2680ee23b0", "glass5-bd953e6d0d25e823", "glass4-b11bc4b06ff9941c", "mug6-11f8110fff32e10e", "mug5-f068d4e6d909ed77", "mug4-d2f5bb286ebcfdab", "mug3-0d37de48b0707a7f", "glass3-f78f94beabc9be54", "mug2-c078941b1c4cd5f9", "glass2-d61d1f1b0efc1d11", "mug-5dcfb59e918a8462", "bowl6-b09afc58215f9a7c", "tupperware-3c937a774f6335a5", "small plate2-a2e8c4448544dc78", "small plate1-6f565b0087ceb29e", "white bowl2-28daf47d63508aca", "white bowl-b9d2d5a55d01145e", "brown bowl2-c23b65d3d78d26a9", "brown bowl-92933582a073bdee", "plate7-845db95c51accf52", "plate6-20944720e4193d28", "plate5-0206cd126c7a5896", "plate4-fb52d61590bc7c9c", "plate3-147dca4c0dd72f8a", "plate2-6d017a4282405e76", "plate1-7ab8af1ed048b21a", "glass-488b336709e4872c", "bowl5-8748ce040bf79fa1", "bowl4-dfbbc9ec54966246", "bowl3-60ee87fd7bc0690d", "saucepan2-c2a19dda0fc3781c", "saucepan-906fe51394744e1f", "frying pan-2e77364cd5a75f6e", "bowl2-b8c577c4759013d1", "chopping board2-cdb1fbee4bb1634e", "chopping board-35bd00cb47bf6f04", "jars-eea006517abed7ce", "container of soup-43020600da90c96f", "cling film-3016ee5ec69afe2f", "cling film tube-43d8d388fb908930", "box of cling film-654a61f021904c84", "garlic clove2-3814360e1e87f24d", "one clove of garlic-6a7a9cda174cdbc1", "garlic cloves-3499d69f0de1cae0", "garlic head-5c91b75e7c7e193f", "garlic peel-b4e149844b17da3b", "jar of pepper-5e1e8f7779d96664", "jar of spice2-a031c86baaa63bcc", "scissors-ffdbd4f4e4440590", "plastic spoon-03f718efd8bc3de9", "lid of pot-eb5a0e4afb621ccf", "lemon half4-b1d92110fb79d30b", "lemon half3-6d32d471b8151a64", "knife2-0bdd948148543acc", "lemon squeezer-4dab8760b1cee85a", "bottle of soya sauce-3e87e92b074230f1", "lemon-01419c851f20d863", "bag of rosemary-49c8aac5dbea17ae", "bag-d25cbcf923d808c8", "second half of lemon-b56f15fa47435854", "half of lemon-ef066479f2b81109", "vinegar bottle-eb7c811ce88c6dea", "small ladder-6e30d1da29527a83", "stack of sauce packets-50f6ca05efb7a3fc", "packet-3008399ee8d4b052", "jar of spice-2923409ed1946c76", "chicken pieces3-30dfb93c005e5f3a", "two chicken legs-2699488fc25b183c", "chicken piece1-ba4f890a06630751", "packaging of chicken-118b72d19313ec0d", "knife-aacf63abf1fdb2f6", "bowl-22c376cd58a97c85", "heat diffuser-34678f4c7aa297d8", "pot-6d571bb446bb1206", "lid-f4f7ea1749546bf3", "chicken pieces4-f69f6f181316e90d", "new chicken thigh-6d5a2a025452cd9a", "top brown bowl-59608aab8e97a782" ], "reasoning": "1. I watched the video to see that the user begins unloading the dishwasher at 12:11.\n2. First, I saw the user grab two cutting boards (chopping board-35bd00cb47bf6f04, chopping board2-cdb1fbee4bb1634e) from the dishwasher , putting them away from 12:13-12:21.\n3. Next, from 12:24-12:28, I observed the user pick up and put away a large white bowl (bowl2-b8c577c4759013d1).\n4. Then, from 12:32-12:33, the user picks up a pan (frying pan-2e77364cd5a75f6e), making it the fourth item grabbed from the dishwasher.", "referenced_object_masks": [ "frying pan-2e77364cd5a75f6e", "bowl2-b8c577c4759013d1", "chopping board-35bd00cb47bf6f04", "chopping board2-cdb1fbee4bb1634e" ] }, { "key": "P01-20240203-184214:6feab8e41eb6391041bcd8331b7bd40c28905bb0", "video_id": "P01-20240203-184214", "question": "What is the third-to-last kitchen utensil that I touched in the video?", "answer_choice_0": "A ladle", "answer_choice_1": "A fork", "answer_choice_2": "A pot", "answer_choice_3": "A strainer", "answer_choice_4": "A spoon", "answer_id": 3, "answer": "A strainer", "question_type": [ "Action Recognition", "Counting", "Event Occurence", "Object Recognition" ], "objects_of_interest": [ "spoon6-33c8bfff1918aace", "fork-45a33725fa74a496", "strainer-c529a71bec097785", "water jug-12ad6038b16867c4", "yogurt tub-f67ff2b38fe2ac89", "oven glove-dbda442d3fcea7a9", "chopping board3-c1372540249ae684", "plate-44c08c86e9c3ea1a", "sink plug strainer-e1bd4587a3b15f61", "brown bowl3-dbf0ca015a75b19d", "utensil-5387e7df63d67ea5", "another spoon-d59e594e9f351f0b", "spoon5-a34483d6809a3eba", "peeler-1eeedcae304d1d22", "brush-7b9681be579c37ab", "garlic presser-cae077c2b9d154b9", "ladle-d4de839ce4831a2d", "spoon4-365b88cd9aa7b189", "pot lid-0bfdf38fce111555", "pot2-e5180d7c0497147a", "spoon3-2cb966a90c8b8255", "tray2-5a353aac35bab6f7", "tray-e66ec5449927e615", "washing up liquid bottle-827b331264949a3f", "sponge-3f2901c19c2cebc9", "left glove-db436861532d57b9", "right glove-bbad1d72146236df", "forks-43a638d22cccd234", "spoons-1787989e2bb2d9bd", "knives-51920bb35853d658", "spoon-833b72c093ab1847", "chopsticks-42ea5f4fc3505ecd", "another wooden spoon-a417baff305bdf7e", "spatula-3a8138906926877c", "two wooden spoons-77faf60e708a52a4", "wooden spoon-33f63db213143c2c", "small spoon-adfed6bd5d907172", "three spoons-9c434f4bb59013a8", "small saucepan-4b8eae2680ee23b0", "glass5-bd953e6d0d25e823", "glass4-b11bc4b06ff9941c", "mug6-11f8110fff32e10e", "mug5-f068d4e6d909ed77", "mug4-d2f5bb286ebcfdab", "mug3-0d37de48b0707a7f", "glass3-f78f94beabc9be54", "mug2-c078941b1c4cd5f9", "glass2-d61d1f1b0efc1d11", "mug-5dcfb59e918a8462", "bowl6-b09afc58215f9a7c", "tupperware-3c937a774f6335a5", "small plate2-a2e8c4448544dc78", "small plate1-6f565b0087ceb29e", "white bowl2-28daf47d63508aca", "white bowl-b9d2d5a55d01145e", "brown bowl2-c23b65d3d78d26a9", "brown bowl-92933582a073bdee", "plate7-845db95c51accf52", "plate6-20944720e4193d28", "plate5-0206cd126c7a5896", "plate4-fb52d61590bc7c9c", "plate3-147dca4c0dd72f8a", "plate2-6d017a4282405e76", "plate1-7ab8af1ed048b21a", "glass-488b336709e4872c", "bowl5-8748ce040bf79fa1", "bowl4-dfbbc9ec54966246", "bowl3-60ee87fd7bc0690d", "saucepan2-c2a19dda0fc3781c", "saucepan-906fe51394744e1f", "frying pan-2e77364cd5a75f6e", "bowl2-b8c577c4759013d1", "chopping board2-cdb1fbee4bb1634e", "chopping board-35bd00cb47bf6f04", "jars-eea006517abed7ce", "container of soup-43020600da90c96f", "cling film-3016ee5ec69afe2f", "cling film tube-43d8d388fb908930", "box of cling film-654a61f021904c84", "garlic clove2-3814360e1e87f24d", "one clove of garlic-6a7a9cda174cdbc1", "garlic cloves-3499d69f0de1cae0", "garlic head-5c91b75e7c7e193f", "garlic peel-b4e149844b17da3b", "jar of pepper-5e1e8f7779d96664", "jar of spice2-a031c86baaa63bcc", "scissors-ffdbd4f4e4440590", "plastic spoon-03f718efd8bc3de9", "lid of pot-eb5a0e4afb621ccf", "lemon half4-b1d92110fb79d30b", "lemon half3-6d32d471b8151a64", "knife2-0bdd948148543acc", "lemon squeezer-4dab8760b1cee85a", "bottle of soya sauce-3e87e92b074230f1", "lemon-01419c851f20d863", "bag of rosemary-49c8aac5dbea17ae", "bag-d25cbcf923d808c8", "second half of lemon-b56f15fa47435854", "half of lemon-ef066479f2b81109", "vinegar bottle-eb7c811ce88c6dea", "small ladder-6e30d1da29527a83", "stack of sauce packets-50f6ca05efb7a3fc", "packet-3008399ee8d4b052", "jar of spice-2923409ed1946c76", "chicken pieces3-30dfb93c005e5f3a", "two chicken legs-2699488fc25b183c", "chicken piece1-ba4f890a06630751", "packaging of chicken-118b72d19313ec0d", "knife-aacf63abf1fdb2f6", "bowl-22c376cd58a97c85", "heat diffuser-34678f4c7aa297d8", "pot-6d571bb446bb1206", "lid-f4f7ea1749546bf3", "chicken pieces4-f69f6f181316e90d", "new chicken thigh-6d5a2a025452cd9a", "top brown bowl-59608aab8e97a782" ], "reasoning": "1. I watched the video to find out what the third-to-last kitchen utensil was that the user touched in the video. As the video ended at 22:09, I looked back from that point.\n2. At 21:43, I noticed the user grabbing a spoon (spoon6-33c8bfff1918aace) from a drawer. Then, at 21:47, I saw the user scooping out yogurt from a container (yogurt tub-f67ff2b38fe2ac89) using the spoon (spoon6-33c8bfff1918aace). Since the user does not touch another utensil after this point, I determined that this was the final utensil they touched.\n3. Right before this, at 21:39, I noticed the user grabbing a fork (fork-45a33725fa74a496) from a drawer. Since this was the last utensil they touched before the spoon (spoon6-33c8bfff1918aace), I determined this was the second-to-last utensil they touched.\n4. Moving backward more, I noticed the user picked up a strainer (strainer-c529a71bec097785) at 21:31. Then, they placed the strainer (strainer-c529a71bec097785) into the sink at 21:34. Since this was the last utensil they touched before the fork (fork-45a33725fa74a496), I determined this was the third-to-last kitchen utensil that they touched.", "referenced_object_masks": [ "spoon6-33c8bfff1918aace", "fork-45a33725fa74a496", "yogurt tub-f67ff2b38fe2ac89", "strainer-c529a71bec097785" ] }, { "key": "P01-20240203-184214:7af053701a456d2a80ea3b74899859da74e97e05", "video_id": "P01-20240203-184214", "question": "After I finish juicing the second lemon half, how long does it take for me to pull out a box of cling film?", "answer_choice_0": "5 minutes and 35 seconds", "answer_choice_1": "6 minutes and 26 seconds", "answer_choice_2": "4 minutes and 55 seconds", "answer_choice_3": "6 minutes and 15 seconds", "answer_choice_4": "5 minutes and 6 seconds", "answer_id": 0, "answer": "5 minutes and 35 seconds", "question_type": [ "Numerical Reasoning", "Object Recognition", "Temporal Reasoning", "Action Recognition" ], "objects_of_interest": [ "spoon6-33c8bfff1918aace", "fork-45a33725fa74a496", "strainer-c529a71bec097785", "water jug-12ad6038b16867c4", "yogurt tub-f67ff2b38fe2ac89", "oven glove-dbda442d3fcea7a9", "chopping board3-c1372540249ae684", "plate-44c08c86e9c3ea1a", "sink plug strainer-e1bd4587a3b15f61", "brown bowl3-dbf0ca015a75b19d", "utensil-5387e7df63d67ea5", "another spoon-d59e594e9f351f0b", "spoon5-a34483d6809a3eba", "peeler-1eeedcae304d1d22", "brush-7b9681be579c37ab", "garlic presser-cae077c2b9d154b9", "ladle-d4de839ce4831a2d", "spoon4-365b88cd9aa7b189", "pot lid-0bfdf38fce111555", "pot2-e5180d7c0497147a", "spoon3-2cb966a90c8b8255", "tray2-5a353aac35bab6f7", "tray-e66ec5449927e615", "washing up liquid bottle-827b331264949a3f", "sponge-3f2901c19c2cebc9", "left glove-db436861532d57b9", "right glove-bbad1d72146236df", "forks-43a638d22cccd234", "spoons-1787989e2bb2d9bd", "knives-51920bb35853d658", "spoon-833b72c093ab1847", "chopsticks-42ea5f4fc3505ecd", "another wooden spoon-a417baff305bdf7e", "spatula-3a8138906926877c", "two wooden spoons-77faf60e708a52a4", "wooden spoon-33f63db213143c2c", "small spoon-adfed6bd5d907172", "three spoons-9c434f4bb59013a8", "small saucepan-4b8eae2680ee23b0", "glass5-bd953e6d0d25e823", "glass4-b11bc4b06ff9941c", "mug6-11f8110fff32e10e", "mug5-f068d4e6d909ed77", "mug4-d2f5bb286ebcfdab", "mug3-0d37de48b0707a7f", "glass3-f78f94beabc9be54", "mug2-c078941b1c4cd5f9", "glass2-d61d1f1b0efc1d11", "mug-5dcfb59e918a8462", "bowl6-b09afc58215f9a7c", "tupperware-3c937a774f6335a5", "small plate2-a2e8c4448544dc78", "small plate1-6f565b0087ceb29e", "white bowl2-28daf47d63508aca", "white bowl-b9d2d5a55d01145e", "brown bowl2-c23b65d3d78d26a9", "brown bowl-92933582a073bdee", "plate7-845db95c51accf52", "plate6-20944720e4193d28", "plate5-0206cd126c7a5896", "plate4-fb52d61590bc7c9c", "plate3-147dca4c0dd72f8a", "plate2-6d017a4282405e76", "plate1-7ab8af1ed048b21a", "glass-488b336709e4872c", "bowl5-8748ce040bf79fa1", "bowl4-dfbbc9ec54966246", "bowl3-60ee87fd7bc0690d", "saucepan2-c2a19dda0fc3781c", "saucepan-906fe51394744e1f", "frying pan-2e77364cd5a75f6e", "bowl2-b8c577c4759013d1", "chopping board2-cdb1fbee4bb1634e", "chopping board-35bd00cb47bf6f04", "jars-eea006517abed7ce", "container of soup-43020600da90c96f", "cling film-3016ee5ec69afe2f", "cling film tube-43d8d388fb908930", "box of cling film-654a61f021904c84", "garlic clove2-3814360e1e87f24d", "one clove of garlic-6a7a9cda174cdbc1", "garlic cloves-3499d69f0de1cae0", "garlic head-5c91b75e7c7e193f", "garlic peel-b4e149844b17da3b", "jar of pepper-5e1e8f7779d96664", "jar of spice2-a031c86baaa63bcc", "scissors-ffdbd4f4e4440590", "plastic spoon-03f718efd8bc3de9", "lid of pot-eb5a0e4afb621ccf", "lemon half4-b1d92110fb79d30b", "lemon half3-6d32d471b8151a64", "knife2-0bdd948148543acc", "lemon squeezer-4dab8760b1cee85a", "bottle of soya sauce-3e87e92b074230f1", "lemon-01419c851f20d863", "bag of rosemary-49c8aac5dbea17ae", "bag-d25cbcf923d808c8", "second half of lemon-b56f15fa47435854", "half of lemon-ef066479f2b81109", "vinegar bottle-eb7c811ce88c6dea", "small ladder-6e30d1da29527a83", "stack of sauce packets-50f6ca05efb7a3fc", "packet-3008399ee8d4b052", "jar of spice-2923409ed1946c76", "chicken pieces3-30dfb93c005e5f3a", "two chicken legs-2699488fc25b183c", "chicken piece1-ba4f890a06630751", "packaging of chicken-118b72d19313ec0d", "knife-aacf63abf1fdb2f6", "bowl-22c376cd58a97c85", "heat diffuser-34678f4c7aa297d8", "pot-6d571bb446bb1206", "lid-f4f7ea1749546bf3", "chicken pieces4-f69f6f181316e90d", "new chicken thigh-6d5a2a025452cd9a", "top brown bowl-59608aab8e97a782" ], "reasoning": "1. I watched the video, looking for times when the user juices a lemon. I found the first such instance from 03:56-04:24, when they use a lemon squeezer (lemon squeezer-4dab8760b1cee85a) to squeeze the juice from a lemon half. (half of lemon-ef066479f2b81109).\n2. Then, I looked for another moment when the user juices a second lemon half (second half of lemon-b56f15fa47435854). I found this from 04:29-04:35. They pause to do another action, and then they juice the lemon (second half of lemon-b56f15fa47435854) again from 04:45-05:15.\n3. Since the user finished juicing the second lemon half (second half of lemon-b56f15fa47435854) at 05:15, I then looked after that point for when they pulled out a box of cling film (box of cling film-654a61f021904c84). I found this at 10:50.\n4. Then, I subtracted 05:15 from 10:50 in order to get the final answer, which is 5 minutes and 35 seconds.", "referenced_object_masks": [ "half of lemon-ef066479f2b81109", "box of cling film-654a61f021904c84", "lemon squeezer-4dab8760b1cee85a", "second half of lemon-b56f15fa47435854" ] }, { "key": "P01-20240203-184214:8a049d0c996381b86ce8d2a05bd9ed690b21971f", "video_id": "P01-20240203-184214", "question": "When I am rinsing the chicken, in what relative direction is the microwave from me?", "answer_choice_0": "3 o'clock", "answer_choice_1": "4 o'clock", "answer_choice_2": "6 o'clock", "answer_choice_3": "2 o'clock", "answer_choice_4": "1 o'clock", "answer_id": 1, "answer": "4 o'clock", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Object Recognition" ], "objects_of_interest": [ "spoon6-33c8bfff1918aace", "fork-45a33725fa74a496", "strainer-c529a71bec097785", "water jug-12ad6038b16867c4", "yogurt tub-f67ff2b38fe2ac89", "oven glove-dbda442d3fcea7a9", "chopping board3-c1372540249ae684", "plate-44c08c86e9c3ea1a", "sink plug strainer-e1bd4587a3b15f61", "brown bowl3-dbf0ca015a75b19d", "utensil-5387e7df63d67ea5", "another spoon-d59e594e9f351f0b", "spoon5-a34483d6809a3eba", "peeler-1eeedcae304d1d22", "brush-7b9681be579c37ab", "garlic presser-cae077c2b9d154b9", "ladle-d4de839ce4831a2d", "spoon4-365b88cd9aa7b189", "pot lid-0bfdf38fce111555", "pot2-e5180d7c0497147a", "spoon3-2cb966a90c8b8255", "tray2-5a353aac35bab6f7", "tray-e66ec5449927e615", "washing up liquid bottle-827b331264949a3f", "sponge-3f2901c19c2cebc9", "left glove-db436861532d57b9", "right glove-bbad1d72146236df", "forks-43a638d22cccd234", "spoons-1787989e2bb2d9bd", "knives-51920bb35853d658", "spoon-833b72c093ab1847", "chopsticks-42ea5f4fc3505ecd", "another wooden spoon-a417baff305bdf7e", "spatula-3a8138906926877c", "two wooden spoons-77faf60e708a52a4", "wooden spoon-33f63db213143c2c", "small spoon-adfed6bd5d907172", "three spoons-9c434f4bb59013a8", "small saucepan-4b8eae2680ee23b0", "glass5-bd953e6d0d25e823", "glass4-b11bc4b06ff9941c", "mug6-11f8110fff32e10e", "mug5-f068d4e6d909ed77", "mug4-d2f5bb286ebcfdab", "mug3-0d37de48b0707a7f", "glass3-f78f94beabc9be54", "mug2-c078941b1c4cd5f9", "glass2-d61d1f1b0efc1d11", "mug-5dcfb59e918a8462", "bowl6-b09afc58215f9a7c", "tupperware-3c937a774f6335a5", "small plate2-a2e8c4448544dc78", "small plate1-6f565b0087ceb29e", "white bowl2-28daf47d63508aca", "white bowl-b9d2d5a55d01145e", "brown bowl2-c23b65d3d78d26a9", "brown bowl-92933582a073bdee", "plate7-845db95c51accf52", "plate6-20944720e4193d28", "plate5-0206cd126c7a5896", "plate4-fb52d61590bc7c9c", "plate3-147dca4c0dd72f8a", "plate2-6d017a4282405e76", "plate1-7ab8af1ed048b21a", "glass-488b336709e4872c", "bowl5-8748ce040bf79fa1", "bowl4-dfbbc9ec54966246", "bowl3-60ee87fd7bc0690d", "saucepan2-c2a19dda0fc3781c", "saucepan-906fe51394744e1f", "frying pan-2e77364cd5a75f6e", "bowl2-b8c577c4759013d1", "chopping board2-cdb1fbee4bb1634e", "chopping board-35bd00cb47bf6f04", "jars-eea006517abed7ce", "container of soup-43020600da90c96f", "cling film-3016ee5ec69afe2f", "cling film tube-43d8d388fb908930", "box of cling film-654a61f021904c84", "garlic clove2-3814360e1e87f24d", "one clove of garlic-6a7a9cda174cdbc1", "garlic cloves-3499d69f0de1cae0", "garlic head-5c91b75e7c7e193f", "garlic peel-b4e149844b17da3b", "jar of pepper-5e1e8f7779d96664", "jar of spice2-a031c86baaa63bcc", "scissors-ffdbd4f4e4440590", "plastic spoon-03f718efd8bc3de9", "lid of pot-eb5a0e4afb621ccf", "lemon half4-b1d92110fb79d30b", "lemon half3-6d32d471b8151a64", "knife2-0bdd948148543acc", "lemon squeezer-4dab8760b1cee85a", "bottle of soya sauce-3e87e92b074230f1", "lemon-01419c851f20d863", "bag of rosemary-49c8aac5dbea17ae", "bag-d25cbcf923d808c8", "second half of lemon-b56f15fa47435854", "half of lemon-ef066479f2b81109", "vinegar bottle-eb7c811ce88c6dea", "small ladder-6e30d1da29527a83", "stack of sauce packets-50f6ca05efb7a3fc", "packet-3008399ee8d4b052", "jar of spice-2923409ed1946c76", "chicken pieces3-30dfb93c005e5f3a", "two chicken legs-2699488fc25b183c", "chicken piece1-ba4f890a06630751", "packaging of chicken-118b72d19313ec0d", "knife-aacf63abf1fdb2f6", "bowl-22c376cd58a97c85", "heat diffuser-34678f4c7aa297d8", "pot-6d571bb446bb1206", "lid-f4f7ea1749546bf3", "chicken pieces4-f69f6f181316e90d", "new chicken thigh-6d5a2a025452cd9a", "top brown bowl-59608aab8e97a782" ], "reasoning": "1. I watched the video to observe the user opening the pack of chicken (packaging of chicken-118b72d19313ec0d) from 00:28-00:32.\n2. Next, the user took the pack of chicken (packaging of chicken-118b72d19313ec0d) and moved it to the sink , from 00:32-00:37.\n3. During this span, at 00:33, the microwave is visible on the counter.\n4. I then observed the user turn approximately 90 degrees to the left and walk to the sink , placing the microwave behind and to the right to the user, at appromiately 4 o'clock.", "referenced_object_masks": [ "packaging of chicken-118b72d19313ec0d" ] }, { "key": "P01-20240203-184214:a6d9af6e7affbd3c1562c282b8d65329a5d4c812", "video_id": "P01-20240203-184214", "question": "When seasoning the chicken, how many spice jars do I touch with my left hand?", "answer_choice_0": "4", "answer_choice_1": "5", "answer_choice_2": "7", "answer_choice_3": "6", "answer_choice_4": "3", "answer_id": 3, "answer": "6", "question_type": [ "Counting", "Action Recognition", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "spoon6-33c8bfff1918aace", "fork-45a33725fa74a496", "strainer-c529a71bec097785", "water jug-12ad6038b16867c4", "yogurt tub-f67ff2b38fe2ac89", "oven glove-dbda442d3fcea7a9", "chopping board3-c1372540249ae684", "plate-44c08c86e9c3ea1a", "sink plug strainer-e1bd4587a3b15f61", "brown bowl3-dbf0ca015a75b19d", "utensil-5387e7df63d67ea5", "another spoon-d59e594e9f351f0b", "spoon5-a34483d6809a3eba", "peeler-1eeedcae304d1d22", "brush-7b9681be579c37ab", "garlic presser-cae077c2b9d154b9", "ladle-d4de839ce4831a2d", "spoon4-365b88cd9aa7b189", "pot lid-0bfdf38fce111555", "pot2-e5180d7c0497147a", "spoon3-2cb966a90c8b8255", "tray2-5a353aac35bab6f7", "tray-e66ec5449927e615", "washing up liquid bottle-827b331264949a3f", "sponge-3f2901c19c2cebc9", "left glove-db436861532d57b9", "right glove-bbad1d72146236df", "forks-43a638d22cccd234", "spoons-1787989e2bb2d9bd", "knives-51920bb35853d658", "spoon-833b72c093ab1847", "chopsticks-42ea5f4fc3505ecd", "another wooden spoon-a417baff305bdf7e", "spatula-3a8138906926877c", "two wooden spoons-77faf60e708a52a4", "wooden spoon-33f63db213143c2c", "small spoon-adfed6bd5d907172", "three spoons-9c434f4bb59013a8", "small saucepan-4b8eae2680ee23b0", "glass5-bd953e6d0d25e823", "glass4-b11bc4b06ff9941c", "mug6-11f8110fff32e10e", "mug5-f068d4e6d909ed77", "mug4-d2f5bb286ebcfdab", "mug3-0d37de48b0707a7f", "glass3-f78f94beabc9be54", "mug2-c078941b1c4cd5f9", "glass2-d61d1f1b0efc1d11", "mug-5dcfb59e918a8462", "bowl6-b09afc58215f9a7c", "tupperware-3c937a774f6335a5", "small plate2-a2e8c4448544dc78", "small plate1-6f565b0087ceb29e", "white bowl2-28daf47d63508aca", "white bowl-b9d2d5a55d01145e", "brown bowl2-c23b65d3d78d26a9", "brown bowl-92933582a073bdee", "plate7-845db95c51accf52", "plate6-20944720e4193d28", "plate5-0206cd126c7a5896", "plate4-fb52d61590bc7c9c", "plate3-147dca4c0dd72f8a", "plate2-6d017a4282405e76", "plate1-7ab8af1ed048b21a", "glass-488b336709e4872c", "bowl5-8748ce040bf79fa1", "bowl4-dfbbc9ec54966246", "bowl3-60ee87fd7bc0690d", "saucepan2-c2a19dda0fc3781c", "saucepan-906fe51394744e1f", "frying pan-2e77364cd5a75f6e", "bowl2-b8c577c4759013d1", "chopping board2-cdb1fbee4bb1634e", "chopping board-35bd00cb47bf6f04", "jars-eea006517abed7ce", "container of soup-43020600da90c96f", "cling film-3016ee5ec69afe2f", "cling film tube-43d8d388fb908930", "box of cling film-654a61f021904c84", "garlic clove2-3814360e1e87f24d", "one clove of garlic-6a7a9cda174cdbc1", "garlic cloves-3499d69f0de1cae0", "garlic head-5c91b75e7c7e193f", "garlic peel-b4e149844b17da3b", "jar of pepper-5e1e8f7779d96664", "jar of spice2-a031c86baaa63bcc", "scissors-ffdbd4f4e4440590", "plastic spoon-03f718efd8bc3de9", "lid of pot-eb5a0e4afb621ccf", "lemon half4-b1d92110fb79d30b", "lemon half3-6d32d471b8151a64", "knife2-0bdd948148543acc", "lemon squeezer-4dab8760b1cee85a", "bottle of soya sauce-3e87e92b074230f1", "lemon-01419c851f20d863", "bag of rosemary-49c8aac5dbea17ae", "bag-d25cbcf923d808c8", "second half of lemon-b56f15fa47435854", "half of lemon-ef066479f2b81109", "vinegar bottle-eb7c811ce88c6dea", "small ladder-6e30d1da29527a83", "stack of sauce packets-50f6ca05efb7a3fc", "packet-3008399ee8d4b052", "jar of spice-2923409ed1946c76", "chicken pieces3-30dfb93c005e5f3a", "two chicken legs-2699488fc25b183c", "chicken piece1-ba4f890a06630751", "packaging of chicken-118b72d19313ec0d", "knife-aacf63abf1fdb2f6", "bowl-22c376cd58a97c85", "heat diffuser-34678f4c7aa297d8", "pot-6d571bb446bb1206", "lid-f4f7ea1749546bf3", "chicken pieces4-f69f6f181316e90d", "new chicken thigh-6d5a2a025452cd9a", "top brown bowl-59608aab8e97a782" ], "reasoning": "1. I watched the video to see when the user seasoned the chicken, noting that it occurred from 06:58-07:46.\n2. The user opened the spice cabinet at 07:00, touching a jar (jar of pepper-5e1e8f7779d96664) with their left hand at 07:02, marking this as the first spice jar touched with the left hand.\n3. Then, the user lifts another jar with the left hand from 07:04-07:06. This is the second jar touched with the left hand.\n4. Quickly, the user grabs another jar with their left hand at 07:08, grabbing it and pulling it out of the cabinet. This is the third jar touched with the left hand.\n5. While holding this jar , the user touches another jar in the front row with a couple fingers of their left hand at 07:10, making this jar number 4 touched with the left hand.\n6. From 07:10-07:24, the user parses through the jars of spices with their right hand until 07:24, when they put the jar in their left hand back on the shelf.\n7. Then, at 07:25, the user touches the new jar with both hands, including their left hand. This is the fifth jar touched with the left hand.\n8. From 07:26-07:34, the user seasons the chicken and then returns to the cabinet, where they grab another jar with both hands from 07:33-07:34. This is the sixth jar the user touches with the left hand.\n9. Lastly, the user returns this jar to the cabinet from 07:45-07:46, marking the end of seasoning. A total of six spice jars are touched with their left hand.", "referenced_object_masks": [ "jar of pepper-5e1e8f7779d96664" ] }, { "key": "P01-20240203-184214:dd688144c0ec0a774ae75d17b6d881a9efce11c8", "video_id": "P01-20240203-184214", "question": "If I wanted to put away the large, orange item sitting to the right of the bowl when I am seasoning the chicken, behind which kitchen item would I put it?", "answer_choice_0": "Refrigerator", "answer_choice_1": "Dishwasher", "answer_choice_2": "Sink", "answer_choice_3": "Stovetop", "answer_choice_4": "Microwave", "answer_id": 2, "answer": "Sink", "question_type": [ "Counterfactual", "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "spoon6-33c8bfff1918aace", "fork-45a33725fa74a496", "strainer-c529a71bec097785", "water jug-12ad6038b16867c4", "yogurt tub-f67ff2b38fe2ac89", "oven glove-dbda442d3fcea7a9", "chopping board3-c1372540249ae684", "plate-44c08c86e9c3ea1a", "sink plug strainer-e1bd4587a3b15f61", "brown bowl3-dbf0ca015a75b19d", "utensil-5387e7df63d67ea5", "another spoon-d59e594e9f351f0b", "spoon5-a34483d6809a3eba", "peeler-1eeedcae304d1d22", "brush-7b9681be579c37ab", "garlic presser-cae077c2b9d154b9", "ladle-d4de839ce4831a2d", "spoon4-365b88cd9aa7b189", "pot lid-0bfdf38fce111555", "pot2-e5180d7c0497147a", "spoon3-2cb966a90c8b8255", "tray2-5a353aac35bab6f7", "tray-e66ec5449927e615", "washing up liquid bottle-827b331264949a3f", "sponge-3f2901c19c2cebc9", "left glove-db436861532d57b9", "right glove-bbad1d72146236df", "forks-43a638d22cccd234", "spoons-1787989e2bb2d9bd", "knives-51920bb35853d658", "spoon-833b72c093ab1847", "chopsticks-42ea5f4fc3505ecd", "another wooden spoon-a417baff305bdf7e", "spatula-3a8138906926877c", "two wooden spoons-77faf60e708a52a4", "wooden spoon-33f63db213143c2c", "small spoon-adfed6bd5d907172", "three spoons-9c434f4bb59013a8", "small saucepan-4b8eae2680ee23b0", "glass5-bd953e6d0d25e823", "glass4-b11bc4b06ff9941c", "mug6-11f8110fff32e10e", "mug5-f068d4e6d909ed77", "mug4-d2f5bb286ebcfdab", "mug3-0d37de48b0707a7f", "glass3-f78f94beabc9be54", "mug2-c078941b1c4cd5f9", "glass2-d61d1f1b0efc1d11", "mug-5dcfb59e918a8462", "bowl6-b09afc58215f9a7c", "tupperware-3c937a774f6335a5", "small plate2-a2e8c4448544dc78", "small plate1-6f565b0087ceb29e", "white bowl2-28daf47d63508aca", "white bowl-b9d2d5a55d01145e", "brown bowl2-c23b65d3d78d26a9", "brown bowl-92933582a073bdee", "plate7-845db95c51accf52", "plate6-20944720e4193d28", "plate5-0206cd126c7a5896", "plate4-fb52d61590bc7c9c", "plate3-147dca4c0dd72f8a", "plate2-6d017a4282405e76", "plate1-7ab8af1ed048b21a", "glass-488b336709e4872c", "bowl5-8748ce040bf79fa1", "bowl4-dfbbc9ec54966246", "bowl3-60ee87fd7bc0690d", "saucepan2-c2a19dda0fc3781c", "saucepan-906fe51394744e1f", "frying pan-2e77364cd5a75f6e", "bowl2-b8c577c4759013d1", "chopping board2-cdb1fbee4bb1634e", "chopping board-35bd00cb47bf6f04", "jars-eea006517abed7ce", "container of soup-43020600da90c96f", "cling film-3016ee5ec69afe2f", "cling film tube-43d8d388fb908930", "box of cling film-654a61f021904c84", "garlic clove2-3814360e1e87f24d", "one clove of garlic-6a7a9cda174cdbc1", "garlic cloves-3499d69f0de1cae0", "garlic head-5c91b75e7c7e193f", "garlic peel-b4e149844b17da3b", "jar of pepper-5e1e8f7779d96664", "jar of spice2-a031c86baaa63bcc", "scissors-ffdbd4f4e4440590", "plastic spoon-03f718efd8bc3de9", "lid of pot-eb5a0e4afb621ccf", "lemon half4-b1d92110fb79d30b", "lemon half3-6d32d471b8151a64", "knife2-0bdd948148543acc", "lemon squeezer-4dab8760b1cee85a", "bottle of soya sauce-3e87e92b074230f1", "lemon-01419c851f20d863", "bag of rosemary-49c8aac5dbea17ae", "bag-d25cbcf923d808c8", "second half of lemon-b56f15fa47435854", "half of lemon-ef066479f2b81109", "vinegar bottle-eb7c811ce88c6dea", "small ladder-6e30d1da29527a83", "stack of sauce packets-50f6ca05efb7a3fc", "packet-3008399ee8d4b052", "jar of spice-2923409ed1946c76", "chicken pieces3-30dfb93c005e5f3a", "two chicken legs-2699488fc25b183c", "chicken piece1-ba4f890a06630751", "packaging of chicken-118b72d19313ec0d", "knife-aacf63abf1fdb2f6", "bowl-22c376cd58a97c85", "heat diffuser-34678f4c7aa297d8", "pot-6d571bb446bb1206", "lid-f4f7ea1749546bf3", "chicken pieces4-f69f6f181316e90d", "new chicken thigh-6d5a2a025452cd9a", "top brown bowl-59608aab8e97a782" ], "reasoning": "1. I watched the video to observe the user seasoning the chicken (packaging of chicken-118b72d19313ec0d) from 06:48-07:46.\n2. During this span, I noted the large item to the right of the bowl (bowl-22c376cd58a97c85) as an orange oven mitt (oven glove-dbda442d3fcea7a9).\n3. Then, after seasoning the chicken (packaging of chicken-118b72d19313ec0d), the user walks left to the kitchen sink from 09:30-09:35.\n4. Once at the sink, at 09:35, I observed other oven gloves hanging on the wall behind the sink , indicating that the orange oven glove (oven glove-dbda442d3fcea7a9) would be stored there.", "referenced_object_masks": [ "packaging of chicken-118b72d19313ec0d", "bowl-22c376cd58a97c85", "oven glove-dbda442d3fcea7a9" ] }, { "key": "P01-20240203-184214:e8cc08ca70376d4f085d3ec7a32240b5f556d930", "video_id": "P01-20240203-184214", "question": "After I stirred the rice in the pot for the third time, what did I pick up next?", "answer_choice_0": "Yogurt", "answer_choice_1": "A plate", "answer_choice_2": "A spoon", "answer_choice_3": "The bowl with chicken", "answer_choice_4": "Gloves", "answer_id": 4, "answer": "Gloves", "question_type": [ "Counting", "Action Recognition", "Event Occurence", "Object Recognition" ], "objects_of_interest": [ "spoon6-33c8bfff1918aace", "fork-45a33725fa74a496", "strainer-c529a71bec097785", "water jug-12ad6038b16867c4", "yogurt tub-f67ff2b38fe2ac89", "oven glove-dbda442d3fcea7a9", "chopping board3-c1372540249ae684", "plate-44c08c86e9c3ea1a", "sink plug strainer-e1bd4587a3b15f61", "brown bowl3-dbf0ca015a75b19d", "utensil-5387e7df63d67ea5", "another spoon-d59e594e9f351f0b", "spoon5-a34483d6809a3eba", "peeler-1eeedcae304d1d22", "brush-7b9681be579c37ab", "garlic presser-cae077c2b9d154b9", "ladle-d4de839ce4831a2d", "spoon4-365b88cd9aa7b189", "pot lid-0bfdf38fce111555", "pot2-e5180d7c0497147a", "spoon3-2cb966a90c8b8255", "tray2-5a353aac35bab6f7", "tray-e66ec5449927e615", "washing up liquid bottle-827b331264949a3f", "sponge-3f2901c19c2cebc9", "left glove-db436861532d57b9", "right glove-bbad1d72146236df", "forks-43a638d22cccd234", "spoons-1787989e2bb2d9bd", "knives-51920bb35853d658", "spoon-833b72c093ab1847", "chopsticks-42ea5f4fc3505ecd", "another wooden spoon-a417baff305bdf7e", "spatula-3a8138906926877c", "two wooden spoons-77faf60e708a52a4", "wooden spoon-33f63db213143c2c", "small spoon-adfed6bd5d907172", "three spoons-9c434f4bb59013a8", "small saucepan-4b8eae2680ee23b0", "glass5-bd953e6d0d25e823", "glass4-b11bc4b06ff9941c", "mug6-11f8110fff32e10e", "mug5-f068d4e6d909ed77", "mug4-d2f5bb286ebcfdab", "mug3-0d37de48b0707a7f", "glass3-f78f94beabc9be54", "mug2-c078941b1c4cd5f9", "glass2-d61d1f1b0efc1d11", "mug-5dcfb59e918a8462", "bowl6-b09afc58215f9a7c", "tupperware-3c937a774f6335a5", "small plate2-a2e8c4448544dc78", "small plate1-6f565b0087ceb29e", "white bowl2-28daf47d63508aca", "white bowl-b9d2d5a55d01145e", "brown bowl2-c23b65d3d78d26a9", "brown bowl-92933582a073bdee", "plate7-845db95c51accf52", "plate6-20944720e4193d28", "plate5-0206cd126c7a5896", "plate4-fb52d61590bc7c9c", "plate3-147dca4c0dd72f8a", "plate2-6d017a4282405e76", "plate1-7ab8af1ed048b21a", "glass-488b336709e4872c", "bowl5-8748ce040bf79fa1", "bowl4-dfbbc9ec54966246", "bowl3-60ee87fd7bc0690d", "saucepan2-c2a19dda0fc3781c", "saucepan-906fe51394744e1f", "frying pan-2e77364cd5a75f6e", "bowl2-b8c577c4759013d1", "chopping board2-cdb1fbee4bb1634e", "chopping board-35bd00cb47bf6f04", "jars-eea006517abed7ce", "container of soup-43020600da90c96f", "cling film-3016ee5ec69afe2f", "cling film tube-43d8d388fb908930", "box of cling film-654a61f021904c84", "garlic clove2-3814360e1e87f24d", "one clove of garlic-6a7a9cda174cdbc1", "garlic cloves-3499d69f0de1cae0", "garlic head-5c91b75e7c7e193f", "garlic peel-b4e149844b17da3b", "jar of pepper-5e1e8f7779d96664", "jar of spice2-a031c86baaa63bcc", "scissors-ffdbd4f4e4440590", "plastic spoon-03f718efd8bc3de9", "lid of pot-eb5a0e4afb621ccf", "lemon half4-b1d92110fb79d30b", "lemon half3-6d32d471b8151a64", "knife2-0bdd948148543acc", "lemon squeezer-4dab8760b1cee85a", "bottle of soya sauce-3e87e92b074230f1", "lemon-01419c851f20d863", "bag of rosemary-49c8aac5dbea17ae", "bag-d25cbcf923d808c8", "second half of lemon-b56f15fa47435854", "half of lemon-ef066479f2b81109", "vinegar bottle-eb7c811ce88c6dea", "small ladder-6e30d1da29527a83", "stack of sauce packets-50f6ca05efb7a3fc", "packet-3008399ee8d4b052", "jar of spice-2923409ed1946c76", "chicken pieces3-30dfb93c005e5f3a", "two chicken legs-2699488fc25b183c", "chicken piece1-ba4f890a06630751", "packaging of chicken-118b72d19313ec0d", "knife-aacf63abf1fdb2f6", "bowl-22c376cd58a97c85", "heat diffuser-34678f4c7aa297d8", "pot-6d571bb446bb1206", "lid-f4f7ea1749546bf3", "chicken pieces4-f69f6f181316e90d", "new chicken thigh-6d5a2a025452cd9a", "top brown bowl-59608aab8e97a782" ], "reasoning": "1. I watched the video to look for the times when the user stirred the rice in the pot. I noticed the user put a closed pot (pot-6d571bb446bb1206) on a burner at 00:08 and then lit it at 00:11.\n2. At 06:12, I noticed the user opening the closed pot (pot-6d571bb446bb1206) for the first time, revealing a small helping of rice. At this time, she stirred the rice, so I counted this as the first occurrence.\n3. At 10:30, I noticed the user opening the closed pot (pot-6d571bb446bb1206) and stirring the rice again. I counted this as the second occurrence.\n4. At 15:10, I noticed the user opening the closed pot (pot-6d571bb446bb1206) and stirring the rice again. I counted this as the third occurrence.\n5. Since this was the third occurrence, I watched to see what the user picked up next. At 15:29, I noticed the user grabbing a glove (right glove-bbad1d72146236df) at the sink to put it on. Then, they grabbed the other glove (left glove-db436861532d57b9). Since this was the first action to appear from the options available, I determined that to be the answer.", "referenced_object_masks": [ "pot-6d571bb446bb1206", "right glove-bbad1d72146236df", "left glove-db436861532d57b9" ] }, { "key": "P01-20240203-184214:f15e904cf519876208c9eadebb6d101fd5ffdf51", "video_id": "P01-20240203-184214", "question": "What is the step-ladder leaning against after I use it?", "answer_choice_0": "Closet door", "answer_choice_1": "Kitchen table", "answer_choice_2": "Dishwasher", "answer_choice_3": "Refrigerator", "answer_choice_4": "Oven door", "answer_id": 0, "answer": "Closet door", "question_type": [ "State Changes", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "spoon6-33c8bfff1918aace", "fork-45a33725fa74a496", "strainer-c529a71bec097785", "water jug-12ad6038b16867c4", "yogurt tub-f67ff2b38fe2ac89", "oven glove-dbda442d3fcea7a9", "chopping board3-c1372540249ae684", "plate-44c08c86e9c3ea1a", "sink plug strainer-e1bd4587a3b15f61", "brown bowl3-dbf0ca015a75b19d", "utensil-5387e7df63d67ea5", "another spoon-d59e594e9f351f0b", "spoon5-a34483d6809a3eba", "peeler-1eeedcae304d1d22", "brush-7b9681be579c37ab", "garlic presser-cae077c2b9d154b9", "ladle-d4de839ce4831a2d", "spoon4-365b88cd9aa7b189", "pot lid-0bfdf38fce111555", "pot2-e5180d7c0497147a", "spoon3-2cb966a90c8b8255", "tray2-5a353aac35bab6f7", "tray-e66ec5449927e615", "washing up liquid bottle-827b331264949a3f", "sponge-3f2901c19c2cebc9", "left glove-db436861532d57b9", "right glove-bbad1d72146236df", "forks-43a638d22cccd234", "spoons-1787989e2bb2d9bd", "knives-51920bb35853d658", "spoon-833b72c093ab1847", "chopsticks-42ea5f4fc3505ecd", "another wooden spoon-a417baff305bdf7e", "spatula-3a8138906926877c", "two wooden spoons-77faf60e708a52a4", "wooden spoon-33f63db213143c2c", "small spoon-adfed6bd5d907172", "three spoons-9c434f4bb59013a8", "small saucepan-4b8eae2680ee23b0", "glass5-bd953e6d0d25e823", "glass4-b11bc4b06ff9941c", "mug6-11f8110fff32e10e", "mug5-f068d4e6d909ed77", "mug4-d2f5bb286ebcfdab", "mug3-0d37de48b0707a7f", "glass3-f78f94beabc9be54", "mug2-c078941b1c4cd5f9", "glass2-d61d1f1b0efc1d11", "mug-5dcfb59e918a8462", "bowl6-b09afc58215f9a7c", "tupperware-3c937a774f6335a5", "small plate2-a2e8c4448544dc78", "small plate1-6f565b0087ceb29e", "white bowl2-28daf47d63508aca", "white bowl-b9d2d5a55d01145e", "brown bowl2-c23b65d3d78d26a9", "brown bowl-92933582a073bdee", "plate7-845db95c51accf52", "plate6-20944720e4193d28", "plate5-0206cd126c7a5896", "plate4-fb52d61590bc7c9c", "plate3-147dca4c0dd72f8a", "plate2-6d017a4282405e76", "plate1-7ab8af1ed048b21a", "glass-488b336709e4872c", "bowl5-8748ce040bf79fa1", "bowl4-dfbbc9ec54966246", "bowl3-60ee87fd7bc0690d", "saucepan2-c2a19dda0fc3781c", "saucepan-906fe51394744e1f", "frying pan-2e77364cd5a75f6e", "bowl2-b8c577c4759013d1", "chopping board2-cdb1fbee4bb1634e", "chopping board-35bd00cb47bf6f04", "jars-eea006517abed7ce", "container of soup-43020600da90c96f", "cling film-3016ee5ec69afe2f", "cling film tube-43d8d388fb908930", "box of cling film-654a61f021904c84", "garlic clove2-3814360e1e87f24d", "one clove of garlic-6a7a9cda174cdbc1", "garlic cloves-3499d69f0de1cae0", "garlic head-5c91b75e7c7e193f", "garlic peel-b4e149844b17da3b", "jar of pepper-5e1e8f7779d96664", "jar of spice2-a031c86baaa63bcc", "scissors-ffdbd4f4e4440590", "plastic spoon-03f718efd8bc3de9", "lid of pot-eb5a0e4afb621ccf", "lemon half4-b1d92110fb79d30b", "lemon half3-6d32d471b8151a64", "knife2-0bdd948148543acc", "lemon squeezer-4dab8760b1cee85a", "bottle of soya sauce-3e87e92b074230f1", "lemon-01419c851f20d863", "bag of rosemary-49c8aac5dbea17ae", "bag-d25cbcf923d808c8", "second half of lemon-b56f15fa47435854", "half of lemon-ef066479f2b81109", "vinegar bottle-eb7c811ce88c6dea", "small ladder-6e30d1da29527a83", "stack of sauce packets-50f6ca05efb7a3fc", "packet-3008399ee8d4b052", "jar of spice-2923409ed1946c76", "chicken pieces3-30dfb93c005e5f3a", "two chicken legs-2699488fc25b183c", "chicken piece1-ba4f890a06630751", "packaging of chicken-118b72d19313ec0d", "knife-aacf63abf1fdb2f6", "bowl-22c376cd58a97c85", "heat diffuser-34678f4c7aa297d8", "pot-6d571bb446bb1206", "lid-f4f7ea1749546bf3", "chicken pieces4-f69f6f181316e90d", "new chicken thigh-6d5a2a025452cd9a", "top brown bowl-59608aab8e97a782" ], "reasoning": "1. I watched the video to see the user grab the step-ladder (small ladder-6e30d1da29527a83) from the office from 02:00-02:04.\n2. After grabbing the ladder (small ladder-6e30d1da29527a83), the user takes it and places it in front of a cabinet from 02:04-02:10.\n3. After grabbing a bottle of vinegar (vinegar bottle-eb7c811ce88c6dea) from 02:11-02:18.\n4. Then, from 02:25-02:47, the user measures and pours vinegar (vinegar bottle-eb7c811ce88c6dea) onto the chicken .\n5. Next, the user takes the vinegar (vinegar bottle-eb7c811ce88c6dea) back to the cabinet from 02:47-02:53.\n6. The user then dismounts the step-ladder (small ladder-6e30d1da29527a83) and folds it up, leaning it against the black closet door in the kitchen from 02:55-02:58.", "referenced_object_masks": [ "small ladder-6e30d1da29527a83", "vinegar bottle-eb7c811ce88c6dea" ] }, { "key": "P01-20240204-095114:229b0f74284b6bcd49dd92db9767689537150bf1", "video_id": "P01-20240204-095114", "question": "What is the third thing I touch with my left hand?", "answer_choice_0": "A coffee mug", "answer_choice_1": "A bottle of milk", "answer_choice_2": "A milk frother", "answer_choice_3": "A coffee maker", "answer_choice_4": "The refrigerator", "answer_id": 0, "answer": "A coffee mug", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Counting" ], "objects_of_interest": [ "wooden stirrer-c47272c3386185c7", "spoon-3e8c9aa942ad9214", "cereal box-f06220f7df3ed723", "bowl-85ba16a0fda99d7c", "plug-56a5ca6e2354ff53", "lid-b5a502a483e03578", "milk frother-c0a7d37b507937ac", "mug-328d666b3cb39e7c", "milk bottle-0eaae18eb7564494" ], "reasoning": "1. I watched the video and counted the times I saw the user touch things with their left hand.\n2. I saw the user first open the refrigerator with their left hand at 00:06, then remove a jug of milk (milk bottle-0eaae18eb7564494) from inside with their left hand at 00:08. They pick up a mug (mug-328d666b3cb39e7c) with their left hand at 00:13.\n3. Therefore the third thing they touch with their left hand is a mug (mug-328d666b3cb39e7c).", "referenced_object_masks": [ "mug-328d666b3cb39e7c", "milk bottle-0eaae18eb7564494" ] }, { "key": "P01-20240204-095114:30142a57dbd2e5127d54de0e570e14f2f3f05ef8", "video_id": "P01-20240204-095114", "question": "What item is directly below the highest item I grab when I pick that item up for the first time?", "answer_choice_0": "A plate", "answer_choice_1": "A coffee mug", "answer_choice_2": "An oven mitt", "answer_choice_3": "A drying rack", "answer_choice_4": "A bottle of cooking oil", "answer_id": 4, "answer": "A bottle of cooking oil", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Object Recognition" ], "objects_of_interest": [ "wooden stirrer-c47272c3386185c7", "spoon-3e8c9aa942ad9214", "cereal box-f06220f7df3ed723", "bowl-85ba16a0fda99d7c", "plug-56a5ca6e2354ff53", "lid-b5a502a483e03578", "milk frother-c0a7d37b507937ac", "mug-328d666b3cb39e7c", "milk bottle-0eaae18eb7564494" ], "reasoning": "1. I watched the video to determine when the user grabs something from up high.\n2. At 01:04 I can see the user fully extend their right arm upward to grab a box of cereal (cereal box-f06220f7df3ed723) from a shelf. At no other point does the user extend their arm as far upward, so this must be the highest item they grab.\n3. I looked below the user's arm to the shelf below and can see a bottle of cooking oil on the shelf immediately below it.", "referenced_object_masks": [ "cereal box-f06220f7df3ed723" ] }, { "key": "P01-20240204-095114:74c8e6ee0505e6d8d514282a1df2f4d4347fd6b4", "video_id": "P01-20240204-095114", "question": "Excluding the bottle from the refrigerator, how many different vessels hold milk by the end of the video?", "answer_choice_0": "2", "answer_choice_1": "4", "answer_choice_2": "0", "answer_choice_3": "1", "answer_choice_4": "3", "answer_id": 4, "answer": "3", "question_type": [ "Counting", "Object Recognition" ], "objects_of_interest": [ "wooden stirrer-c47272c3386185c7", "spoon-3e8c9aa942ad9214", "cereal box-f06220f7df3ed723", "bowl-85ba16a0fda99d7c", "plug-56a5ca6e2354ff53", "lid-b5a502a483e03578", "milk frother-c0a7d37b507937ac", "mug-328d666b3cb39e7c", "milk bottle-0eaae18eb7564494" ], "reasoning": "1. I watched the video and looked for segments where the user was pouring milk.\n2. I can see the user pour milk into the frother (milk frother-c0a7d37b507937ac) at 00:28, into a bowl (bowl-85ba16a0fda99d7c) at 00:50, and into a mug (mug-328d666b3cb39e7c) at 01:56.\n3. Therefore, the answer is 3 vessels.", "referenced_object_masks": [ "milk frother-c0a7d37b507937ac", "bowl-85ba16a0fda99d7c", "mug-328d666b3cb39e7c" ] }, { "key": "P01-20240204-095114:a8629d207152b25f1b8f7e1550fe8a1b6dbe8dab", "video_id": "P01-20240204-095114", "question": "How many things do I stir clockwise with my right hand?", "answer_choice_0": "4", "answer_choice_1": "1", "answer_choice_2": "2", "answer_choice_3": "3", "answer_choice_4": "0", "answer_id": 1, "answer": "1", "question_type": [ "Action Recognition", "Counting", "Event Occurence", "Object Recognition" ], "objects_of_interest": [ "wooden stirrer-c47272c3386185c7", "spoon-3e8c9aa942ad9214", "cereal box-f06220f7df3ed723", "bowl-85ba16a0fda99d7c", "plug-56a5ca6e2354ff53", "lid-b5a502a483e03578", "milk frother-c0a7d37b507937ac", "mug-328d666b3cb39e7c", "milk bottle-0eaae18eb7564494" ], "reasoning": "1. I watched the video and looked for instances of the user stirring things in a clear clockwise direction with their right hand.\n2. I see the user stir their coffee (mug-328d666b3cb39e7c) with a wooden stirrer (wooden stirrer-c47272c3386185c7) in a clockwise direction from 02:31-02:33.\n3. I watched the rest of the video and did not see the user stir any other things in a clockwise direction, therefore the answer is 1.", "referenced_object_masks": [ "mug-328d666b3cb39e7c", "wooden stirrer-c47272c3386185c7" ] }, { "key": "P01-20240204-095114:cf580792959a5336e5525658719a7e53e3f2e19c", "video_id": "P01-20240204-095114", "question": "How many more things do I touch that are resting on the wooden counter to the left of the stove than on the right?", "answer_choice_0": "4", "answer_choice_1": "5", "answer_choice_2": "8", "answer_choice_3": "7", "answer_choice_4": "6", "answer_id": 3, "answer": "7", "question_type": [ "Action Recognition", "Counting", "Egocentric & Spatial Perception", "Numerical Reasoning", "Object Recognition" ], "objects_of_interest": [ "wooden stirrer-c47272c3386185c7", "spoon-3e8c9aa942ad9214", "cereal box-f06220f7df3ed723", "bowl-85ba16a0fda99d7c", "plug-56a5ca6e2354ff53", "lid-b5a502a483e03578", "milk frother-c0a7d37b507937ac", "mug-328d666b3cb39e7c", "milk bottle-0eaae18eb7564494" ], "reasoning": "1. I watched the video and counted the things that the user touched on either side of the stove .\n2. On the left side of the stove , I saw the user touch the coffee maker at 00:18, the milk frother (milk frother-c0a7d37b507937ac) at 00:19, the box of coffee pods , a jug of milk (milk bottle-0eaae18eb7564494) at 00:26, the milk frother plug (plug-56a5ca6e2354ff53) at 00:35, the box of stirrer sticks at 01:42, and a coffee mug (mug-328d666b3cb39e7c) at 01:54 for a total of 7 things.\n3. I did not see the user touch anything resting on the counter to the right of the stove .\n4. I subtracted 0 from 7 to reach a total of 7.", "referenced_object_masks": [ "milk frother-c0a7d37b507937ac", "mug-328d666b3cb39e7c", "plug-56a5ca6e2354ff53", "milk bottle-0eaae18eb7564494" ] }, { "key": "P01-20240204-120411:06c5d37fe7b3b87a4f9b7adc2a5cc2abbcf50b80", "video_id": "P01-20240204-120411", "question": "When I retrieve the glass from the cabinet, where is the cupboard where the bag of rice is kept in relation to me?", "answer_choice_0": "In front of me", "answer_choice_1": "On my right", "answer_choice_2": "On my left", "answer_choice_3": "Above my head", "answer_choice_4": "Behind me", "answer_id": 4, "answer": "Behind me", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "saucepan-7a0aa6ace18eb430", "pan1-3b8a0ddfbe6292bb", "jug of water-3aefd0f85c03f7fc", "black flask-48fdccc0ea7c3e15", "kettle-e885a7ecf53a2046", "salt bottle-2a413f263184eb61", "clipper-bf625587a7d1aff6", "glass-7c8bcd2feb5989d8", "bag of rice-72d7033242d16fa8", "small tupperware-2e3d5e6fd7c93708", "tupperware-40f4d47f6d954343", "plate-fabe78dc1b88ea2e", "blue chopping board-6c178931f245edc4", "peeler-423c6af903141e28", "plastic cover of peeler-c26cf2fb9f9b8f3e", "card box-5977d999fccdf085", "fuse extension-f4658ec83e1d1aec", "hand blender piece-a5c8fb71e528d748", "bowl-46eb790e4b52cc7a", "mug-c3f977c97141d70c" ], "reasoning": "1. First, I determine when the user retrieves the glass (glass-7c8bcd2feb5989d8) from the cabinet. This occurs at 02:14, where the user is facing the left side of the kitchen.\n2. Next, I determine where the bag of rice (bag of rice-72d7033242d16fa8) is stored. The user retrieves the rice (bag of rice-72d7033242d16fa8) from the bottom cabinet on the right of the kitchen at 02:10. \n3. Since the user turned around 180 degrees to access the cabinet with the glass (glass-7c8bcd2feb5989d8), the cupboard with the rice (bag of rice-72d7033242d16fa8) is behind the user.", "referenced_object_masks": [ "glass-7c8bcd2feb5989d8", "bag of rice-72d7033242d16fa8" ] }, { "key": "P01-20240204-120411:2f2026e8e43c70888998df6791becebabebd5abb", "video_id": "P01-20240204-120411", "question": "What appliances do I face on my right side each separate time I pour rice from the bag?", "answer_choice_0": "Kettle and microwave", "answer_choice_1": "Coffee pot and kettle", "answer_choice_2": "Microwave and airfryer", "answer_choice_3": "Kettle and blender", "answer_choice_4": "Airfryer and blender", "answer_id": 0, "answer": "Kettle and microwave", "question_type": [ "Object Recognition", "Egocentric & Spatial Perception", "Situational Awareness", "Action Recognition" ], "objects_of_interest": [ "saucepan-7a0aa6ace18eb430", "pan1-3b8a0ddfbe6292bb", "jug of water-3aefd0f85c03f7fc", "black flask-48fdccc0ea7c3e15", "kettle-e885a7ecf53a2046", "salt bottle-2a413f263184eb61", "clipper-bf625587a7d1aff6", "glass-7c8bcd2feb5989d8", "bag of rice-72d7033242d16fa8", "small tupperware-2e3d5e6fd7c93708", "tupperware-40f4d47f6d954343", "plate-fabe78dc1b88ea2e", "blue chopping board-6c178931f245edc4", "peeler-423c6af903141e28", "plastic cover of peeler-c26cf2fb9f9b8f3e", "card box-5977d999fccdf085", "fuse extension-f4658ec83e1d1aec", "hand blender piece-a5c8fb71e528d748", "bowl-46eb790e4b52cc7a", "mug-c3f977c97141d70c" ], "reasoning": "1. The first time a user pours rice from the bag (bag of rice-72d7033242d16fa8) occurs at 02:26 when the user pours it into a glass (glass-7c8bcd2feb5989d8) sitting in a small plastic container (small tupperware-2e3d5e6fd7c93708). In this first instance, the user is facing a black and chrome kettle (kettle-e885a7ecf53a2046) that is positioned on the counter on the right. \n2. The second time a user pours rice from the bag (bag of rice-72d7033242d16fa8) occurs at 02:53 when they pour it into a plastic container (small tupperware-2e3d5e6fd7c93708). Here, a chrome microwave is on the counter to the right of the user.\n3. There is no other time the user pours from the bag of rice (bag of rice-72d7033242d16fa8).", "referenced_object_masks": [ "kettle-e885a7ecf53a2046", "small tupperware-2e3d5e6fd7c93708", "glass-7c8bcd2feb5989d8", "bag of rice-72d7033242d16fa8" ] }, { "key": "P01-20240204-120411:41fc8028e2c67e8ae22bbc6cee790f83570af2d3", "video_id": "P01-20240204-120411", "question": "What other objects handled after drying my hands could I have used to soak the rice, instead of the one I used, to lessen the number of items I would have to wash later?", "answer_choice_0": "Rice cooker", "answer_choice_1": "Pot", "answer_choice_2": "Pitcher", "answer_choice_3": "Bowl", "answer_choice_4": "Measuring cup", "answer_id": 1, "answer": "Pot", "question_type": [ "Object Recognition", "Counterfactual", "Action Recognition", "Goal Reasoning" ], "objects_of_interest": [ "saucepan-7a0aa6ace18eb430", "pan1-3b8a0ddfbe6292bb", "jug of water-3aefd0f85c03f7fc", "black flask-48fdccc0ea7c3e15", "kettle-e885a7ecf53a2046", "salt bottle-2a413f263184eb61", "clipper-bf625587a7d1aff6", "glass-7c8bcd2feb5989d8", "bag of rice-72d7033242d16fa8", "small tupperware-2e3d5e6fd7c93708", "tupperware-40f4d47f6d954343", "plate-fabe78dc1b88ea2e", "blue chopping board-6c178931f245edc4", "peeler-423c6af903141e28", "plastic cover of peeler-c26cf2fb9f9b8f3e", "card box-5977d999fccdf085", "fuse extension-f4658ec83e1d1aec", "hand blender piece-a5c8fb71e528d748", "bowl-46eb790e4b52cc7a", "mug-c3f977c97141d70c" ], "reasoning": "1. I watched the video to determine what object the user first used to soak the rice (bag of rice-72d7033242d16fa8). \n2. At 02:53, the user pours the rice (bag of rice-72d7033242d16fa8) into a small plastic container (small tupperware-2e3d5e6fd7c93708), fills it with water at 02:59, and then sets it on the counter at 03:06 to soak. \n3. Next, I determine when the user dries their hands. This occurs at 03:23. \n4. The next object the user retrieves from the cabinet is a pot (saucepan-7a0aa6ace18eb430) at 03:45. The user could have used this same pot (saucepan-7a0aa6ace18eb430) to soak the rice (bag of rice-72d7033242d16fa8), lessening the amount of items to wash later. Also, if the user would later cook the rice (bag of rice-72d7033242d16fa8), the user has now washed and cooked the rice in the same pot (saucepan-7a0aa6ace18eb430).", "referenced_object_masks": [ "small tupperware-2e3d5e6fd7c93708", "bag of rice-72d7033242d16fa8", "saucepan-7a0aa6ace18eb430" ] }, { "key": "P01-20240204-120411:70bfa73e00f00d178a7414db4a20d31a47d6cfd6", "video_id": "P01-20240204-120411", "question": "Where, in relation to me when I grab it, is the last item I handle with the same color as my chopping board?", "answer_choice_0": "Plate, above my head", "answer_choice_1": "Salt lid, below my waist", "answer_choice_2": "Rice bag, above my head", "answer_choice_3": "Water pitcher, on my right", "answer_choice_4": "Dish towel, on my left", "answer_id": 3, "answer": "Water pitcher, on my right", "question_type": [ "Object Recognition", "Action Recognition", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "saucepan-7a0aa6ace18eb430", "pan1-3b8a0ddfbe6292bb", "jug of water-3aefd0f85c03f7fc", "black flask-48fdccc0ea7c3e15", "kettle-e885a7ecf53a2046", "salt bottle-2a413f263184eb61", "clipper-bf625587a7d1aff6", "glass-7c8bcd2feb5989d8", "bag of rice-72d7033242d16fa8", "small tupperware-2e3d5e6fd7c93708", "tupperware-40f4d47f6d954343", "plate-fabe78dc1b88ea2e", "blue chopping board-6c178931f245edc4", "peeler-423c6af903141e28", "plastic cover of peeler-c26cf2fb9f9b8f3e", "card box-5977d999fccdf085", "fuse extension-f4658ec83e1d1aec", "hand blender piece-a5c8fb71e528d748", "bowl-46eb790e4b52cc7a", "mug-c3f977c97141d70c" ], "reasoning": "1. The user handles a cutting board (blue chopping board-6c178931f245edc4) at 01:21, retrieving it from a drawer. I determine the cutting board (blue chopping board-6c178931f245edc4) is blue. \n2. Next, I determine all the blue items the user touches after that: a plate (plate-fabe78dc1b88ea2e) at 01:24, a bag of rice (bag of rice-72d7033242d16fa8) at 02:09, lid on a container of salt (salt bottle-2a413f263184eb61) at 03:11, and the water pitcher (jug of water-3aefd0f85c03f7fc) at 03:37. \n3. The water pitcher is the last blue item the user touched, and it is positioned on the user's right.", "referenced_object_masks": [ "salt bottle-2a413f263184eb61", "plate-fabe78dc1b88ea2e", "bag of rice-72d7033242d16fa8", "blue chopping board-6c178931f245edc4", "jug of water-3aefd0f85c03f7fc" ] }, { "key": "P01-20240204-120411:750afc64df3d4dcb7c368d7da03ac5ad9980e7f3", "video_id": "P01-20240204-120411", "question": "I'm creating a table setting for 6 guests. If I use matching plates like the 1 I picked up at 01:24 and ignore the one on the counter near the sink, how many additional plates would I need in my cupboard to make sure each guest has a plate?", "answer_choice_0": "2", "answer_choice_1": "None", "answer_choice_2": "4", "answer_choice_3": "1", "answer_choice_4": "3", "answer_id": 0, "answer": "2", "question_type": [ "Object Recognition", "Situational Awareness", "Counterfactual", "Numerical Reasoning" ], "objects_of_interest": [ "saucepan-7a0aa6ace18eb430", "pan1-3b8a0ddfbe6292bb", "jug of water-3aefd0f85c03f7fc", "black flask-48fdccc0ea7c3e15", "kettle-e885a7ecf53a2046", "salt bottle-2a413f263184eb61", "clipper-bf625587a7d1aff6", "glass-7c8bcd2feb5989d8", "bag of rice-72d7033242d16fa8", "small tupperware-2e3d5e6fd7c93708", "tupperware-40f4d47f6d954343", "plate-fabe78dc1b88ea2e", "blue chopping board-6c178931f245edc4", "peeler-423c6af903141e28", "plastic cover of peeler-c26cf2fb9f9b8f3e", "card box-5977d999fccdf085", "fuse extension-f4658ec83e1d1aec", "hand blender piece-a5c8fb71e528d748", "bowl-46eb790e4b52cc7a", "mug-c3f977c97141d70c" ], "reasoning": "1. First, I locate the 01:24 time stamp to determine the type of plate (plate-fabe78dc1b88ea2e) the user will use to serve their guests. The plate (plate-fabe78dc1b88ea2e) is light blue. \n2. Next, I locate an interior shot of the cupboard that shows additional plates. I found this shot at 01:43 when the user opens the cupboard and reveals 3 more matching light blue plates . \n3. Adding those plates together makes a total of 4 matching light blue plates in total, ignoring the plate at the sink at 00:07.\n4. If the user is serving 6 guests, I subtract 4 from 6 which equals 2. \n5. The user needs 2 more light blue plates to serve all their guests.", "referenced_object_masks": [ "plate-fabe78dc1b88ea2e" ] }, { "key": "P01-20240204-121042:25f22e65fb996b121c57566cc4aac2fee44d3e0d", "video_id": "P01-20240204-121042", "question": "What is the order of the following actions from the longest duration to the shortest?", "answer_choice_0": "weighing dates, removing pits from dates, washing dates", "answer_choice_1": "removing pits from dates, washing dates, weighing dates", "answer_choice_2": "washing dates, removing pits from dates, weighing dates", "answer_choice_3": "removing pits from dates, weighing dates, washing dates", "answer_choice_4": "washing dates, weighing dates, removing pits from dates", "answer_id": 3, "answer": "removing pits from dates, weighing dates, washing dates", "question_type": [ "Action Recognition", "Temporal Reasoning", "Object Recognition" ], "objects_of_interest": [ "bag of bagels-153c0673fdd61477", "medicine box-a3d8fcaa7aad548d", "glass5-d7419820dc2c2641", "tupperware-152598eab58ac975", "lid of tupperware-c7e0c66d804d4e6a", "roll of kitchen towel-65dd1a2f043c18dc", "can opener-a5b94da9cd002cb7", "piece of plastic-9da6992cbd62fa8d", "oranges mesh2-8a6cf06a891a70fa", "oranges mesh-8ebdee21e22805ba", "scissors guard-246ff4a57cbce4ad", "scissors-bb8255e02ae1e4ae", "blade cover2-1d9e63e2b58a5123", "blade cover1-c7d0d86a80bf0c0f", "plastic piece circular-53271ebfb0d03eb1", "glass4-1700579fc9dddb25", "tin of cocoa powder-0c144f3fc7cbbd66", "bowl3-ea4d5dca7d1fb5b6", "plate4-d749259fc8ca2033", "salt bottle-2df853a63e0f2951", "soup packets box-bebacc4e1b2fc36e", "jar-b346c4ac3de06bae", "packet-a0581a0332e74695", "mug-0e8f1bb541fa3ae9", "plate3-2840bba104267155", "knife4-c1f0b9e79aba6417", "large knife-780aa12f43058319", "chopping board2-0b038e808fd27ccc", "baking tray-109b3f8aab229736", "saucepan3-709ea5142ba420c8", "plate2-3f813a620242f873", "pot-210720946e78f619", "large plastic spoon-e9f1a2d844db1353", "strainer-380139925db102d4", "fork2-ffeca99db06d5dd2", "knife3-5502c51df2590d66", "fork-0a9345c7b8cd5880", "washing up liquid bottle-8547e271ebec77c9", "sponge-04aff40e7c88fd05", "spoon-6dbb321dd6b43d48", "left glove-9ac2e9e9046d5f25", "right glove-3ee9b1fdbc1dfef4", "plastic bag2-05805d3737847b90", "chopping board-f8de86a2d50d4ed9", "glass3-d669ab66f4974fc2", "large plastic bowl-16df4d9d4af0e47b", "lid of food processor-ee24ee2a177c4dc9", "biscuits3-e683eacaf91a71c4", "biscuits2-11ece4abb786ae11", "second packet of biscuits-b79a6911948f131d", "one tube of biscuits-43d560e9bfd95aab", "new packet of biscuits-16867c2cb8adb2ba", "plastic packaging2-eada36ce4f8f73e5", "biscuits1-2f44aa8e57cb6b98", "plastic packaging-149dac4d39715129", "blade-7b1da9ab2a4df098", "saucepan2-9da700035a9786b3", "towel-00625522044ee91f", "plate-94998c22011a2b48", "lid2-b841e1dc27a59183", "lid-7fb7c1b6985182e4", "food processor plug-5efeebd54e423176", "food processor-62c72d1a1a65365b", "centre piece of food processor-f176edcdf638e3a8", "food processor bowl-e5a819f6be32d6e9", "wooden spatula-08a837e2faee1615", "one fallen date-60fdca734b28ede9", "pitted dates20-d8b699b34e79c319", "pitted dates19-d366f77432264f09", "pitted dates18-3ef341bd5f56fa4b", "pitted dates17-d5e248a04b63b96b", "pitted dates16-bf91f9ff1d0f5528", "pitted dates15-ed2ae9851cbbfde9", "pitted dates14-ba3300155986a412", "pitted dates13-1afc573162d3c0cb", "pitted dates12-bd78da1768e2a388", "pit23-60a5ac1615e34a07", "date23-b8f8cd9918a89fa0", "pit22-b214e8c39519c7c6", "date22-332d25b1af8723a9", "pit21-e0ad27141d45f334", "date21-58e03f20607143da", "pit20-d48d7ec92570fffd", "date20-34d2f4160ec2b610", "pit19-7d79ff0908e1f810", "date19-ebd1e8368e774501", "remaining dates-280db61b861bfb14", "few pitted dates11-6c47844eb5473eab", "few pitted dates10-06b3c845e1fe2226", "few pitted dates9-237e6ce575b658f5", "few pitted dates8-1218a8655ba27a48", "few pitted dates7-5a66788e54540761", "few pitted dates6-dc3f6fe36a481b5f", "pitted date5-6b3ed4fcf5f9a86e", "pitted date4-27e75bea33e5c90f", "pitted date3-8a0d58d957ec7697", "two pitted dates-69bbb53ab894e261", "handful of pitted dates-b55a740c8e3fb0f0", "glass2-86974b5f919d4e60", "glass1-0ac4b6c1e31f8bd4", "pit18-9a43224881f13bdd", "date18-27f5c16b635948b9", "pit17-91f81d267577509f", "date17-9e023ce14d581242", "pit16-ae4215e902509d3d", "date16-8b1065206ddefb90", "pit15-305264da14998461", "date15-f538d6ce88b034a5", "pit14-d270e2a6687c24f9", "date14-54127f608e4d0caf", "pit13-256856040eec03c8", "date13-8d1028ecbc426468", "pit12-5d712e79351c4d81", "date12-5e5e0a7f8913e8c5", "pit11-e23810ca2db561be", "date11-part-7999e894a0df25c5", "date11-5de863c9e90b2879", "pit10-63299058578ae8ab", "date10-part-2351ed68668bd070", "date10-7d36aa35cd8718d4", "pit9-0c92c3471984818a", "date9-53c1beb54cd290d7", "pit8-4ed09c1e3af29c4e", "date8-be88c56d1dd28da4", "pit7-51b1c77a2cc5d7c0", "date7-75b6f299949b1e8e", "pit6-1cc3757996ca2ac6", "date6-a2ba809bbc011fd5", "pit5-1a046d61fc220511", "date5-558eed3b8d1cd355", "pit4-d055d33ebde2939c", "date4-c401a39cb66dd9b6", "pit3-1d67a59dc4c4619c", "date3-part-b82f41b43f9d2cb6", "date3-ce85176fff8cd296", "pit2-01c32435697228df", "date2-part-27cb927eeb4edac8", "date2-4e97581c69f3b6a2", "pit1-5ff35b63a8c554f1", "date1-21bf734a43bca0ad", "bowl2-f3ecbf5e23c95e49", "handful of dates4-4602e528bc3e674e", "one date-6cb95d350d4f74d2", "water filter jug-c6165db33e0b1705", "handful of dates3-1f5b6a6f26d4d92d", "handful of dates2-df4f37501ad70175", "handful of dates-984a4976525d101d", "bowl-034c87f8aaff2a97", "plastic bag of dates-6db555c7bee29c52", "bag containing bag of dates-5607343d67c9897a", "second bread bag-3b44dd5d6c8cde97", "bread bag-96341b192e706311", "butter cube-fbd8caf969a21e94", "knife-2edd39f49e4937da", "butter packaging-fb7c3fe948bbb4f6", "notepad-254d634920ab4db3", "pen-23749739cc4dc902", "saucepan-9ab0ba1deef4dbd1", "plastic piece-dda20dcf99e9cafb", "scale-d0e06fd11d88bbc4", "bottle-1b8e66c7851ecd60", "new spoon-aa98f768eb8c2565", "new knife-007f39e9b56757a1" ], "reasoning": "1. I examined each of the three actions on its own to determine its duration. I begin by observing the user washing the dates (handful of dates-984a4976525d101d) beginning at 01:17 and stopping at 01:54. The user then washes the last 2 remaining dates (handful of dates4-4602e528bc3e674e) from 05:23 to 05:29. This totals a duration of 43 seconds.\n2. I observe the user remove a pit (pit1-5ff35b63a8c554f1) from the dates (handful of dates-984a4976525d101d) beginning at 02:11. The user finishes at 04:06, then begins again at 05:36 and stops at 05:58. This totals a duration of 2 minutes and 17 seconds.\n3. I observe the user placing the dates (handful of dates-984a4976525d101d) into a cup (glass1-0ac4b6c1e31f8bd4) on the scale (scale-d0e06fd11d88bbc4) at 04:27 to weigh them. The user finishes measuring at 05:15, then begins again at 06:00 and stops at 06:30. This totals a duration of 1 minute and 18 seconds.\n4. From the longest duration to the shortest, the order of these actions are removing the pits, weighing the dates, and washing the dates.", "referenced_object_masks": [ "glass1-0ac4b6c1e31f8bd4", "scale-d0e06fd11d88bbc4", "pit1-5ff35b63a8c554f1", "handful of dates-984a4976525d101d", "handful of dates4-4602e528bc3e674e" ] }, { "key": "P01-20240204-121042:28cff72c25e2c0bd43534d634a65211a3ab7586f", "video_id": "P01-20240204-121042", "question": "How much time passed between when I wrote on the paper for the second time and when I rinsed the date that I picked up off the floor?", "answer_choice_0": "57 seconds", "answer_choice_1": "2 minutes and 10 seconds", "answer_choice_2": "2 minutes and 51 seconds", "answer_choice_3": "1 minute and 34 seconds", "answer_choice_4": "1 minute and 27 seconds", "answer_id": 0, "answer": "57 seconds", "question_type": [ "Numerical Reasoning", "Object Recognition", "Action Recognition", "Counting", "Temporal Reasoning" ], "objects_of_interest": [ "bag of bagels-153c0673fdd61477", "medicine box-a3d8fcaa7aad548d", "glass5-d7419820dc2c2641", "tupperware-152598eab58ac975", "lid of tupperware-c7e0c66d804d4e6a", "roll of kitchen towel-65dd1a2f043c18dc", "can opener-a5b94da9cd002cb7", "piece of plastic-9da6992cbd62fa8d", "oranges mesh2-8a6cf06a891a70fa", "oranges mesh-8ebdee21e22805ba", "scissors guard-246ff4a57cbce4ad", "scissors-bb8255e02ae1e4ae", "blade cover2-1d9e63e2b58a5123", "blade cover1-c7d0d86a80bf0c0f", "plastic piece circular-53271ebfb0d03eb1", "glass4-1700579fc9dddb25", "tin of cocoa powder-0c144f3fc7cbbd66", "bowl3-ea4d5dca7d1fb5b6", "plate4-d749259fc8ca2033", "salt bottle-2df853a63e0f2951", "soup packets box-bebacc4e1b2fc36e", "jar-b346c4ac3de06bae", "packet-a0581a0332e74695", "mug-0e8f1bb541fa3ae9", "plate3-2840bba104267155", "knife4-c1f0b9e79aba6417", "large knife-780aa12f43058319", "chopping board2-0b038e808fd27ccc", "baking tray-109b3f8aab229736", "saucepan3-709ea5142ba420c8", "plate2-3f813a620242f873", "pot-210720946e78f619", "large plastic spoon-e9f1a2d844db1353", "strainer-380139925db102d4", "fork2-ffeca99db06d5dd2", "knife3-5502c51df2590d66", "fork-0a9345c7b8cd5880", "washing up liquid bottle-8547e271ebec77c9", "sponge-04aff40e7c88fd05", "spoon-6dbb321dd6b43d48", "left glove-9ac2e9e9046d5f25", "right glove-3ee9b1fdbc1dfef4", "plastic bag2-05805d3737847b90", "chopping board-f8de86a2d50d4ed9", "glass3-d669ab66f4974fc2", "large plastic bowl-16df4d9d4af0e47b", "lid of food processor-ee24ee2a177c4dc9", "biscuits3-e683eacaf91a71c4", "biscuits2-11ece4abb786ae11", "second packet of biscuits-b79a6911948f131d", "one tube of biscuits-43d560e9bfd95aab", "new packet of biscuits-16867c2cb8adb2ba", "plastic packaging2-eada36ce4f8f73e5", "biscuits1-2f44aa8e57cb6b98", "plastic packaging-149dac4d39715129", "blade-7b1da9ab2a4df098", "saucepan2-9da700035a9786b3", "towel-00625522044ee91f", "plate-94998c22011a2b48", "lid2-b841e1dc27a59183", "lid-7fb7c1b6985182e4", "food processor plug-5efeebd54e423176", "food processor-62c72d1a1a65365b", "centre piece of food processor-f176edcdf638e3a8", "food processor bowl-e5a819f6be32d6e9", "wooden spatula-08a837e2faee1615", "one fallen date-60fdca734b28ede9", "pitted dates20-d8b699b34e79c319", "pitted dates19-d366f77432264f09", "pitted dates18-3ef341bd5f56fa4b", "pitted dates17-d5e248a04b63b96b", "pitted dates16-bf91f9ff1d0f5528", "pitted dates15-ed2ae9851cbbfde9", "pitted dates14-ba3300155986a412", "pitted dates13-1afc573162d3c0cb", "pitted dates12-bd78da1768e2a388", "pit23-60a5ac1615e34a07", "date23-b8f8cd9918a89fa0", "pit22-b214e8c39519c7c6", "date22-332d25b1af8723a9", "pit21-e0ad27141d45f334", "date21-58e03f20607143da", "pit20-d48d7ec92570fffd", "date20-34d2f4160ec2b610", "pit19-7d79ff0908e1f810", "date19-ebd1e8368e774501", "remaining dates-280db61b861bfb14", "few pitted dates11-6c47844eb5473eab", "few pitted dates10-06b3c845e1fe2226", "few pitted dates9-237e6ce575b658f5", "few pitted dates8-1218a8655ba27a48", "few pitted dates7-5a66788e54540761", "few pitted dates6-dc3f6fe36a481b5f", "pitted date5-6b3ed4fcf5f9a86e", "pitted date4-27e75bea33e5c90f", "pitted date3-8a0d58d957ec7697", "two pitted dates-69bbb53ab894e261", "handful of pitted dates-b55a740c8e3fb0f0", "glass2-86974b5f919d4e60", "glass1-0ac4b6c1e31f8bd4", "pit18-9a43224881f13bdd", "date18-27f5c16b635948b9", "pit17-91f81d267577509f", "date17-9e023ce14d581242", "pit16-ae4215e902509d3d", "date16-8b1065206ddefb90", "pit15-305264da14998461", "date15-f538d6ce88b034a5", "pit14-d270e2a6687c24f9", "date14-54127f608e4d0caf", "pit13-256856040eec03c8", "date13-8d1028ecbc426468", "pit12-5d712e79351c4d81", "date12-5e5e0a7f8913e8c5", "pit11-e23810ca2db561be", "date11-part-7999e894a0df25c5", "date11-5de863c9e90b2879", "pit10-63299058578ae8ab", "date10-part-2351ed68668bd070", "date10-7d36aa35cd8718d4", "pit9-0c92c3471984818a", "date9-53c1beb54cd290d7", "pit8-4ed09c1e3af29c4e", "date8-be88c56d1dd28da4", "pit7-51b1c77a2cc5d7c0", "date7-75b6f299949b1e8e", "pit6-1cc3757996ca2ac6", "date6-a2ba809bbc011fd5", "pit5-1a046d61fc220511", "date5-558eed3b8d1cd355", "pit4-d055d33ebde2939c", "date4-c401a39cb66dd9b6", "pit3-1d67a59dc4c4619c", "date3-part-b82f41b43f9d2cb6", "date3-ce85176fff8cd296", "pit2-01c32435697228df", "date2-part-27cb927eeb4edac8", "date2-4e97581c69f3b6a2", "pit1-5ff35b63a8c554f1", "date1-21bf734a43bca0ad", "bowl2-f3ecbf5e23c95e49", "handful of dates4-4602e528bc3e674e", "one date-6cb95d350d4f74d2", "water filter jug-c6165db33e0b1705", "handful of dates3-1f5b6a6f26d4d92d", "handful of dates2-df4f37501ad70175", "handful of dates-984a4976525d101d", "bowl-034c87f8aaff2a97", "plastic bag of dates-6db555c7bee29c52", "bag containing bag of dates-5607343d67c9897a", "second bread bag-3b44dd5d6c8cde97", "bread bag-96341b192e706311", "butter cube-fbd8caf969a21e94", "knife-2edd39f49e4937da", "butter packaging-fb7c3fe948bbb4f6", "notepad-254d634920ab4db3", "pen-23749739cc4dc902", "saucepan-9ab0ba1deef4dbd1", "plastic piece-dda20dcf99e9cafb", "scale-d0e06fd11d88bbc4", "bottle-1b8e66c7851ecd60", "new spoon-aa98f768eb8c2565", "new knife-007f39e9b56757a1" ], "reasoning": "1. I watched the video and saw the user write on the paper (notepad-254d634920ab4db3) for the first time from 00:10-00:12. \n2. I saw the user write on the paper (notepad-254d634920ab4db3) for the second time from 00:42-00:44. \n3. I saw the user pick up 1 date (one date-6cb95d350d4f74d2) off the floor from 01:38-01:40. \n4. I saw the user rinse the 1 date (one date-6cb95d350d4f74d2) from the floor at 01:41. \n5. I subtracted 01:41-00:44 = 00:57.", "referenced_object_masks": [ "notepad-254d634920ab4db3", "one date-6cb95d350d4f74d2" ] }, { "key": "P01-20240204-121042:34cae875de4b19e4e655bc31d2b69c7fe5be9d5a", "video_id": "P01-20240204-121042", "question": "Rank these from highest to lowest: The number of times I picked up a bottle of dish soap, one-half of the number of things I put into the dishwasher, and the number of times I pressed a button on the scale.", "answer_choice_0": "The number of times I picked up a bottle of dish soap; one-half of the number of things I put into the dishwasher; the number of times I pressed a button on the scale", "answer_choice_1": "The number of times I pressed a button on the scale; one-half of the number of things I put into the dishwasher; the number of times I picked up a bottle of dish soap", "answer_choice_2": "The number of times I picked up a bottle of dish soap; the number of times I pressed a button on the scale; one-half of the number of things I put into the dishwasher", "answer_choice_3": "The number of times I pressed a button on the scale; the number of times I picked up a bottle of dish soap; one-half of the number of things I put into the dishwasher", "answer_choice_4": "One-half of the number of things I put into the dishwasher; the number of times I pressed a button on the scale; the number of times I picked up a bottle of dish soap", "answer_id": 4, "answer": "One-half of the number of things I put into the dishwasher; the number of times I pressed a button on the scale; the number of times I picked up a bottle of dish soap", "question_type": [ "Numerical Reasoning", "Object Recognition", "Action Recognition", "Counting", "Temporal Reasoning" ], "objects_of_interest": [ "bag of bagels-153c0673fdd61477", "medicine box-a3d8fcaa7aad548d", "glass5-d7419820dc2c2641", "tupperware-152598eab58ac975", "lid of tupperware-c7e0c66d804d4e6a", "roll of kitchen towel-65dd1a2f043c18dc", "can opener-a5b94da9cd002cb7", "piece of plastic-9da6992cbd62fa8d", "oranges mesh2-8a6cf06a891a70fa", "oranges mesh-8ebdee21e22805ba", "scissors guard-246ff4a57cbce4ad", "scissors-bb8255e02ae1e4ae", "blade cover2-1d9e63e2b58a5123", "blade cover1-c7d0d86a80bf0c0f", "plastic piece circular-53271ebfb0d03eb1", "glass4-1700579fc9dddb25", "tin of cocoa powder-0c144f3fc7cbbd66", "bowl3-ea4d5dca7d1fb5b6", "plate4-d749259fc8ca2033", "salt bottle-2df853a63e0f2951", "soup packets box-bebacc4e1b2fc36e", "jar-b346c4ac3de06bae", "packet-a0581a0332e74695", "mug-0e8f1bb541fa3ae9", "plate3-2840bba104267155", "knife4-c1f0b9e79aba6417", "large knife-780aa12f43058319", "chopping board2-0b038e808fd27ccc", "baking tray-109b3f8aab229736", "saucepan3-709ea5142ba420c8", "plate2-3f813a620242f873", "pot-210720946e78f619", "large plastic spoon-e9f1a2d844db1353", "strainer-380139925db102d4", "fork2-ffeca99db06d5dd2", "knife3-5502c51df2590d66", "fork-0a9345c7b8cd5880", "washing up liquid bottle-8547e271ebec77c9", "sponge-04aff40e7c88fd05", "spoon-6dbb321dd6b43d48", "left glove-9ac2e9e9046d5f25", "right glove-3ee9b1fdbc1dfef4", "plastic bag2-05805d3737847b90", "chopping board-f8de86a2d50d4ed9", "glass3-d669ab66f4974fc2", "large plastic bowl-16df4d9d4af0e47b", "lid of food processor-ee24ee2a177c4dc9", "biscuits3-e683eacaf91a71c4", "biscuits2-11ece4abb786ae11", "second packet of biscuits-b79a6911948f131d", "one tube of biscuits-43d560e9bfd95aab", "new packet of biscuits-16867c2cb8adb2ba", "plastic packaging2-eada36ce4f8f73e5", "biscuits1-2f44aa8e57cb6b98", "plastic packaging-149dac4d39715129", "blade-7b1da9ab2a4df098", "saucepan2-9da700035a9786b3", "towel-00625522044ee91f", "plate-94998c22011a2b48", "lid2-b841e1dc27a59183", "lid-7fb7c1b6985182e4", "food processor plug-5efeebd54e423176", "food processor-62c72d1a1a65365b", "centre piece of food processor-f176edcdf638e3a8", "food processor bowl-e5a819f6be32d6e9", "wooden spatula-08a837e2faee1615", "one fallen date-60fdca734b28ede9", "pitted dates20-d8b699b34e79c319", "pitted dates19-d366f77432264f09", "pitted dates18-3ef341bd5f56fa4b", "pitted dates17-d5e248a04b63b96b", "pitted dates16-bf91f9ff1d0f5528", "pitted dates15-ed2ae9851cbbfde9", "pitted dates14-ba3300155986a412", "pitted dates13-1afc573162d3c0cb", "pitted dates12-bd78da1768e2a388", "pit23-60a5ac1615e34a07", "date23-b8f8cd9918a89fa0", "pit22-b214e8c39519c7c6", "date22-332d25b1af8723a9", "pit21-e0ad27141d45f334", "date21-58e03f20607143da", "pit20-d48d7ec92570fffd", "date20-34d2f4160ec2b610", "pit19-7d79ff0908e1f810", "date19-ebd1e8368e774501", "remaining dates-280db61b861bfb14", "few pitted dates11-6c47844eb5473eab", "few pitted dates10-06b3c845e1fe2226", "few pitted dates9-237e6ce575b658f5", "few pitted dates8-1218a8655ba27a48", "few pitted dates7-5a66788e54540761", "few pitted dates6-dc3f6fe36a481b5f", "pitted date5-6b3ed4fcf5f9a86e", "pitted date4-27e75bea33e5c90f", "pitted date3-8a0d58d957ec7697", "two pitted dates-69bbb53ab894e261", "handful of pitted dates-b55a740c8e3fb0f0", "glass2-86974b5f919d4e60", "glass1-0ac4b6c1e31f8bd4", "pit18-9a43224881f13bdd", "date18-27f5c16b635948b9", "pit17-91f81d267577509f", "date17-9e023ce14d581242", "pit16-ae4215e902509d3d", "date16-8b1065206ddefb90", "pit15-305264da14998461", "date15-f538d6ce88b034a5", "pit14-d270e2a6687c24f9", "date14-54127f608e4d0caf", "pit13-256856040eec03c8", "date13-8d1028ecbc426468", "pit12-5d712e79351c4d81", "date12-5e5e0a7f8913e8c5", "pit11-e23810ca2db561be", "date11-part-7999e894a0df25c5", "date11-5de863c9e90b2879", "pit10-63299058578ae8ab", "date10-part-2351ed68668bd070", "date10-7d36aa35cd8718d4", "pit9-0c92c3471984818a", "date9-53c1beb54cd290d7", "pit8-4ed09c1e3af29c4e", "date8-be88c56d1dd28da4", "pit7-51b1c77a2cc5d7c0", "date7-75b6f299949b1e8e", "pit6-1cc3757996ca2ac6", "date6-a2ba809bbc011fd5", "pit5-1a046d61fc220511", "date5-558eed3b8d1cd355", "pit4-d055d33ebde2939c", "date4-c401a39cb66dd9b6", "pit3-1d67a59dc4c4619c", "date3-part-b82f41b43f9d2cb6", "date3-ce85176fff8cd296", "pit2-01c32435697228df", "date2-part-27cb927eeb4edac8", "date2-4e97581c69f3b6a2", "pit1-5ff35b63a8c554f1", "date1-21bf734a43bca0ad", "bowl2-f3ecbf5e23c95e49", "handful of dates4-4602e528bc3e674e", "one date-6cb95d350d4f74d2", "water filter jug-c6165db33e0b1705", "handful of dates3-1f5b6a6f26d4d92d", "handful of dates2-df4f37501ad70175", "handful of dates-984a4976525d101d", "bowl-034c87f8aaff2a97", "plastic bag of dates-6db555c7bee29c52", "bag containing bag of dates-5607343d67c9897a", "second bread bag-3b44dd5d6c8cde97", "bread bag-96341b192e706311", "butter cube-fbd8caf969a21e94", "knife-2edd39f49e4937da", "butter packaging-fb7c3fe948bbb4f6", "notepad-254d634920ab4db3", "pen-23749739cc4dc902", "saucepan-9ab0ba1deef4dbd1", "plastic piece-dda20dcf99e9cafb", "scale-d0e06fd11d88bbc4", "bottle-1b8e66c7851ecd60", "new spoon-aa98f768eb8c2565", "new knife-007f39e9b56757a1" ], "reasoning": "1. I watched the video and counted each time the user picked up a bottle of dish soap (washing up liquid bottle-8547e271ebec77c9). This occurred at 13:50 and 17:14, making a total of 2.\n2. I counted each time the user put an item in the dishwasher . This occurred at 14:13, 14:17, 14:20, 14:22, 16:23, 17:33, 17:43, 18:02, 18:04, 18:14, 21:06, and 21:07, making a total of 12.\n3. I counted each time the user pressed a button on the scale (scale-d0e06fd11d88bbc4). This occurred at 00:29, 04:24, 04:29, 06:05, and 12:03, making a total of 5. \n4. I halved the number of times an item was placed in the dishwasher by calculating 12/2=6. \n5. I arranged the resulting numbers in order from highest to lowest: Items placed in the dishwasher , halved (6), times the button was pressed on the scale (scale-d0e06fd11d88bbc4) (5), and times dish soap (washing up liquid bottle-8547e271ebec77c9) was picked up (2).", "referenced_object_masks": [ "washing up liquid bottle-8547e271ebec77c9", "scale-d0e06fd11d88bbc4" ] }, { "key": "P01-20240204-121042:3ba7d5668304618e8a74f236e56428b516cdffc3", "video_id": "P01-20240204-121042", "question": "How much did all the pitted dates weigh?", "answer_choice_0": "545", "answer_choice_1": "161", "answer_choice_2": "255", "answer_choice_3": "416", "answer_choice_4": "290", "answer_id": 0, "answer": "545", "question_type": [ "Numerical Reasoning", "Object Recognition", "Temporal Reasoning", "Action Recognition", "Reading" ], "objects_of_interest": [ "bag of bagels-153c0673fdd61477", "medicine box-a3d8fcaa7aad548d", "glass5-d7419820dc2c2641", "tupperware-152598eab58ac975", "lid of tupperware-c7e0c66d804d4e6a", "roll of kitchen towel-65dd1a2f043c18dc", "can opener-a5b94da9cd002cb7", "piece of plastic-9da6992cbd62fa8d", "oranges mesh2-8a6cf06a891a70fa", "oranges mesh-8ebdee21e22805ba", "scissors guard-246ff4a57cbce4ad", "scissors-bb8255e02ae1e4ae", "blade cover2-1d9e63e2b58a5123", "blade cover1-c7d0d86a80bf0c0f", "plastic piece circular-53271ebfb0d03eb1", "glass4-1700579fc9dddb25", "tin of cocoa powder-0c144f3fc7cbbd66", "bowl3-ea4d5dca7d1fb5b6", "plate4-d749259fc8ca2033", "salt bottle-2df853a63e0f2951", "soup packets box-bebacc4e1b2fc36e", "jar-b346c4ac3de06bae", "packet-a0581a0332e74695", "mug-0e8f1bb541fa3ae9", "plate3-2840bba104267155", "knife4-c1f0b9e79aba6417", "large knife-780aa12f43058319", "chopping board2-0b038e808fd27ccc", "baking tray-109b3f8aab229736", "saucepan3-709ea5142ba420c8", "plate2-3f813a620242f873", "pot-210720946e78f619", "large plastic spoon-e9f1a2d844db1353", "strainer-380139925db102d4", "fork2-ffeca99db06d5dd2", "knife3-5502c51df2590d66", "fork-0a9345c7b8cd5880", "washing up liquid bottle-8547e271ebec77c9", "sponge-04aff40e7c88fd05", "spoon-6dbb321dd6b43d48", "left glove-9ac2e9e9046d5f25", "right glove-3ee9b1fdbc1dfef4", "plastic bag2-05805d3737847b90", "chopping board-f8de86a2d50d4ed9", "glass3-d669ab66f4974fc2", "large plastic bowl-16df4d9d4af0e47b", "lid of food processor-ee24ee2a177c4dc9", "biscuits3-e683eacaf91a71c4", "biscuits2-11ece4abb786ae11", "second packet of biscuits-b79a6911948f131d", "one tube of biscuits-43d560e9bfd95aab", "new packet of biscuits-16867c2cb8adb2ba", "plastic packaging2-eada36ce4f8f73e5", "biscuits1-2f44aa8e57cb6b98", "plastic packaging-149dac4d39715129", "blade-7b1da9ab2a4df098", "saucepan2-9da700035a9786b3", "towel-00625522044ee91f", "plate-94998c22011a2b48", "lid2-b841e1dc27a59183", "lid-7fb7c1b6985182e4", "food processor plug-5efeebd54e423176", "food processor-62c72d1a1a65365b", "centre piece of food processor-f176edcdf638e3a8", "food processor bowl-e5a819f6be32d6e9", "wooden spatula-08a837e2faee1615", "one fallen date-60fdca734b28ede9", "pitted dates20-d8b699b34e79c319", "pitted dates19-d366f77432264f09", "pitted dates18-3ef341bd5f56fa4b", "pitted dates17-d5e248a04b63b96b", "pitted dates16-bf91f9ff1d0f5528", "pitted dates15-ed2ae9851cbbfde9", "pitted dates14-ba3300155986a412", "pitted dates13-1afc573162d3c0cb", "pitted dates12-bd78da1768e2a388", "pit23-60a5ac1615e34a07", "date23-b8f8cd9918a89fa0", "pit22-b214e8c39519c7c6", "date22-332d25b1af8723a9", "pit21-e0ad27141d45f334", "date21-58e03f20607143da", "pit20-d48d7ec92570fffd", "date20-34d2f4160ec2b610", "pit19-7d79ff0908e1f810", "date19-ebd1e8368e774501", "remaining dates-280db61b861bfb14", "few pitted dates11-6c47844eb5473eab", "few pitted dates10-06b3c845e1fe2226", "few pitted dates9-237e6ce575b658f5", "few pitted dates8-1218a8655ba27a48", "few pitted dates7-5a66788e54540761", "few pitted dates6-dc3f6fe36a481b5f", "pitted date5-6b3ed4fcf5f9a86e", "pitted date4-27e75bea33e5c90f", "pitted date3-8a0d58d957ec7697", "two pitted dates-69bbb53ab894e261", "handful of pitted dates-b55a740c8e3fb0f0", "glass2-86974b5f919d4e60", "glass1-0ac4b6c1e31f8bd4", "pit18-9a43224881f13bdd", "date18-27f5c16b635948b9", "pit17-91f81d267577509f", "date17-9e023ce14d581242", "pit16-ae4215e902509d3d", "date16-8b1065206ddefb90", "pit15-305264da14998461", "date15-f538d6ce88b034a5", "pit14-d270e2a6687c24f9", "date14-54127f608e4d0caf", "pit13-256856040eec03c8", "date13-8d1028ecbc426468", "pit12-5d712e79351c4d81", "date12-5e5e0a7f8913e8c5", "pit11-e23810ca2db561be", "date11-part-7999e894a0df25c5", "date11-5de863c9e90b2879", "pit10-63299058578ae8ab", "date10-part-2351ed68668bd070", "date10-7d36aa35cd8718d4", "pit9-0c92c3471984818a", "date9-53c1beb54cd290d7", "pit8-4ed09c1e3af29c4e", "date8-be88c56d1dd28da4", "pit7-51b1c77a2cc5d7c0", "date7-75b6f299949b1e8e", "pit6-1cc3757996ca2ac6", "date6-a2ba809bbc011fd5", "pit5-1a046d61fc220511", "date5-558eed3b8d1cd355", "pit4-d055d33ebde2939c", "date4-c401a39cb66dd9b6", "pit3-1d67a59dc4c4619c", "date3-part-b82f41b43f9d2cb6", "date3-ce85176fff8cd296", "pit2-01c32435697228df", "date2-part-27cb927eeb4edac8", "date2-4e97581c69f3b6a2", "pit1-5ff35b63a8c554f1", "date1-21bf734a43bca0ad", "bowl2-f3ecbf5e23c95e49", "handful of dates4-4602e528bc3e674e", "one date-6cb95d350d4f74d2", "water filter jug-c6165db33e0b1705", "handful of dates3-1f5b6a6f26d4d92d", "handful of dates2-df4f37501ad70175", "handful of dates-984a4976525d101d", "bowl-034c87f8aaff2a97", "plastic bag of dates-6db555c7bee29c52", "bag containing bag of dates-5607343d67c9897a", "second bread bag-3b44dd5d6c8cde97", "bread bag-96341b192e706311", "butter cube-fbd8caf969a21e94", "knife-2edd39f49e4937da", "butter packaging-fb7c3fe948bbb4f6", "notepad-254d634920ab4db3", "pen-23749739cc4dc902", "saucepan-9ab0ba1deef4dbd1", "plastic piece-dda20dcf99e9cafb", "scale-d0e06fd11d88bbc4", "bottle-1b8e66c7851ecd60", "new spoon-aa98f768eb8c2565", "new knife-007f39e9b56757a1" ], "reasoning": "1. I saw the user remove the pits from a bowlful of dates in the colorful bowl (bowl-034c87f8aaff2a97) and add the pitted dates to a second bowl (bowl2-f3ecbf5e23c95e49) from 02:10-04:07. \n2. I saw the user add about half of the bowlful of pitted dates to a glass (glass2-86974b5f919d4e60) on the scale (scale-d0e06fd11d88bbc4) from 04:31-04:47. In this time, I observed the digital screen on the scale (scale-d0e06fd11d88bbc4) increase as more dates were added. \n3. When the user took their hands away from the scale (scale-d0e06fd11d88bbc4) at 04:47, I saw that the screen read \"255\". \n4. I saw the user empty the glass (glass2-86974b5f919d4e60) into the second bowl (bowl2-f3ecbf5e23c95e49) from 04:52-04:53. \n5. I saw the user place the glass (glass2-86974b5f919d4e60) back onto the scale (scale-d0e06fd11d88bbc4) at 04:57. \n6. From 05:04-05:14 I saw the user add the second half of a bowlful of pitted dates to the glass (glass2-86974b5f919d4e60) on the scale (scale-d0e06fd11d88bbc4). When the user took their hands away at 05:14, I saw that the screen on the scale (scale-d0e06fd11d88bbc4) read \"161\". \n7. At 05:17 I saw the user pick up the plastic bag of dates (plastic bag of dates-6db555c7bee29c52), grab the remaining dates (remaining dates-280db61b861bfb14), and put them in the colorful bowl (bowl-034c87f8aaff2a97). \n8. From 05:35-05:58 I saw the user remove the pits from the remaining dates (remaining dates-280db61b861bfb14) in the colorful bowl (bowl-034c87f8aaff2a97). I also saw the user put the pitted remaining dates back into the colorful bowl (bowl-034c87f8aaff2a97). \n9. At 06:00 I saw the user add 1 date (pitted dates12-bd78da1768e2a388) to the glass (glass2-86974b5f919d4e60) on the scale (scale-d0e06fd11d88bbc4).\n6. From 06:04-06:05 I saw the user pour the contents of the glass (glass2-86974b5f919d4e60) back into the colorful bowl (bowl-034c87f8aaff2a97). \n10. From 06:08-06:29 I saw the user add all of the pitted dates in the colorful bowl (bowl-034c87f8aaff2a97) into the glass on the scale (scale-d0e06fd11d88bbc4). When the user took their hands away at 06:29, I saw that the screen on the scale (scale-d0e06fd11d88bbc4) read \"290\". \n11. I did not see any more dates pitted or weighed, so I added the two weights together: 255+290=545.", "referenced_object_masks": [ "plastic bag of dates-6db555c7bee29c52", "glass2-86974b5f919d4e60", "pitted dates12-bd78da1768e2a388", "bowl-034c87f8aaff2a97", "remaining dates-280db61b861bfb14", "bowl2-f3ecbf5e23c95e49", "scale-d0e06fd11d88bbc4" ] }, { "key": "P01-20240204-121042:8283d444da53369e3cacee56aa7666cba2b730e1", "video_id": "P01-20240204-121042", "question": "If I faced the drawer where utensils are kept, what would I do to face the area where I put away the unopened sleeve of biscuits?", "answer_choice_0": "Turn 180 degrees", "answer_choice_1": "Turn 90 degrees right", "answer_choice_2": "Walk straight, turn right, turn left", "answer_choice_3": "Turn 90 degrees left", "answer_choice_4": "Walk backwards, turn right, walk straight", "answer_id": 0, "answer": "Turn 180 degrees", "question_type": [ "Egocentric & Spatial Perception", "Temporal Reasoning", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "bag of bagels-153c0673fdd61477", "medicine box-a3d8fcaa7aad548d", "glass5-d7419820dc2c2641", "tupperware-152598eab58ac975", "lid of tupperware-c7e0c66d804d4e6a", "roll of kitchen towel-65dd1a2f043c18dc", "can opener-a5b94da9cd002cb7", "piece of plastic-9da6992cbd62fa8d", "oranges mesh2-8a6cf06a891a70fa", "oranges mesh-8ebdee21e22805ba", "scissors guard-246ff4a57cbce4ad", "scissors-bb8255e02ae1e4ae", "blade cover2-1d9e63e2b58a5123", "blade cover1-c7d0d86a80bf0c0f", "plastic piece circular-53271ebfb0d03eb1", "glass4-1700579fc9dddb25", "tin of cocoa powder-0c144f3fc7cbbd66", "bowl3-ea4d5dca7d1fb5b6", "plate4-d749259fc8ca2033", "salt bottle-2df853a63e0f2951", "soup packets box-bebacc4e1b2fc36e", "jar-b346c4ac3de06bae", "packet-a0581a0332e74695", "mug-0e8f1bb541fa3ae9", "plate3-2840bba104267155", "knife4-c1f0b9e79aba6417", "large knife-780aa12f43058319", "chopping board2-0b038e808fd27ccc", "baking tray-109b3f8aab229736", "saucepan3-709ea5142ba420c8", "plate2-3f813a620242f873", "pot-210720946e78f619", "large plastic spoon-e9f1a2d844db1353", "strainer-380139925db102d4", "fork2-ffeca99db06d5dd2", "knife3-5502c51df2590d66", "fork-0a9345c7b8cd5880", "washing up liquid bottle-8547e271ebec77c9", "sponge-04aff40e7c88fd05", "spoon-6dbb321dd6b43d48", "left glove-9ac2e9e9046d5f25", "right glove-3ee9b1fdbc1dfef4", "plastic bag2-05805d3737847b90", "chopping board-f8de86a2d50d4ed9", "glass3-d669ab66f4974fc2", "large plastic bowl-16df4d9d4af0e47b", "lid of food processor-ee24ee2a177c4dc9", "biscuits3-e683eacaf91a71c4", "biscuits2-11ece4abb786ae11", "second packet of biscuits-b79a6911948f131d", "one tube of biscuits-43d560e9bfd95aab", "new packet of biscuits-16867c2cb8adb2ba", "plastic packaging2-eada36ce4f8f73e5", "biscuits1-2f44aa8e57cb6b98", "plastic packaging-149dac4d39715129", "blade-7b1da9ab2a4df098", "saucepan2-9da700035a9786b3", "towel-00625522044ee91f", "plate-94998c22011a2b48", "lid2-b841e1dc27a59183", "lid-7fb7c1b6985182e4", "food processor plug-5efeebd54e423176", "food processor-62c72d1a1a65365b", "centre piece of food processor-f176edcdf638e3a8", "food processor bowl-e5a819f6be32d6e9", "wooden spatula-08a837e2faee1615", "one fallen date-60fdca734b28ede9", "pitted dates20-d8b699b34e79c319", "pitted dates19-d366f77432264f09", "pitted dates18-3ef341bd5f56fa4b", "pitted dates17-d5e248a04b63b96b", "pitted dates16-bf91f9ff1d0f5528", "pitted dates15-ed2ae9851cbbfde9", "pitted dates14-ba3300155986a412", "pitted dates13-1afc573162d3c0cb", "pitted dates12-bd78da1768e2a388", "pit23-60a5ac1615e34a07", "date23-b8f8cd9918a89fa0", "pit22-b214e8c39519c7c6", "date22-332d25b1af8723a9", "pit21-e0ad27141d45f334", "date21-58e03f20607143da", "pit20-d48d7ec92570fffd", "date20-34d2f4160ec2b610", "pit19-7d79ff0908e1f810", "date19-ebd1e8368e774501", "remaining dates-280db61b861bfb14", "few pitted dates11-6c47844eb5473eab", "few pitted dates10-06b3c845e1fe2226", "few pitted dates9-237e6ce575b658f5", "few pitted dates8-1218a8655ba27a48", "few pitted dates7-5a66788e54540761", "few pitted dates6-dc3f6fe36a481b5f", "pitted date5-6b3ed4fcf5f9a86e", "pitted date4-27e75bea33e5c90f", "pitted date3-8a0d58d957ec7697", "two pitted dates-69bbb53ab894e261", "handful of pitted dates-b55a740c8e3fb0f0", "glass2-86974b5f919d4e60", "glass1-0ac4b6c1e31f8bd4", "pit18-9a43224881f13bdd", "date18-27f5c16b635948b9", "pit17-91f81d267577509f", "date17-9e023ce14d581242", "pit16-ae4215e902509d3d", "date16-8b1065206ddefb90", "pit15-305264da14998461", "date15-f538d6ce88b034a5", "pit14-d270e2a6687c24f9", "date14-54127f608e4d0caf", "pit13-256856040eec03c8", "date13-8d1028ecbc426468", "pit12-5d712e79351c4d81", "date12-5e5e0a7f8913e8c5", "pit11-e23810ca2db561be", "date11-part-7999e894a0df25c5", "date11-5de863c9e90b2879", "pit10-63299058578ae8ab", "date10-part-2351ed68668bd070", "date10-7d36aa35cd8718d4", "pit9-0c92c3471984818a", "date9-53c1beb54cd290d7", "pit8-4ed09c1e3af29c4e", "date8-be88c56d1dd28da4", "pit7-51b1c77a2cc5d7c0", "date7-75b6f299949b1e8e", "pit6-1cc3757996ca2ac6", "date6-a2ba809bbc011fd5", "pit5-1a046d61fc220511", "date5-558eed3b8d1cd355", "pit4-d055d33ebde2939c", "date4-c401a39cb66dd9b6", "pit3-1d67a59dc4c4619c", "date3-part-b82f41b43f9d2cb6", "date3-ce85176fff8cd296", "pit2-01c32435697228df", "date2-part-27cb927eeb4edac8", "date2-4e97581c69f3b6a2", "pit1-5ff35b63a8c554f1", "date1-21bf734a43bca0ad", "bowl2-f3ecbf5e23c95e49", "handful of dates4-4602e528bc3e674e", "one date-6cb95d350d4f74d2", "water filter jug-c6165db33e0b1705", "handful of dates3-1f5b6a6f26d4d92d", "handful of dates2-df4f37501ad70175", "handful of dates-984a4976525d101d", "bowl-034c87f8aaff2a97", "plastic bag of dates-6db555c7bee29c52", "bag containing bag of dates-5607343d67c9897a", "second bread bag-3b44dd5d6c8cde97", "bread bag-96341b192e706311", "butter cube-fbd8caf969a21e94", "knife-2edd39f49e4937da", "butter packaging-fb7c3fe948bbb4f6", "notepad-254d634920ab4db3", "pen-23749739cc4dc902", "saucepan-9ab0ba1deef4dbd1", "plastic piece-dda20dcf99e9cafb", "scale-d0e06fd11d88bbc4", "bottle-1b8e66c7851ecd60", "new spoon-aa98f768eb8c2565", "new knife-007f39e9b56757a1" ], "reasoning": "1. I watched the video and first saw the user retrieve a knife (knife-2edd39f49e4937da) from a drawer from 00:21-00:24. I identified this as the utensil drawer. \n2. I observed the surroundings from 00:21-00:24 and noted that the drawer was directly across from a counter with cabinets beneath. \n3. At 10:19 I saw the user put an unopened sleeve of biscuits (one tube of biscuits-43d560e9bfd95aab) under the counter directly across from the utensil drawer. \n4. Reasoning about the scene, I concluded that if the user was facing the utensil drawer, they would need to turn 180 degrees to face the place where they put the unopened sleeve of biscuits (one tube of biscuits-43d560e9bfd95aab).", "referenced_object_masks": [ "one tube of biscuits-43d560e9bfd95aab", "knife-2edd39f49e4937da" ] }, { "key": "P01-20240204-121042:a0021e03932c6edf9db740cd3b572638accc3195", "video_id": "P01-20240204-121042", "question": "While I looked at the refrigerator, where was the dishwasher?", "answer_choice_0": "Directly behind", "answer_choice_1": "A few steps to the left", "answer_choice_2": "Behind and to the right--close to the sink", "answer_choice_3": "In front and to the right--close to the sink", "answer_choice_4": "In front and directly to the right", "answer_id": 3, "answer": "In front and to the right--close to the sink", "question_type": [ "Egocentric & Spatial Perception", "Temporal Reasoning", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "bag of bagels-153c0673fdd61477", "medicine box-a3d8fcaa7aad548d", "glass5-d7419820dc2c2641", "tupperware-152598eab58ac975", "lid of tupperware-c7e0c66d804d4e6a", "roll of kitchen towel-65dd1a2f043c18dc", "can opener-a5b94da9cd002cb7", "piece of plastic-9da6992cbd62fa8d", "oranges mesh2-8a6cf06a891a70fa", "oranges mesh-8ebdee21e22805ba", "scissors guard-246ff4a57cbce4ad", "scissors-bb8255e02ae1e4ae", "blade cover2-1d9e63e2b58a5123", "blade cover1-c7d0d86a80bf0c0f", "plastic piece circular-53271ebfb0d03eb1", "glass4-1700579fc9dddb25", "tin of cocoa powder-0c144f3fc7cbbd66", "bowl3-ea4d5dca7d1fb5b6", "plate4-d749259fc8ca2033", "salt bottle-2df853a63e0f2951", "soup packets box-bebacc4e1b2fc36e", "jar-b346c4ac3de06bae", "packet-a0581a0332e74695", "mug-0e8f1bb541fa3ae9", "plate3-2840bba104267155", "knife4-c1f0b9e79aba6417", "large knife-780aa12f43058319", "chopping board2-0b038e808fd27ccc", "baking tray-109b3f8aab229736", "saucepan3-709ea5142ba420c8", "plate2-3f813a620242f873", "pot-210720946e78f619", "large plastic spoon-e9f1a2d844db1353", "strainer-380139925db102d4", "fork2-ffeca99db06d5dd2", "knife3-5502c51df2590d66", "fork-0a9345c7b8cd5880", "washing up liquid bottle-8547e271ebec77c9", "sponge-04aff40e7c88fd05", "spoon-6dbb321dd6b43d48", "left glove-9ac2e9e9046d5f25", "right glove-3ee9b1fdbc1dfef4", "plastic bag2-05805d3737847b90", "chopping board-f8de86a2d50d4ed9", "glass3-d669ab66f4974fc2", "large plastic bowl-16df4d9d4af0e47b", "lid of food processor-ee24ee2a177c4dc9", "biscuits3-e683eacaf91a71c4", "biscuits2-11ece4abb786ae11", "second packet of biscuits-b79a6911948f131d", "one tube of biscuits-43d560e9bfd95aab", "new packet of biscuits-16867c2cb8adb2ba", "plastic packaging2-eada36ce4f8f73e5", "biscuits1-2f44aa8e57cb6b98", "plastic packaging-149dac4d39715129", "blade-7b1da9ab2a4df098", "saucepan2-9da700035a9786b3", "towel-00625522044ee91f", "plate-94998c22011a2b48", "lid2-b841e1dc27a59183", "lid-7fb7c1b6985182e4", "food processor plug-5efeebd54e423176", "food processor-62c72d1a1a65365b", "centre piece of food processor-f176edcdf638e3a8", "food processor bowl-e5a819f6be32d6e9", "wooden spatula-08a837e2faee1615", "one fallen date-60fdca734b28ede9", "pitted dates20-d8b699b34e79c319", "pitted dates19-d366f77432264f09", "pitted dates18-3ef341bd5f56fa4b", "pitted dates17-d5e248a04b63b96b", "pitted dates16-bf91f9ff1d0f5528", "pitted dates15-ed2ae9851cbbfde9", "pitted dates14-ba3300155986a412", "pitted dates13-1afc573162d3c0cb", "pitted dates12-bd78da1768e2a388", "pit23-60a5ac1615e34a07", "date23-b8f8cd9918a89fa0", "pit22-b214e8c39519c7c6", "date22-332d25b1af8723a9", "pit21-e0ad27141d45f334", "date21-58e03f20607143da", "pit20-d48d7ec92570fffd", "date20-34d2f4160ec2b610", "pit19-7d79ff0908e1f810", "date19-ebd1e8368e774501", "remaining dates-280db61b861bfb14", "few pitted dates11-6c47844eb5473eab", "few pitted dates10-06b3c845e1fe2226", "few pitted dates9-237e6ce575b658f5", "few pitted dates8-1218a8655ba27a48", "few pitted dates7-5a66788e54540761", "few pitted dates6-dc3f6fe36a481b5f", "pitted date5-6b3ed4fcf5f9a86e", "pitted date4-27e75bea33e5c90f", "pitted date3-8a0d58d957ec7697", "two pitted dates-69bbb53ab894e261", "handful of pitted dates-b55a740c8e3fb0f0", "glass2-86974b5f919d4e60", "glass1-0ac4b6c1e31f8bd4", "pit18-9a43224881f13bdd", "date18-27f5c16b635948b9", "pit17-91f81d267577509f", "date17-9e023ce14d581242", "pit16-ae4215e902509d3d", "date16-8b1065206ddefb90", "pit15-305264da14998461", "date15-f538d6ce88b034a5", "pit14-d270e2a6687c24f9", "date14-54127f608e4d0caf", "pit13-256856040eec03c8", "date13-8d1028ecbc426468", "pit12-5d712e79351c4d81", "date12-5e5e0a7f8913e8c5", "pit11-e23810ca2db561be", "date11-part-7999e894a0df25c5", "date11-5de863c9e90b2879", "pit10-63299058578ae8ab", "date10-part-2351ed68668bd070", "date10-7d36aa35cd8718d4", "pit9-0c92c3471984818a", "date9-53c1beb54cd290d7", "pit8-4ed09c1e3af29c4e", "date8-be88c56d1dd28da4", "pit7-51b1c77a2cc5d7c0", "date7-75b6f299949b1e8e", "pit6-1cc3757996ca2ac6", "date6-a2ba809bbc011fd5", "pit5-1a046d61fc220511", "date5-558eed3b8d1cd355", "pit4-d055d33ebde2939c", "date4-c401a39cb66dd9b6", "pit3-1d67a59dc4c4619c", "date3-part-b82f41b43f9d2cb6", "date3-ce85176fff8cd296", "pit2-01c32435697228df", "date2-part-27cb927eeb4edac8", "date2-4e97581c69f3b6a2", "pit1-5ff35b63a8c554f1", "date1-21bf734a43bca0ad", "bowl2-f3ecbf5e23c95e49", "handful of dates4-4602e528bc3e674e", "one date-6cb95d350d4f74d2", "water filter jug-c6165db33e0b1705", "handful of dates3-1f5b6a6f26d4d92d", "handful of dates2-df4f37501ad70175", "handful of dates-984a4976525d101d", "bowl-034c87f8aaff2a97", "plastic bag of dates-6db555c7bee29c52", "bag containing bag of dates-5607343d67c9897a", "second bread bag-3b44dd5d6c8cde97", "bread bag-96341b192e706311", "butter cube-fbd8caf969a21e94", "knife-2edd39f49e4937da", "butter packaging-fb7c3fe948bbb4f6", "notepad-254d634920ab4db3", "pen-23749739cc4dc902", "saucepan-9ab0ba1deef4dbd1", "plastic piece-dda20dcf99e9cafb", "scale-d0e06fd11d88bbc4", "bottle-1b8e66c7851ecd60", "new spoon-aa98f768eb8c2565", "new knife-007f39e9b56757a1" ], "reasoning": "1. I watched the video and saw the user open the refrigerator from 00:14-00:16. \n2. I saw the user close the refrigerator at 00:18. \n3. At 00:18, I saw a view of the kitchen and noted the two parallel countertops leading toward the sink on the far end of the kitchen. \n4. At 14:10, I saw the user open the dishwasher and put in silverware. \n5. I observed the location of the dishwasher from 14:05-14:10, noting that the user had to turn 90 degrees counterclockwise and take only a small step from the sink to open the dishwasher. This meant it was on the same side of the kitchen as the refrigerator , and was not directly adjacent to the refrigerator .", "referenced_object_masks": [] }, { "key": "P01-20240204-121042:b0df5d64994ddd11031d6de3b0b27d19fcef6674", "video_id": "P01-20240204-121042", "question": "What is the difference between the number of visible kitchen outlets that were plugged in and the number of visible kitchen outlets that were never plugged in by the end of the video?", "answer_choice_0": "4", "answer_choice_1": "6", "answer_choice_2": "7", "answer_choice_3": "5", "answer_choice_4": "8", "answer_id": 1, "answer": "6", "question_type": [ "Action Recognition", "Counting", "Numerical Reasoning", "Object Recognition", "Situational Awareness", "State Changes" ], "objects_of_interest": [ "bag of bagels-153c0673fdd61477", "medicine box-a3d8fcaa7aad548d", "glass5-d7419820dc2c2641", "tupperware-152598eab58ac975", "lid of tupperware-c7e0c66d804d4e6a", "roll of kitchen towel-65dd1a2f043c18dc", "can opener-a5b94da9cd002cb7", "piece of plastic-9da6992cbd62fa8d", "oranges mesh2-8a6cf06a891a70fa", "oranges mesh-8ebdee21e22805ba", "scissors guard-246ff4a57cbce4ad", "scissors-bb8255e02ae1e4ae", "blade cover2-1d9e63e2b58a5123", "blade cover1-c7d0d86a80bf0c0f", "plastic piece circular-53271ebfb0d03eb1", "glass4-1700579fc9dddb25", "tin of cocoa powder-0c144f3fc7cbbd66", "bowl3-ea4d5dca7d1fb5b6", "plate4-d749259fc8ca2033", "salt bottle-2df853a63e0f2951", "soup packets box-bebacc4e1b2fc36e", "jar-b346c4ac3de06bae", "packet-a0581a0332e74695", "mug-0e8f1bb541fa3ae9", "plate3-2840bba104267155", "knife4-c1f0b9e79aba6417", "large knife-780aa12f43058319", "chopping board2-0b038e808fd27ccc", "baking tray-109b3f8aab229736", "saucepan3-709ea5142ba420c8", "plate2-3f813a620242f873", "pot-210720946e78f619", "large plastic spoon-e9f1a2d844db1353", "strainer-380139925db102d4", "fork2-ffeca99db06d5dd2", "knife3-5502c51df2590d66", "fork-0a9345c7b8cd5880", "washing up liquid bottle-8547e271ebec77c9", "sponge-04aff40e7c88fd05", "spoon-6dbb321dd6b43d48", "left glove-9ac2e9e9046d5f25", "right glove-3ee9b1fdbc1dfef4", "plastic bag2-05805d3737847b90", "chopping board-f8de86a2d50d4ed9", "glass3-d669ab66f4974fc2", "large plastic bowl-16df4d9d4af0e47b", "lid of food processor-ee24ee2a177c4dc9", "biscuits3-e683eacaf91a71c4", "biscuits2-11ece4abb786ae11", "second packet of biscuits-b79a6911948f131d", "one tube of biscuits-43d560e9bfd95aab", "new packet of biscuits-16867c2cb8adb2ba", "plastic packaging2-eada36ce4f8f73e5", "biscuits1-2f44aa8e57cb6b98", "plastic packaging-149dac4d39715129", "blade-7b1da9ab2a4df098", "saucepan2-9da700035a9786b3", "towel-00625522044ee91f", "plate-94998c22011a2b48", "lid2-b841e1dc27a59183", "lid-7fb7c1b6985182e4", "food processor plug-5efeebd54e423176", "food processor-62c72d1a1a65365b", "centre piece of food processor-f176edcdf638e3a8", "food processor bowl-e5a819f6be32d6e9", "wooden spatula-08a837e2faee1615", "one fallen date-60fdca734b28ede9", "pitted dates20-d8b699b34e79c319", "pitted dates19-d366f77432264f09", "pitted dates18-3ef341bd5f56fa4b", "pitted dates17-d5e248a04b63b96b", "pitted dates16-bf91f9ff1d0f5528", "pitted dates15-ed2ae9851cbbfde9", "pitted dates14-ba3300155986a412", "pitted dates13-1afc573162d3c0cb", "pitted dates12-bd78da1768e2a388", "pit23-60a5ac1615e34a07", "date23-b8f8cd9918a89fa0", "pit22-b214e8c39519c7c6", "date22-332d25b1af8723a9", "pit21-e0ad27141d45f334", "date21-58e03f20607143da", "pit20-d48d7ec92570fffd", "date20-34d2f4160ec2b610", "pit19-7d79ff0908e1f810", "date19-ebd1e8368e774501", "remaining dates-280db61b861bfb14", "few pitted dates11-6c47844eb5473eab", "few pitted dates10-06b3c845e1fe2226", "few pitted dates9-237e6ce575b658f5", "few pitted dates8-1218a8655ba27a48", "few pitted dates7-5a66788e54540761", "few pitted dates6-dc3f6fe36a481b5f", "pitted date5-6b3ed4fcf5f9a86e", "pitted date4-27e75bea33e5c90f", "pitted date3-8a0d58d957ec7697", "two pitted dates-69bbb53ab894e261", "handful of pitted dates-b55a740c8e3fb0f0", "glass2-86974b5f919d4e60", "glass1-0ac4b6c1e31f8bd4", "pit18-9a43224881f13bdd", "date18-27f5c16b635948b9", "pit17-91f81d267577509f", "date17-9e023ce14d581242", "pit16-ae4215e902509d3d", "date16-8b1065206ddefb90", "pit15-305264da14998461", "date15-f538d6ce88b034a5", "pit14-d270e2a6687c24f9", "date14-54127f608e4d0caf", "pit13-256856040eec03c8", "date13-8d1028ecbc426468", "pit12-5d712e79351c4d81", "date12-5e5e0a7f8913e8c5", "pit11-e23810ca2db561be", "date11-part-7999e894a0df25c5", "date11-5de863c9e90b2879", "pit10-63299058578ae8ab", "date10-part-2351ed68668bd070", "date10-7d36aa35cd8718d4", "pit9-0c92c3471984818a", "date9-53c1beb54cd290d7", "pit8-4ed09c1e3af29c4e", "date8-be88c56d1dd28da4", "pit7-51b1c77a2cc5d7c0", "date7-75b6f299949b1e8e", "pit6-1cc3757996ca2ac6", "date6-a2ba809bbc011fd5", "pit5-1a046d61fc220511", "date5-558eed3b8d1cd355", "pit4-d055d33ebde2939c", "date4-c401a39cb66dd9b6", "pit3-1d67a59dc4c4619c", "date3-part-b82f41b43f9d2cb6", "date3-ce85176fff8cd296", "pit2-01c32435697228df", "date2-part-27cb927eeb4edac8", "date2-4e97581c69f3b6a2", "pit1-5ff35b63a8c554f1", "date1-21bf734a43bca0ad", "bowl2-f3ecbf5e23c95e49", "handful of dates4-4602e528bc3e674e", "one date-6cb95d350d4f74d2", "water filter jug-c6165db33e0b1705", "handful of dates3-1f5b6a6f26d4d92d", "handful of dates2-df4f37501ad70175", "handful of dates-984a4976525d101d", "bowl-034c87f8aaff2a97", "plastic bag of dates-6db555c7bee29c52", "bag containing bag of dates-5607343d67c9897a", "second bread bag-3b44dd5d6c8cde97", "bread bag-96341b192e706311", "butter cube-fbd8caf969a21e94", "knife-2edd39f49e4937da", "butter packaging-fb7c3fe948bbb4f6", "notepad-254d634920ab4db3", "pen-23749739cc4dc902", "saucepan-9ab0ba1deef4dbd1", "plastic piece-dda20dcf99e9cafb", "scale-d0e06fd11d88bbc4", "bottle-1b8e66c7851ecd60", "new spoon-aa98f768eb8c2565", "new knife-007f39e9b56757a1" ], "reasoning": "1. From the beginning of the video, I counted each outlet that has been plugged in. I spotted 2 in the wall by the air fryer at 00:13, 2 next to the oven at 00:21, 1 at 01:18 next to the window, and 1 above the brown trash container on the kitchen counter at 02:14. This totals 6 outlets that have been plugged in.\n2. I observed the setting of the kitchen for any unused outlets. I saw 2 at 02:09 that have no plugs in them.\n3. I watched until the end of the video to see if the user plugged in any additional outlets. At 07:40, the user plugs in the food processor plug (food processor plug-5efeebd54e423176). This totals 7 outlets that have been plugged in by the end of the video.\n4. Because the user plugged in 1 of the free outlets, only 1 outlet remained free to the end of the video. The difference between the outlets plugged, 7, and the outlets that were never plugged, 1, is 6.", "referenced_object_masks": [ "food processor plug-5efeebd54e423176" ] }, { "key": "P01-20240204-121042:c3356cabbe77d75be574c97fe4088b3cea04b9c4", "video_id": "P01-20240204-121042", "question": "Which set of directions would lead me to the red umbrella from the place where I heard 10 consecutive clicks?", "answer_choice_0": "Walk toward the exercise equipment and turn right", "answer_choice_1": "Walk toward the stove, turn right, then turn left", "answer_choice_2": "Walk toward the refrigerator, turn left, then turn left", "answer_choice_3": "Walk toward the microwave, turn right, then turn right", "answer_choice_4": "Walk toward the sink and turn right", "answer_id": 2, "answer": "Walk toward the refrigerator, turn left, then turn left", "question_type": [ "Audio Reasoning", "Egocentric & Spatial Perception", "Situational Awareness", "Object Recognition", "Counterfactual" ], "objects_of_interest": [ "bag of bagels-153c0673fdd61477", "medicine box-a3d8fcaa7aad548d", "glass5-d7419820dc2c2641", "tupperware-152598eab58ac975", "lid of tupperware-c7e0c66d804d4e6a", "roll of kitchen towel-65dd1a2f043c18dc", "can opener-a5b94da9cd002cb7", "piece of plastic-9da6992cbd62fa8d", "oranges mesh2-8a6cf06a891a70fa", "oranges mesh-8ebdee21e22805ba", "scissors guard-246ff4a57cbce4ad", "scissors-bb8255e02ae1e4ae", "blade cover2-1d9e63e2b58a5123", "blade cover1-c7d0d86a80bf0c0f", "plastic piece circular-53271ebfb0d03eb1", "glass4-1700579fc9dddb25", "tin of cocoa powder-0c144f3fc7cbbd66", "bowl3-ea4d5dca7d1fb5b6", "plate4-d749259fc8ca2033", "salt bottle-2df853a63e0f2951", "soup packets box-bebacc4e1b2fc36e", "jar-b346c4ac3de06bae", "packet-a0581a0332e74695", "mug-0e8f1bb541fa3ae9", "plate3-2840bba104267155", "knife4-c1f0b9e79aba6417", "large knife-780aa12f43058319", "chopping board2-0b038e808fd27ccc", "baking tray-109b3f8aab229736", "saucepan3-709ea5142ba420c8", "plate2-3f813a620242f873", "pot-210720946e78f619", "large plastic spoon-e9f1a2d844db1353", "strainer-380139925db102d4", "fork2-ffeca99db06d5dd2", "knife3-5502c51df2590d66", "fork-0a9345c7b8cd5880", "washing up liquid bottle-8547e271ebec77c9", "sponge-04aff40e7c88fd05", "spoon-6dbb321dd6b43d48", "left glove-9ac2e9e9046d5f25", "right glove-3ee9b1fdbc1dfef4", "plastic bag2-05805d3737847b90", "chopping board-f8de86a2d50d4ed9", "glass3-d669ab66f4974fc2", "large plastic bowl-16df4d9d4af0e47b", "lid of food processor-ee24ee2a177c4dc9", "biscuits3-e683eacaf91a71c4", "biscuits2-11ece4abb786ae11", "second packet of biscuits-b79a6911948f131d", "one tube of biscuits-43d560e9bfd95aab", "new packet of biscuits-16867c2cb8adb2ba", "plastic packaging2-eada36ce4f8f73e5", "biscuits1-2f44aa8e57cb6b98", "plastic packaging-149dac4d39715129", "blade-7b1da9ab2a4df098", "saucepan2-9da700035a9786b3", "towel-00625522044ee91f", "plate-94998c22011a2b48", "lid2-b841e1dc27a59183", "lid-7fb7c1b6985182e4", "food processor plug-5efeebd54e423176", "food processor-62c72d1a1a65365b", "centre piece of food processor-f176edcdf638e3a8", "food processor bowl-e5a819f6be32d6e9", "wooden spatula-08a837e2faee1615", "one fallen date-60fdca734b28ede9", "pitted dates20-d8b699b34e79c319", "pitted dates19-d366f77432264f09", "pitted dates18-3ef341bd5f56fa4b", "pitted dates17-d5e248a04b63b96b", "pitted dates16-bf91f9ff1d0f5528", "pitted dates15-ed2ae9851cbbfde9", "pitted dates14-ba3300155986a412", "pitted dates13-1afc573162d3c0cb", "pitted dates12-bd78da1768e2a388", "pit23-60a5ac1615e34a07", "date23-b8f8cd9918a89fa0", "pit22-b214e8c39519c7c6", "date22-332d25b1af8723a9", "pit21-e0ad27141d45f334", "date21-58e03f20607143da", "pit20-d48d7ec92570fffd", "date20-34d2f4160ec2b610", "pit19-7d79ff0908e1f810", "date19-ebd1e8368e774501", "remaining dates-280db61b861bfb14", "few pitted dates11-6c47844eb5473eab", "few pitted dates10-06b3c845e1fe2226", "few pitted dates9-237e6ce575b658f5", "few pitted dates8-1218a8655ba27a48", "few pitted dates7-5a66788e54540761", "few pitted dates6-dc3f6fe36a481b5f", "pitted date5-6b3ed4fcf5f9a86e", "pitted date4-27e75bea33e5c90f", "pitted date3-8a0d58d957ec7697", "two pitted dates-69bbb53ab894e261", "handful of pitted dates-b55a740c8e3fb0f0", "glass2-86974b5f919d4e60", "glass1-0ac4b6c1e31f8bd4", "pit18-9a43224881f13bdd", "date18-27f5c16b635948b9", "pit17-91f81d267577509f", "date17-9e023ce14d581242", "pit16-ae4215e902509d3d", "date16-8b1065206ddefb90", "pit15-305264da14998461", "date15-f538d6ce88b034a5", "pit14-d270e2a6687c24f9", "date14-54127f608e4d0caf", "pit13-256856040eec03c8", "date13-8d1028ecbc426468", "pit12-5d712e79351c4d81", "date12-5e5e0a7f8913e8c5", "pit11-e23810ca2db561be", "date11-part-7999e894a0df25c5", "date11-5de863c9e90b2879", "pit10-63299058578ae8ab", "date10-part-2351ed68668bd070", "date10-7d36aa35cd8718d4", "pit9-0c92c3471984818a", "date9-53c1beb54cd290d7", "pit8-4ed09c1e3af29c4e", "date8-be88c56d1dd28da4", "pit7-51b1c77a2cc5d7c0", "date7-75b6f299949b1e8e", "pit6-1cc3757996ca2ac6", "date6-a2ba809bbc011fd5", "pit5-1a046d61fc220511", "date5-558eed3b8d1cd355", "pit4-d055d33ebde2939c", "date4-c401a39cb66dd9b6", "pit3-1d67a59dc4c4619c", "date3-part-b82f41b43f9d2cb6", "date3-ce85176fff8cd296", "pit2-01c32435697228df", "date2-part-27cb927eeb4edac8", "date2-4e97581c69f3b6a2", "pit1-5ff35b63a8c554f1", "date1-21bf734a43bca0ad", "bowl2-f3ecbf5e23c95e49", "handful of dates4-4602e528bc3e674e", "one date-6cb95d350d4f74d2", "water filter jug-c6165db33e0b1705", "handful of dates3-1f5b6a6f26d4d92d", "handful of dates2-df4f37501ad70175", "handful of dates-984a4976525d101d", "bowl-034c87f8aaff2a97", "plastic bag of dates-6db555c7bee29c52", "bag containing bag of dates-5607343d67c9897a", "second bread bag-3b44dd5d6c8cde97", "bread bag-96341b192e706311", "butter cube-fbd8caf969a21e94", "knife-2edd39f49e4937da", "butter packaging-fb7c3fe948bbb4f6", "notepad-254d634920ab4db3", "pen-23749739cc4dc902", "saucepan-9ab0ba1deef4dbd1", "plastic piece-dda20dcf99e9cafb", "scale-d0e06fd11d88bbc4", "bottle-1b8e66c7851ecd60", "new spoon-aa98f768eb8c2565", "new knife-007f39e9b56757a1" ], "reasoning": "1. I listened carefully to the video until I heard 10 consecutive clicks. This happens when the user turns the knob at 07:13 to turn on the stove . The rapid clicks are heard from 07:13-07:15.\n2. I then identified the red umbrella at 00:02 hanging from a skylight. It hangs in the room adjacent to the kitchen, across from the opening in the wall above the kitchen counter. This opening is directly opposite of the front of the stove .\n3. Then, I determined what the user would need to do to reach the red umbrella from the stove . I saw a wide shot of the kitchen and its adjacent room at 00:03, and in this shot, both the stove and the red umbrella are visible.\n4. Using this as a reference point, the final steps would be: walk toward the refrigerator, turn left to exit the kitchen, and turn left again to enter the space framed by the opening above the kitchen counter.", "referenced_object_masks": [] }, { "key": "P01-20240204-121042:e20a9b86c00b48d66be4dae6c34d61ba57f2e744", "video_id": "P01-20240204-121042", "question": "During the first 4 minutes of the video, what items do I place on top of another item before I write in the notepad?", "answer_choice_0": "pan, notepad, scale", "answer_choice_1": "cup, pen, pan", "answer_choice_2": "notepad, pan, plastic piece", "answer_choice_3": "plastic piece, pan, cup", "answer_choice_4": "notepad, plastic piece, scale", "answer_id": 2, "answer": "notepad, pan, plastic piece", "question_type": [ "Object Recognition", "Action Recognition", "Temporal Reasoning", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "bag of bagels-153c0673fdd61477", "medicine box-a3d8fcaa7aad548d", "glass5-d7419820dc2c2641", "tupperware-152598eab58ac975", "lid of tupperware-c7e0c66d804d4e6a", "roll of kitchen towel-65dd1a2f043c18dc", "can opener-a5b94da9cd002cb7", "piece of plastic-9da6992cbd62fa8d", "oranges mesh2-8a6cf06a891a70fa", "oranges mesh-8ebdee21e22805ba", "scissors guard-246ff4a57cbce4ad", "scissors-bb8255e02ae1e4ae", "blade cover2-1d9e63e2b58a5123", "blade cover1-c7d0d86a80bf0c0f", "plastic piece circular-53271ebfb0d03eb1", "glass4-1700579fc9dddb25", "tin of cocoa powder-0c144f3fc7cbbd66", "bowl3-ea4d5dca7d1fb5b6", "plate4-d749259fc8ca2033", "salt bottle-2df853a63e0f2951", "soup packets box-bebacc4e1b2fc36e", "jar-b346c4ac3de06bae", "packet-a0581a0332e74695", "mug-0e8f1bb541fa3ae9", "plate3-2840bba104267155", "knife4-c1f0b9e79aba6417", "large knife-780aa12f43058319", "chopping board2-0b038e808fd27ccc", "baking tray-109b3f8aab229736", "saucepan3-709ea5142ba420c8", "plate2-3f813a620242f873", "pot-210720946e78f619", "large plastic spoon-e9f1a2d844db1353", "strainer-380139925db102d4", "fork2-ffeca99db06d5dd2", "knife3-5502c51df2590d66", "fork-0a9345c7b8cd5880", "washing up liquid bottle-8547e271ebec77c9", "sponge-04aff40e7c88fd05", "spoon-6dbb321dd6b43d48", "left glove-9ac2e9e9046d5f25", "right glove-3ee9b1fdbc1dfef4", "plastic bag2-05805d3737847b90", "chopping board-f8de86a2d50d4ed9", "glass3-d669ab66f4974fc2", "large plastic bowl-16df4d9d4af0e47b", "lid of food processor-ee24ee2a177c4dc9", "biscuits3-e683eacaf91a71c4", "biscuits2-11ece4abb786ae11", "second packet of biscuits-b79a6911948f131d", "one tube of biscuits-43d560e9bfd95aab", "new packet of biscuits-16867c2cb8adb2ba", "plastic packaging2-eada36ce4f8f73e5", "biscuits1-2f44aa8e57cb6b98", "plastic packaging-149dac4d39715129", "blade-7b1da9ab2a4df098", "saucepan2-9da700035a9786b3", "towel-00625522044ee91f", "plate-94998c22011a2b48", "lid2-b841e1dc27a59183", "lid-7fb7c1b6985182e4", "food processor plug-5efeebd54e423176", "food processor-62c72d1a1a65365b", "centre piece of food processor-f176edcdf638e3a8", "food processor bowl-e5a819f6be32d6e9", "wooden spatula-08a837e2faee1615", "one fallen date-60fdca734b28ede9", "pitted dates20-d8b699b34e79c319", "pitted dates19-d366f77432264f09", "pitted dates18-3ef341bd5f56fa4b", "pitted dates17-d5e248a04b63b96b", "pitted dates16-bf91f9ff1d0f5528", "pitted dates15-ed2ae9851cbbfde9", "pitted dates14-ba3300155986a412", "pitted dates13-1afc573162d3c0cb", "pitted dates12-bd78da1768e2a388", "pit23-60a5ac1615e34a07", "date23-b8f8cd9918a89fa0", "pit22-b214e8c39519c7c6", "date22-332d25b1af8723a9", "pit21-e0ad27141d45f334", "date21-58e03f20607143da", "pit20-d48d7ec92570fffd", "date20-34d2f4160ec2b610", "pit19-7d79ff0908e1f810", "date19-ebd1e8368e774501", "remaining dates-280db61b861bfb14", "few pitted dates11-6c47844eb5473eab", "few pitted dates10-06b3c845e1fe2226", "few pitted dates9-237e6ce575b658f5", "few pitted dates8-1218a8655ba27a48", "few pitted dates7-5a66788e54540761", "few pitted dates6-dc3f6fe36a481b5f", "pitted date5-6b3ed4fcf5f9a86e", "pitted date4-27e75bea33e5c90f", "pitted date3-8a0d58d957ec7697", "two pitted dates-69bbb53ab894e261", "handful of pitted dates-b55a740c8e3fb0f0", "glass2-86974b5f919d4e60", "glass1-0ac4b6c1e31f8bd4", "pit18-9a43224881f13bdd", "date18-27f5c16b635948b9", "pit17-91f81d267577509f", "date17-9e023ce14d581242", "pit16-ae4215e902509d3d", "date16-8b1065206ddefb90", "pit15-305264da14998461", "date15-f538d6ce88b034a5", "pit14-d270e2a6687c24f9", "date14-54127f608e4d0caf", "pit13-256856040eec03c8", "date13-8d1028ecbc426468", "pit12-5d712e79351c4d81", "date12-5e5e0a7f8913e8c5", "pit11-e23810ca2db561be", "date11-part-7999e894a0df25c5", "date11-5de863c9e90b2879", "pit10-63299058578ae8ab", "date10-part-2351ed68668bd070", "date10-7d36aa35cd8718d4", "pit9-0c92c3471984818a", "date9-53c1beb54cd290d7", "pit8-4ed09c1e3af29c4e", "date8-be88c56d1dd28da4", "pit7-51b1c77a2cc5d7c0", "date7-75b6f299949b1e8e", "pit6-1cc3757996ca2ac6", "date6-a2ba809bbc011fd5", "pit5-1a046d61fc220511", "date5-558eed3b8d1cd355", "pit4-d055d33ebde2939c", "date4-c401a39cb66dd9b6", "pit3-1d67a59dc4c4619c", "date3-part-b82f41b43f9d2cb6", "date3-ce85176fff8cd296", "pit2-01c32435697228df", "date2-part-27cb927eeb4edac8", "date2-4e97581c69f3b6a2", "pit1-5ff35b63a8c554f1", "date1-21bf734a43bca0ad", "bowl2-f3ecbf5e23c95e49", "handful of dates4-4602e528bc3e674e", "one date-6cb95d350d4f74d2", "water filter jug-c6165db33e0b1705", "handful of dates3-1f5b6a6f26d4d92d", "handful of dates2-df4f37501ad70175", "handful of dates-984a4976525d101d", "bowl-034c87f8aaff2a97", "plastic bag of dates-6db555c7bee29c52", "bag containing bag of dates-5607343d67c9897a", "second bread bag-3b44dd5d6c8cde97", "bread bag-96341b192e706311", "butter cube-fbd8caf969a21e94", "knife-2edd39f49e4937da", "butter packaging-fb7c3fe948bbb4f6", "notepad-254d634920ab4db3", "pen-23749739cc4dc902", "saucepan-9ab0ba1deef4dbd1", "plastic piece-dda20dcf99e9cafb", "scale-d0e06fd11d88bbc4", "bottle-1b8e66c7851ecd60", "new spoon-aa98f768eb8c2565", "new knife-007f39e9b56757a1" ], "reasoning": "1. The user first writes in the notepad (notepad-254d634920ab4db3) at 00:10.\n2. I watched the video from the beginning to 00:10 to determine which items the user places on another item.\n3. At 00:05, the user places the plastic piece (plastic piece-dda20dcf99e9cafb) on the cutting board (chopping board-f8de86a2d50d4ed9).\n4. At 00:06, the user places the pan (saucepan-9ab0ba1deef4dbd1) on top of the scale (scale-d0e06fd11d88bbc4).\n5. At 00:09, the user places the notepad (notepad-254d634920ab4db3) on top of the cutting board (chopping board-f8de86a2d50d4ed9). This is the last item before the user writes in the notepad (notepad-254d634920ab4db3) at 00:10.", "referenced_object_masks": [ "plastic piece-dda20dcf99e9cafb", "scale-d0e06fd11d88bbc4", "notepad-254d634920ab4db3", "chopping board-f8de86a2d50d4ed9", "saucepan-9ab0ba1deef4dbd1" ] }, { "key": "P01-20240204-121042:ff8a0cb07540217edab38f91e8b626054223cd24", "video_id": "P01-20240204-121042", "question": "After I use the sink for the seventh time, what is the third action I do that involves my left hand?", "answer_choice_0": "hold down the notepad", "answer_choice_1": "Cover up a bowl", "answer_choice_2": "use a towel", "answer_choice_3": "pick up dates", "answer_choice_4": "write on the notepad", "answer_id": 1, "answer": "Cover up a bowl", "question_type": [ "Action Recognition", "Temporal Reasoning", "Object Recognition" ], "objects_of_interest": [ "bag of bagels-153c0673fdd61477", "medicine box-a3d8fcaa7aad548d", "glass5-d7419820dc2c2641", "tupperware-152598eab58ac975", "lid of tupperware-c7e0c66d804d4e6a", "roll of kitchen towel-65dd1a2f043c18dc", "can opener-a5b94da9cd002cb7", "piece of plastic-9da6992cbd62fa8d", "oranges mesh2-8a6cf06a891a70fa", "oranges mesh-8ebdee21e22805ba", "scissors guard-246ff4a57cbce4ad", "scissors-bb8255e02ae1e4ae", "blade cover2-1d9e63e2b58a5123", "blade cover1-c7d0d86a80bf0c0f", "plastic piece circular-53271ebfb0d03eb1", "glass4-1700579fc9dddb25", "tin of cocoa powder-0c144f3fc7cbbd66", "bowl3-ea4d5dca7d1fb5b6", "plate4-d749259fc8ca2033", "salt bottle-2df853a63e0f2951", "soup packets box-bebacc4e1b2fc36e", "jar-b346c4ac3de06bae", "packet-a0581a0332e74695", "mug-0e8f1bb541fa3ae9", "plate3-2840bba104267155", "knife4-c1f0b9e79aba6417", "large knife-780aa12f43058319", "chopping board2-0b038e808fd27ccc", "baking tray-109b3f8aab229736", "saucepan3-709ea5142ba420c8", "plate2-3f813a620242f873", "pot-210720946e78f619", "large plastic spoon-e9f1a2d844db1353", "strainer-380139925db102d4", "fork2-ffeca99db06d5dd2", "knife3-5502c51df2590d66", "fork-0a9345c7b8cd5880", "washing up liquid bottle-8547e271ebec77c9", "sponge-04aff40e7c88fd05", "spoon-6dbb321dd6b43d48", "left glove-9ac2e9e9046d5f25", "right glove-3ee9b1fdbc1dfef4", "plastic bag2-05805d3737847b90", "chopping board-f8de86a2d50d4ed9", "glass3-d669ab66f4974fc2", "large plastic bowl-16df4d9d4af0e47b", "lid of food processor-ee24ee2a177c4dc9", "biscuits3-e683eacaf91a71c4", "biscuits2-11ece4abb786ae11", "second packet of biscuits-b79a6911948f131d", "one tube of biscuits-43d560e9bfd95aab", "new packet of biscuits-16867c2cb8adb2ba", "plastic packaging2-eada36ce4f8f73e5", "biscuits1-2f44aa8e57cb6b98", "plastic packaging-149dac4d39715129", "blade-7b1da9ab2a4df098", "saucepan2-9da700035a9786b3", "towel-00625522044ee91f", "plate-94998c22011a2b48", "lid2-b841e1dc27a59183", "lid-7fb7c1b6985182e4", "food processor plug-5efeebd54e423176", "food processor-62c72d1a1a65365b", "centre piece of food processor-f176edcdf638e3a8", "food processor bowl-e5a819f6be32d6e9", "wooden spatula-08a837e2faee1615", "one fallen date-60fdca734b28ede9", "pitted dates20-d8b699b34e79c319", "pitted dates19-d366f77432264f09", "pitted dates18-3ef341bd5f56fa4b", "pitted dates17-d5e248a04b63b96b", "pitted dates16-bf91f9ff1d0f5528", "pitted dates15-ed2ae9851cbbfde9", "pitted dates14-ba3300155986a412", "pitted dates13-1afc573162d3c0cb", "pitted dates12-bd78da1768e2a388", "pit23-60a5ac1615e34a07", "date23-b8f8cd9918a89fa0", "pit22-b214e8c39519c7c6", "date22-332d25b1af8723a9", "pit21-e0ad27141d45f334", "date21-58e03f20607143da", "pit20-d48d7ec92570fffd", "date20-34d2f4160ec2b610", "pit19-7d79ff0908e1f810", "date19-ebd1e8368e774501", "remaining dates-280db61b861bfb14", "few pitted dates11-6c47844eb5473eab", "few pitted dates10-06b3c845e1fe2226", "few pitted dates9-237e6ce575b658f5", "few pitted dates8-1218a8655ba27a48", "few pitted dates7-5a66788e54540761", "few pitted dates6-dc3f6fe36a481b5f", "pitted date5-6b3ed4fcf5f9a86e", "pitted date4-27e75bea33e5c90f", "pitted date3-8a0d58d957ec7697", "two pitted dates-69bbb53ab894e261", "handful of pitted dates-b55a740c8e3fb0f0", "glass2-86974b5f919d4e60", "glass1-0ac4b6c1e31f8bd4", "pit18-9a43224881f13bdd", "date18-27f5c16b635948b9", "pit17-91f81d267577509f", "date17-9e023ce14d581242", "pit16-ae4215e902509d3d", "date16-8b1065206ddefb90", "pit15-305264da14998461", "date15-f538d6ce88b034a5", "pit14-d270e2a6687c24f9", "date14-54127f608e4d0caf", "pit13-256856040eec03c8", "date13-8d1028ecbc426468", "pit12-5d712e79351c4d81", "date12-5e5e0a7f8913e8c5", "pit11-e23810ca2db561be", "date11-part-7999e894a0df25c5", "date11-5de863c9e90b2879", "pit10-63299058578ae8ab", "date10-part-2351ed68668bd070", "date10-7d36aa35cd8718d4", "pit9-0c92c3471984818a", "date9-53c1beb54cd290d7", "pit8-4ed09c1e3af29c4e", "date8-be88c56d1dd28da4", "pit7-51b1c77a2cc5d7c0", "date7-75b6f299949b1e8e", "pit6-1cc3757996ca2ac6", "date6-a2ba809bbc011fd5", "pit5-1a046d61fc220511", "date5-558eed3b8d1cd355", "pit4-d055d33ebde2939c", "date4-c401a39cb66dd9b6", "pit3-1d67a59dc4c4619c", "date3-part-b82f41b43f9d2cb6", "date3-ce85176fff8cd296", "pit2-01c32435697228df", "date2-part-27cb927eeb4edac8", "date2-4e97581c69f3b6a2", "pit1-5ff35b63a8c554f1", "date1-21bf734a43bca0ad", "bowl2-f3ecbf5e23c95e49", "handful of dates4-4602e528bc3e674e", "one date-6cb95d350d4f74d2", "water filter jug-c6165db33e0b1705", "handful of dates3-1f5b6a6f26d4d92d", "handful of dates2-df4f37501ad70175", "handful of dates-984a4976525d101d", "bowl-034c87f8aaff2a97", "plastic bag of dates-6db555c7bee29c52", "bag containing bag of dates-5607343d67c9897a", "second bread bag-3b44dd5d6c8cde97", "bread bag-96341b192e706311", "butter cube-fbd8caf969a21e94", "knife-2edd39f49e4937da", "butter packaging-fb7c3fe948bbb4f6", "notepad-254d634920ab4db3", "pen-23749739cc4dc902", "saucepan-9ab0ba1deef4dbd1", "plastic piece-dda20dcf99e9cafb", "scale-d0e06fd11d88bbc4", "bottle-1b8e66c7851ecd60", "new spoon-aa98f768eb8c2565", "new knife-007f39e9b56757a1" ], "reasoning": "1. I counted each time the user used the sink until I reached the seventh time. The user used the sink at 01:17, 01:25, 1:33, 01:41, 01:49, 05:24, and at 06:34 for the seventh time.\n2. I then followed each action that involved the user's left hand until I reached the third action. The user dries their hands with a towel at 06:38, uses their left index finger to hold down the notepad (notepad-254d634920ab4db3) at 06:42, and then uses their left hand to cover up the bowl (bowl2-f3ecbf5e23c95e49) at 06:49. This is the third action the user takes that invovles their left hand.", "referenced_object_masks": [ "notepad-254d634920ab4db3", "bowl2-f3ecbf5e23c95e49" ] }, { "key": "P01-20240204-124504:48db410fff9669d0ad7547d3e8753af07992279b", "video_id": "P01-20240204-124504", "question": "Where did I set down the spatula for the second time?", "answer_choice_0": "On the cutting board", "answer_choice_1": "Next to the oranges", "answer_choice_2": "On the notepad", "answer_choice_3": "There isn't a second time", "answer_choice_4": "To the right of the bowl", "answer_id": 0, "answer": "On the cutting board", "question_type": [ "Object Recognition", "Situational Awareness", "Temporal Reasoning", "Action Recognition" ], "objects_of_interest": [ "bowl2-ce24f1368b7fc2bc", "bowl-220edc4e74ff8c1a", "spatula-80872b8215e4d7cf" ], "reasoning": "1. I watched and noted when the user set down the spatula (spatula-80872b8215e4d7cf). This occurred at 00:11 and 00:43.\n2. In the second instance, at 00:43, the user set the spatula (spatula-80872b8215e4d7cf) down on the cutting board .", "referenced_object_masks": [ "spatula-80872b8215e4d7cf" ] }, { "key": "P01-20240204-124504:7322cc0d30a08a16c357768d268222895b39299d", "video_id": "P01-20240204-124504", "question": "What item am I holding when the umbrella hanging from the skylight is visible for the second time?", "answer_choice_0": "a plastic container", "answer_choice_1": "the big bowl", "answer_choice_2": "the spatula", "answer_choice_3": "no item", "answer_choice_4": "the small bowl", "answer_id": 1, "answer": "the big bowl", "question_type": [ "Situational Awareness", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "bowl2-ce24f1368b7fc2bc", "bowl-220edc4e74ff8c1a", "spatula-80872b8215e4d7cf" ], "reasoning": "1. I watched the video until the skylights came into full view. The first opportunity is at 00:03, when the red umbrella is hanging clearly from one of the skylights. The second instance is at 01:22.\n2. At 01:22, the user is holding the big white bowl (bowl2-ce24f1368b7fc2bc) with both hands.", "referenced_object_masks": [ "bowl2-ce24f1368b7fc2bc" ] }, { "key": "P01-20240204-124504:7c8ce9140b2e141ee2870d7b773c827d5a079d9b", "video_id": "P01-20240204-124504", "question": "What item is not visible on the counter right as I set down bowl2?", "answer_choice_0": "a roll of paper towels", "answer_choice_1": "the airfyer", "answer_choice_2": "the bowl", "answer_choice_3": "a bag of oranges", "answer_choice_4": "a spatula", "answer_id": 4, "answer": "a spatula", "question_type": [ "Situational Awareness", "Action Recognition", "Object Recognition", "Counterfactual" ], "objects_of_interest": [ "bowl2-ce24f1368b7fc2bc", "bowl-220edc4e74ff8c1a", "spatula-80872b8215e4d7cf" ], "reasoning": "1. I observed the user retrieving bowl2 (bowl2-ce24f1368b7fc2bc) from the refrigerator at 01:14 and setting it down on the counter at 01:24.\n2. I observed the counter at 01:24 for the items that were visible. This includes the bowl (bowl-220edc4e74ff8c1a) to the left, the bag of oranges on the top counter, the airfryer to the right of the oranges, and the roll of paper towels on the microwave. The spatula (spatula-80872b8215e4d7cf) is not in the frame at this time.", "referenced_object_masks": [ "bowl-220edc4e74ff8c1a", "spatula-80872b8215e4d7cf", "bowl2-ce24f1368b7fc2bc" ] }, { "key": "P01-20240204-124504:861602a989e3cb8976c2530612f1a4c5b1bfab3c", "video_id": "P01-20240204-124504", "question": "If I added the number of exercise balls in the video to the number of vessels on top of the refrigerator, what would be the total?", "answer_choice_0": "2", "answer_choice_1": "5", "answer_choice_2": "4", "answer_choice_3": "3", "answer_choice_4": "6", "answer_id": 4, "answer": "6", "question_type": [ "Situational Awareness", "Object Recognition", "Numerical Reasoning" ], "objects_of_interest": [ "bowl2-ce24f1368b7fc2bc", "bowl-220edc4e74ff8c1a", "spatula-80872b8215e4d7cf" ], "reasoning": "1. I counted 1 exercise ball in the living room in the background at 00:05, resting next to a bicycle. I watched the rest of the video and there wasn't another exercise ball present.\n2. I continued watching until the user opened up the refrigerator to retrieve the bowl with the mixture (bowl2-ce24f1368b7fc2bc) at 01:14. I observed the top of the refrigerator and counted 4 white containers and 1 yellow container, totalling 5.\n3. I then added 5 to 1, which totals 6.", "referenced_object_masks": [ "bowl2-ce24f1368b7fc2bc" ] }, { "key": "P01-20240204-124504:af1a3f4d048619ea9ffab2b476b9b9288252a479", "video_id": "P01-20240204-124504", "question": "What kitchen appliance is directly behind me while I mix the contents of the bowl?", "answer_choice_0": "the refrigerator", "answer_choice_1": "the blender", "answer_choice_2": "the stove", "answer_choice_3": "the dishwasher", "answer_choice_4": "the microwave", "answer_id": 2, "answer": "the stove", "question_type": [ "Egocentric & Spatial Perception", "Action Recognition", "Situational Awareness", "Object Recognition" ], "objects_of_interest": [ "bowl2-ce24f1368b7fc2bc", "bowl-220edc4e74ff8c1a", "spatula-80872b8215e4d7cf" ], "reasoning": "1. I observed that the user began mixing the contents of the bowl (bowl-220edc4e74ff8c1a) at 00:06.\n2. I watched the video from the beginning and saw a clear view of the kitchen as the user was walking into it at 00:03.\n3. From this view, I can see that the bowl (bowl-220edc4e74ff8c1a) is on the counter directly across from the stove.", "referenced_object_masks": [ "bowl-220edc4e74ff8c1a" ] }, { "key": "P01-20240204-130448:05390f6ce0c9f3bee462f637b3c42296a4e90d28", "video_id": "P01-20240204-130448", "question": "Which item stored back into the refrigerator do I store below eye level?", "answer_choice_0": "Bag of oranges", "answer_choice_1": "Ketchup", "answer_choice_2": "Maple syrup", "answer_choice_3": "Tupperware of balls", "answer_choice_4": "Soda", "answer_id": 0, "answer": "Bag of oranges", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition" ], "objects_of_interest": [ "orange3-d6441c11b4445a31", "orange2-ad98c9aef283a5e6", "orange1-2e3511ef5fe02090", "mesh-b5947bda8d03576b", "scissors-a6843b16785bd4b8", "bowl2-df25ecd43cd09609", "tray-bac8757e1a78a9d3", "sponge-557716e6e9c7909b", "food processor bowl lid-ce6f069e76bfd3f3", "food processor bowl-dec296a800268256", "washing up liquid bottle-327b1f0823d43b9d", "tupperware-4cb91d42601819f4", "spatula-681779e229189ae8", "chopping board-b90e7efb8c59d19f", "bowl-c48a26000400b731", "new tupperware lid-21348aa4f7a294a7" ], "reasoning": "1. I look for the times the user puts items away into the fridge .\n2. At 06:25, the user puts the tupperware (tupperware-4cb91d42601819f4) into the fridge .\n3. At 06:25, the user needs to reach the fridge's top shelf, so the tupperware (tupperware-4cb91d42601819f4) isn't below eye level.\n4. At 08:18, the user puts the mesh bag of oranges (mesh-b5947bda8d03576b) into the fridge .\n5. The user puts the oranges (mesh-b5947bda8d03576b) in the fridge's bottom drawer, which is clearly below eye level.\n6. Therefore, the bag of oranges (mesh-b5947bda8d03576b) is the answer.", "referenced_object_masks": [ "tupperware-4cb91d42601819f4", "mesh-b5947bda8d03576b" ] }, { "key": "P01-20240204-130448:8e6ad6986d44da951aac71355a6ae09e62da6528", "video_id": "P01-20240204-130448", "question": "If I added the fraction of balls placed on the cutting board to the fraction of oranges taken from the mesh bag, what would be the result?", "answer_choice_0": "4/13", "answer_choice_1": "7/15", "answer_choice_2": "10/23", "answer_choice_3": "2/7", "answer_choice_4": "1/11", "answer_id": 1, "answer": "7/15", "question_type": [ "Numerical Reasoning", "Counting", "Action Recognition", "Situational Awareness", "Object Recognition" ], "objects_of_interest": [ "orange3-d6441c11b4445a31", "orange2-ad98c9aef283a5e6", "orange1-2e3511ef5fe02090", "mesh-b5947bda8d03576b", "scissors-a6843b16785bd4b8", "bowl2-df25ecd43cd09609", "tray-bac8757e1a78a9d3", "sponge-557716e6e9c7909b", "food processor bowl lid-ce6f069e76bfd3f3", "food processor bowl-dec296a800268256", "washing up liquid bottle-327b1f0823d43b9d", "tupperware-4cb91d42601819f4", "spatula-681779e229189ae8", "chopping board-b90e7efb8c59d19f", "bowl-c48a26000400b731", "new tupperware lid-21348aa4f7a294a7" ], "reasoning": "1. I look for the user to place a ball on the cutting board (chopping board-b90e7efb8c59d19f).\n2. This happens at 01:41.\n3. I look for the total amount of balls formed and find my answer at 06:14, where I see that the user has placed 15 balls into a tupperware (tupperware-4cb91d42601819f4).\n4. I notice that the whole time the user forms the balls , from 00:04 to 06:14, the user only places one ball on the cutting board (chopping board-b90e7efb8c59d19f).\n4. I put these numbers into a fraction and come up with 1/15.\n5. I look for oranges to be taken from the mesh bag (mesh-b5947bda8d03576b).\n6. I see the user slide one orange (orange1-2e3511ef5fe02090) out of the bag from 08:03-08:04, and I see the user slide another orange (orange2-ad98c9aef283a5e6) out of the bag at 08:05.\n7. At 08:04, I find a good view of all the oranges and realize that, counting the ones that were just pulled, there were originally 5 oranges in the bag (mesh-b5947bda8d03576b).\n8. I put this into a fraction and see that the user took 2/5 oranges out of the bag.\n9. I add 1/15 to 2/5 and I get 7/15.\n10. Therefore, 7/15 is the answer.", "referenced_object_masks": [ "orange2-ad98c9aef283a5e6", "chopping board-b90e7efb8c59d19f", "tupperware-4cb91d42601819f4", "orange1-2e3511ef5fe02090", "mesh-b5947bda8d03576b" ] }, { "key": "P01-20240204-130448:cb047267f69db6addab85e3f4e2f1e55ba1fda0e", "video_id": "P01-20240204-130448", "question": "After I pass the fridge, how long does it take me to get back to it and open the fridge door?", "answer_choice_0": "6 minutes and 19 seconds", "answer_choice_1": "2 minutes and 42 seconds", "answer_choice_2": "5 minutes and 38 seconds", "answer_choice_3": "1 minute and 12 seconds", "answer_choice_4": "3 minutes and 39 seconds", "answer_id": 0, "answer": "6 minutes and 19 seconds", "question_type": [ "Action Recognition", "Temporal Reasoning", "Egocentric & Spatial Perception", "Object Recognition" ], "objects_of_interest": [ "orange3-d6441c11b4445a31", "orange2-ad98c9aef283a5e6", "orange1-2e3511ef5fe02090", "mesh-b5947bda8d03576b", "scissors-a6843b16785bd4b8", "bowl2-df25ecd43cd09609", "tray-bac8757e1a78a9d3", "sponge-557716e6e9c7909b", "food processor bowl lid-ce6f069e76bfd3f3", "food processor bowl-dec296a800268256", "washing up liquid bottle-327b1f0823d43b9d", "tupperware-4cb91d42601819f4", "spatula-681779e229189ae8", "chopping board-b90e7efb8c59d19f", "bowl-c48a26000400b731", "new tupperware lid-21348aa4f7a294a7" ], "reasoning": "1. I look for the first time the user passes the fridge and I notice it around the 00:04 timestamp.\n2. After this, I notice the user knead the mix, form balls with it, place them in a tupperware (tupperware-4cb91d42601819f4), close the tupperware off with a lid (new tupperware lid-21348aa4f7a294a7), and place the tupperware (tupperware-4cb91d42601819f4) into the fridge .\n3. The opening of the fridge door happens at 06:23.\n4. The time that has elapsed from 00:04 to 06:23 equals 6 minutes and 19 seconds.\n5. Therefore, 6 minutes and 19 seconds is the answer.", "referenced_object_masks": [ "new tupperware lid-21348aa4f7a294a7", "tupperware-4cb91d42601819f4" ] }, { "key": "P01-20240204-130448:cbea7f355d856c6454d99c1258665f1d099a1cbb", "video_id": "P01-20240204-130448", "question": "Which of the following activities do I take the longest to do?", "answer_choice_0": "Washing my hands", "answer_choice_1": "Forming the balls", "answer_choice_2": "Cutting the orange out of the mesh", "answer_choice_3": "Washing the food processor", "answer_choice_4": "Kneading the mix", "answer_id": 1, "answer": "Forming the balls", "question_type": [ "Temporal Reasoning", "State Changes", "Action Recognition", "Object Recognition" ], "objects_of_interest": [ "orange3-d6441c11b4445a31", "orange2-ad98c9aef283a5e6", "orange1-2e3511ef5fe02090", "mesh-b5947bda8d03576b", "scissors-a6843b16785bd4b8", "bowl2-df25ecd43cd09609", "tray-bac8757e1a78a9d3", "sponge-557716e6e9c7909b", "food processor bowl lid-ce6f069e76bfd3f3", "food processor bowl-dec296a800268256", "washing up liquid bottle-327b1f0823d43b9d", "tupperware-4cb91d42601819f4", "spatula-681779e229189ae8", "chopping board-b90e7efb8c59d19f", "bowl-c48a26000400b731", "new tupperware lid-21348aa4f7a294a7" ], "reasoning": "1. I examine each of the following activities and measure the time it takes to complete them.\n2. The user kneads the mix in a white bowl (bowl-c48a26000400b731) from 00:04 to 01:32, a total of 1 minute and 28 seconds.\n3. The user forms the balls from 01:36-06:14, a total of 4 minutes and 38 seconds.\n4. The user washes their hands from 06:35-06:52, a total of 17 seconds.\n5. The user washes the food processor (food processor bowl-dec296a800268256) from 07:18-07:35, a total of 17 seconds.\n6. The user takes the oranges in the mesh bag (mesh-b5947bda8d03576b) to the fridge from 08:11-08:19, a total of 8 seconds.\n7. The user forms the balls for the longest time, since no other activity lasts longer than 4 minutes and 38 seconds.\n8. Therefore, forming the balls is the answer.", "referenced_object_masks": [ "bowl-c48a26000400b731", "food processor bowl-dec296a800268256", "mesh-b5947bda8d03576b" ] }, { "key": "P01-20240204-130448:d769b05e5ac431ed425c2ef9f3ace73391abd3a9", "video_id": "P01-20240204-130448", "question": "If I wanted to wash the scissors after I sheath them, how many degrees to the right would I need to turn to face the sink?", "answer_choice_0": "45 degrees", "answer_choice_1": "90 degrees", "answer_choice_2": "315 degrees", "answer_choice_3": "135 degrees", "answer_choice_4": "270 degrees", "answer_id": 4, "answer": "270 degrees", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Counterfactual" ], "objects_of_interest": [ "orange3-d6441c11b4445a31", "orange2-ad98c9aef283a5e6", "orange1-2e3511ef5fe02090", "mesh-b5947bda8d03576b", "scissors-a6843b16785bd4b8", "bowl2-df25ecd43cd09609", "tray-bac8757e1a78a9d3", "sponge-557716e6e9c7909b", "food processor bowl lid-ce6f069e76bfd3f3", "food processor bowl-dec296a800268256", "washing up liquid bottle-327b1f0823d43b9d", "tupperware-4cb91d42601819f4", "spatula-681779e229189ae8", "chopping board-b90e7efb8c59d19f", "bowl-c48a26000400b731", "new tupperware lid-21348aa4f7a294a7" ], "reasoning": "1. I look for the time the user sheathes the scissors (scissors-a6843b16785bd4b8), and I find the moment at 08:09.\n2. I realize that the sink is directly to the left of the user, since they were just at the sink at 07:45.\n3. If the user were to turn right to face the sink , therefore, the user would need to make a wide, 270 degree turn.\n4. Therefore, 270 degrees is the answer.", "referenced_object_masks": [ "scissors-a6843b16785bd4b8" ] }, { "key": "P01-20240204-142301:0c0f55e095ae48264fa991f215383bc18931d78f", "video_id": "P01-20240204-142301", "question": "Where do I take the cutting board from the moment I pull it out of the drawer at 06:35 to the moment I leave it in its final place?", "answer_choice_0": "To the counter space, the compost bin, back to the counter space, to the sink, and back to the drawer", "answer_choice_1": "To the sink, the counter space, the compost bin, and back to the sink", "answer_choice_2": "To the counter space, then two times to the compost bin and back to the counter space, then to the stovetop, the compost bin, and the sink", "answer_choice_3": "To the sink, three times to the counter space and the compost bin, the sink, and back to the drawer", "answer_choice_4": "To the counter space, the compost bin, and the sink two times, then back to the drawer", "answer_id": 2, "answer": "To the counter space, then two times to the compost bin and back to the counter space, then to the stovetop, the compost bin, and the sink", "question_type": [ "Egocentric & Spatial Perception", "State Changes", "Object Recognition", "Action Recognition", "Situational Awareness" ], "objects_of_interest": [ "food waste bin-2ec33e268849da1c", "pot lid-f632e97c78eb5302", "juicer central part-0ef9446b6b32000e", "pint glass-82a00cd50f5e4611", "orange4 half-4fff2a83e7f88844", "orange3 half-a8423a6289658c5f", "orange mesh-3aa1e0fffbe6c2df", "orange2 half-a70502769b2a278d", "orange1 half-1b3a7815da8b768b", "food processor plug-47b5d020e71c6f14", "wrong plug-bd4a484bf4e5c893", "knife3-95697c2a2bbd5ddb", "chicken stock cube2-46dcbf252def833e", "teaspoon-89dcd301e47966d8", "chicken stock cube packaging2-8b7aecbb71521156", "mug-86debf886202916a", "plastic bag-8a541364d1e004fa", "kettle-37cc90555dd7bf83", "flask-ef4405e57730661c", "garlic presser-10dd11fd7a42a745", "pot-4f03adbe4f0f4713", "orange4-c365a9eef4feec35", "orange3-71cb26d5ab3aa6e3", "orange2-a732115852b6d410", "orange1-f7f6f1e812a63898", "wooden spatula-9592527836440d66", "scale-dc8c680d89d8b622", "juicer part-d49ba76878cb8dcc", "food processor bowl-bcb3dc03f39509ca", "heat diffuser-59aea36aad36ece6", "pan lid6-8671d1eaa61cebd2", "pan lid5-912af158ada88380", "pan lid4-9bbade45672fe4e7", "pan lid3-142f890ea752a459", "pan lid2-7a76675708c86623", "pan lid-55a6df77c6737a16", "pan-98bd4c0c7ef13902", "saucepan-fd9b979eb149cf10", "wooden spoon-7be668b519476e5e", "onion piece-aacc24f8849a666f", "half onion-3e17212325230128", "knife2-fa84f037a2db6757", "one onion-eefb212c97a4e74c", "scissors guard cover-0880d4396cb17028", "scissors-71ca5a5e668d4493", "another garlic clove-8c188aee4bae3cb3", "one garlic clove-5dccb4aa10df781a", "garlic pulp-d88829ce5c941361", "mesh of onions-8bddee00e3a058e9", "can opener-7d6f7980f515c26b", "chopping board2-1dddb247ed4076ae", "chopping board-b43c436faa839b48", "chicken stock cube-c7f49be3dc0b05d7", "packaging of meat-b96d7221c1da4827", "chicken stock cube packaging-5438b04a72c67012", "box of chicken stock cubes-632ae16822563d5c", "knife-9e007c9ad3a4ae17", "garlic piece3-188fe3e716ae5e8c", "garlic piece2-fe0794c82a598c90", "garlic piece1-1681159d8efe0802", "bottle of vegetable oil-8f37f20c9984b5c5", "large bowl2-4ccf0fdff4b2e903", "chicken piece5-2e242ddd02c4ac85", "chicken piece4-800aba2339a15b1d", "chicken piece3-03abf52571bb29be", "chicken piece2-0d7345a28bf70924", "chicken piece1-cd350488e54fba12", "cling film-fb8d988440446d77", "baking tray-783309f74e7cd029", "tupperware-11d17e118b8d1a46", "water filter jug-222231917b2f66da", "large bowl-f0bc9ffbb1831abe", "washing up liquid bottle-011cf12411c66663", "sponge-de576ed0ce7ac77e", "left glove-fb2d6fb4138d2350", "right glove-ec4db5c45409608d", "food processor lid-5c9b90c05feab9b2", "strainer-32b82b308bd95a1e" ], "reasoning": "1. I watch the moment the user pulls the cutting board (chopping board2-1dddb247ed4076ae) out of the drawer at 06:35 and track where the user takes it.\n2. At 06:38, the user, without looking, places the cutting board (chopping board2-1dddb247ed4076ae) on the counter to the left of the baking tray (baking tray-783309f74e7cd029). I hear it hit the countertop, which is how I know it is placed there.\n3. The cutting board (chopping board2-1dddb247ed4076ae) stays there until 08:16-08:24, when the user picks it up, moves it to the compost bin (food waste bin-2ec33e268849da1c), and scrapes off leftover onion skins .\n4. After 08:24, at 08:26, the user puts the cutting board (chopping board2-1dddb247ed4076ae) back in the spot to the left of the baking tray (baking tray-783309f74e7cd029).\n5. From 09:14-09:23, the user picks up the cutting board (chopping board2-1dddb247ed4076ae) once more and scrapes off leftover garlic skins into the compost bin (food waste bin-2ec33e268849da1c).\n6. At 09:26, the user puts the cutting board (chopping board2-1dddb247ed4076ae) back in its spot to the left of the baking tray (baking tray-783309f74e7cd029).\n7. At 13:11, the user picks the cutting board (chopping board2-1dddb247ed4076ae) back up and turns left to scrape diced onion off it into a pan (pan-98bd4c0c7ef13902).\n8. From 13:34-13:38, the user takes the cutting board (chopping board2-1dddb247ed4076ae) away from the pan (pan-98bd4c0c7ef13902) and to the compost bin (food waste bin-2ec33e268849da1c) to scrape off a chunk of cut onion .\n9. At 13:40, the user turns left to leave the cutting board (chopping board2-1dddb247ed4076ae) by the sink .\n10. After 13:40, the user doesn't touch the cutting board (chopping board2-1dddb247ed4076ae) again.\n11.The answer, therefore, is that after 06:38, the user moves it to the counter space, then two times to the compost bin and back to the counter space, then to the stovetop, the compost bin, and the sink.", "referenced_object_masks": [ "chopping board2-1dddb247ed4076ae", "food waste bin-2ec33e268849da1c", "pan-98bd4c0c7ef13902", "baking tray-783309f74e7cd029" ] }, { "key": "P01-20240204-142301:16246b1b33f26162f229af165c007424c6447c4b", "video_id": "P01-20240204-142301", "question": "Where did I move the strainer to after putting it inside the white bowl?", "answer_choice_0": "To the left of the stovetop", "answer_choice_1": "To the left of the sink", "answer_choice_2": "To the right of the stovetop", "answer_choice_3": "Directly on the stovetop", "answer_choice_4": "To the right of the sink", "answer_id": 2, "answer": "To the right of the stovetop", "question_type": [ "Object Recognition", "Action Recognition", "Egocentric & Spatial Perception", "Situational Awareness", "State Changes", "Temporal Reasoning" ], "objects_of_interest": [ "food waste bin-2ec33e268849da1c", "pot lid-f632e97c78eb5302", "juicer central part-0ef9446b6b32000e", "pint glass-82a00cd50f5e4611", "orange4 half-4fff2a83e7f88844", "orange3 half-a8423a6289658c5f", "orange mesh-3aa1e0fffbe6c2df", "orange2 half-a70502769b2a278d", "orange1 half-1b3a7815da8b768b", "food processor plug-47b5d020e71c6f14", "wrong plug-bd4a484bf4e5c893", "knife3-95697c2a2bbd5ddb", "chicken stock cube2-46dcbf252def833e", "teaspoon-89dcd301e47966d8", "chicken stock cube packaging2-8b7aecbb71521156", "mug-86debf886202916a", "plastic bag-8a541364d1e004fa", "kettle-37cc90555dd7bf83", "flask-ef4405e57730661c", "garlic presser-10dd11fd7a42a745", "pot-4f03adbe4f0f4713", "orange4-c365a9eef4feec35", "orange3-71cb26d5ab3aa6e3", "orange2-a732115852b6d410", "orange1-f7f6f1e812a63898", "wooden spatula-9592527836440d66", "scale-dc8c680d89d8b622", "juicer part-d49ba76878cb8dcc", "food processor bowl-bcb3dc03f39509ca", "heat diffuser-59aea36aad36ece6", "pan lid6-8671d1eaa61cebd2", "pan lid5-912af158ada88380", "pan lid4-9bbade45672fe4e7", "pan lid3-142f890ea752a459", "pan lid2-7a76675708c86623", "pan lid-55a6df77c6737a16", "pan-98bd4c0c7ef13902", "saucepan-fd9b979eb149cf10", "wooden spoon-7be668b519476e5e", "onion piece-aacc24f8849a666f", "half onion-3e17212325230128", "knife2-fa84f037a2db6757", "one onion-eefb212c97a4e74c", "scissors guard cover-0880d4396cb17028", "scissors-71ca5a5e668d4493", "another garlic clove-8c188aee4bae3cb3", "one garlic clove-5dccb4aa10df781a", "garlic pulp-d88829ce5c941361", "mesh of onions-8bddee00e3a058e9", "can opener-7d6f7980f515c26b", "chopping board2-1dddb247ed4076ae", "chopping board-b43c436faa839b48", "chicken stock cube-c7f49be3dc0b05d7", "packaging of meat-b96d7221c1da4827", "chicken stock cube packaging-5438b04a72c67012", "box of chicken stock cubes-632ae16822563d5c", "knife-9e007c9ad3a4ae17", "garlic piece3-188fe3e716ae5e8c", "garlic piece2-fe0794c82a598c90", "garlic piece1-1681159d8efe0802", "bottle of vegetable oil-8f37f20c9984b5c5", "large bowl2-4ccf0fdff4b2e903", "chicken piece5-2e242ddd02c4ac85", "chicken piece4-800aba2339a15b1d", "chicken piece3-03abf52571bb29be", "chicken piece2-0d7345a28bf70924", "chicken piece1-cd350488e54fba12", "cling film-fb8d988440446d77", "baking tray-783309f74e7cd029", "tupperware-11d17e118b8d1a46", "water filter jug-222231917b2f66da", "large bowl-f0bc9ffbb1831abe", "washing up liquid bottle-011cf12411c66663", "sponge-de576ed0ce7ac77e", "left glove-fb2d6fb4138d2350", "right glove-ec4db5c45409608d", "food processor lid-5c9b90c05feab9b2", "strainer-32b82b308bd95a1e" ], "reasoning": "1. I watched the video and found the strainer (strainer-32b82b308bd95a1e) at 00:20.\n2. I found the white bowl (large bowl-f0bc9ffbb1831abe) at 00:36.\n3. I saw the user put the strainer (strainer-32b82b308bd95a1e) inside the white bowl (large bowl-f0bc9ffbb1831abe) at 03:05.\n4. After 03:05, I saw the user move the white bowl (large bowl-f0bc9ffbb1831abe) with the strainer (strainer-32b82b308bd95a1e) to the right of the stovetop at 03:08.\n5. Therefore, the user moved the strainer (strainer-32b82b308bd95a1e) to the right of the stovetop after putting it inside the white bowl (large bowl-f0bc9ffbb1831abe).", "referenced_object_masks": [ "strainer-32b82b308bd95a1e", "large bowl-f0bc9ffbb1831abe" ] }, { "key": "P01-20240204-142301:1c6cf60f8930614d28c1086b94b995d08ce2030f", "video_id": "P01-20240204-142301", "question": "What do the last ingredient I prepared before turning the lights on and the first ingredient I prepared after I turning on the lights have in common?", "answer_choice_0": "Both ingredients are vegetables", "answer_choice_1": "I seasoned both with salt", "answer_choice_2": "I pulled them both from the fridge", "answer_choice_3": "I rinsed both under the faucet", "answer_choice_4": "I used white bowls to prepare them", "answer_id": 4, "answer": "I used white bowls to prepare them", "question_type": [ "Temporal Reasoning", "Situational Awareness", "Action Recognition", "Object Recognition" ], "objects_of_interest": [ "food waste bin-2ec33e268849da1c", "pot lid-f632e97c78eb5302", "juicer central part-0ef9446b6b32000e", "pint glass-82a00cd50f5e4611", "orange4 half-4fff2a83e7f88844", "orange3 half-a8423a6289658c5f", "orange mesh-3aa1e0fffbe6c2df", "orange2 half-a70502769b2a278d", "orange1 half-1b3a7815da8b768b", "food processor plug-47b5d020e71c6f14", "wrong plug-bd4a484bf4e5c893", "knife3-95697c2a2bbd5ddb", "chicken stock cube2-46dcbf252def833e", "teaspoon-89dcd301e47966d8", "chicken stock cube packaging2-8b7aecbb71521156", "mug-86debf886202916a", "plastic bag-8a541364d1e004fa", "kettle-37cc90555dd7bf83", "flask-ef4405e57730661c", "garlic presser-10dd11fd7a42a745", "pot-4f03adbe4f0f4713", "orange4-c365a9eef4feec35", "orange3-71cb26d5ab3aa6e3", "orange2-a732115852b6d410", "orange1-f7f6f1e812a63898", "wooden spatula-9592527836440d66", "scale-dc8c680d89d8b622", "juicer part-d49ba76878cb8dcc", "food processor bowl-bcb3dc03f39509ca", "heat diffuser-59aea36aad36ece6", "pan lid6-8671d1eaa61cebd2", "pan lid5-912af158ada88380", "pan lid4-9bbade45672fe4e7", "pan lid3-142f890ea752a459", "pan lid2-7a76675708c86623", "pan lid-55a6df77c6737a16", "pan-98bd4c0c7ef13902", "saucepan-fd9b979eb149cf10", "wooden spoon-7be668b519476e5e", "onion piece-aacc24f8849a666f", "half onion-3e17212325230128", "knife2-fa84f037a2db6757", "one onion-eefb212c97a4e74c", "scissors guard cover-0880d4396cb17028", "scissors-71ca5a5e668d4493", "another garlic clove-8c188aee4bae3cb3", "one garlic clove-5dccb4aa10df781a", "garlic pulp-d88829ce5c941361", "mesh of onions-8bddee00e3a058e9", "can opener-7d6f7980f515c26b", "chopping board2-1dddb247ed4076ae", "chopping board-b43c436faa839b48", "chicken stock cube-c7f49be3dc0b05d7", "packaging of meat-b96d7221c1da4827", "chicken stock cube packaging-5438b04a72c67012", "box of chicken stock cubes-632ae16822563d5c", "knife-9e007c9ad3a4ae17", "garlic piece3-188fe3e716ae5e8c", "garlic piece2-fe0794c82a598c90", "garlic piece1-1681159d8efe0802", "bottle of vegetable oil-8f37f20c9984b5c5", "large bowl2-4ccf0fdff4b2e903", "chicken piece5-2e242ddd02c4ac85", "chicken piece4-800aba2339a15b1d", "chicken piece3-03abf52571bb29be", "chicken piece2-0d7345a28bf70924", "chicken piece1-cd350488e54fba12", "cling film-fb8d988440446d77", "baking tray-783309f74e7cd029", "tupperware-11d17e118b8d1a46", "water filter jug-222231917b2f66da", "large bowl-f0bc9ffbb1831abe", "washing up liquid bottle-011cf12411c66663", "sponge-de576ed0ce7ac77e", "left glove-fb2d6fb4138d2350", "right glove-ec4db5c45409608d", "food processor lid-5c9b90c05feab9b2", "strainer-32b82b308bd95a1e" ], "reasoning": "1. I notice the user turn the lights on at 03:26, so I look for an ingredient prepared before this time that is still being prepared after.\n2. From 01:05-03:08, I notice the user wash rice , strain the rice , place it on a strainer (strainer-32b82b308bd95a1e), and place the strainer on a white bowl (large bowl-f0bc9ffbb1831abe) on the counter to the right of the stovetop.\n3. There is no other ingredient being prepared before the user turns on the lights, so this is the ingredient that is being prepared before.\n4. I look for the first ingredient prepared after 03:26 and see from 03:28-04:14, the user begins to prepare chicken by peeling the cling wrap (cling film-fb8d988440446d77) off the white bowl (large bowl2-4ccf0fdff4b2e903) it is in and places the chicken (chicken piece1-cd350488e54fba12, chicken piece2-0d7345a28bf70924, chicken piece3-03abf52571bb29be, chicken piece4-800aba2339a15b1d, chicken piece5-2e242ddd02c4ac85) on a baking sheet (baking tray-783309f74e7cd029) with some of its marinade.\n5. For the rest of the video, the user doesn't touch the rice . \n6. Therefore, the answer is that the user used white bowls in preparing both ingredients.", "referenced_object_masks": [ "large bowl-f0bc9ffbb1831abe", "large bowl2-4ccf0fdff4b2e903", "cling film-fb8d988440446d77", "strainer-32b82b308bd95a1e", "chicken piece1-cd350488e54fba12", "chicken piece2-0d7345a28bf70924", "chicken piece3-03abf52571bb29be", "chicken piece4-800aba2339a15b1d", "chicken piece5-2e242ddd02c4ac85", "baking tray-783309f74e7cd029" ] }, { "key": "P01-20240204-142301:71ce4686f83f08ae723ea8d623411953958c6a93", "video_id": "P01-20240204-142301", "question": "Which finger do I use both to press the second oven knob and to help hold the knife just before cutting the first chicken stock cube?", "answer_choice_0": "Ring", "answer_choice_1": "Thumb", "answer_choice_2": "Pointer", "answer_choice_3": "Middle", "answer_choice_4": "Pinkie", "answer_id": 3, "answer": "Middle", "question_type": [ "Egocentric & Spatial Perception", "Action Recognition", "Object Recognition" ], "objects_of_interest": [ "food waste bin-2ec33e268849da1c", "pot lid-f632e97c78eb5302", "juicer central part-0ef9446b6b32000e", "pint glass-82a00cd50f5e4611", "orange4 half-4fff2a83e7f88844", "orange3 half-a8423a6289658c5f", "orange mesh-3aa1e0fffbe6c2df", "orange2 half-a70502769b2a278d", "orange1 half-1b3a7815da8b768b", "food processor plug-47b5d020e71c6f14", "wrong plug-bd4a484bf4e5c893", "knife3-95697c2a2bbd5ddb", "chicken stock cube2-46dcbf252def833e", "teaspoon-89dcd301e47966d8", "chicken stock cube packaging2-8b7aecbb71521156", "mug-86debf886202916a", "plastic bag-8a541364d1e004fa", "kettle-37cc90555dd7bf83", "flask-ef4405e57730661c", "garlic presser-10dd11fd7a42a745", "pot-4f03adbe4f0f4713", "orange4-c365a9eef4feec35", "orange3-71cb26d5ab3aa6e3", "orange2-a732115852b6d410", "orange1-f7f6f1e812a63898", "wooden spatula-9592527836440d66", "scale-dc8c680d89d8b622", "juicer part-d49ba76878cb8dcc", "food processor bowl-bcb3dc03f39509ca", "heat diffuser-59aea36aad36ece6", "pan lid6-8671d1eaa61cebd2", "pan lid5-912af158ada88380", "pan lid4-9bbade45672fe4e7", "pan lid3-142f890ea752a459", "pan lid2-7a76675708c86623", "pan lid-55a6df77c6737a16", "pan-98bd4c0c7ef13902", "saucepan-fd9b979eb149cf10", "wooden spoon-7be668b519476e5e", "onion piece-aacc24f8849a666f", "half onion-3e17212325230128", "knife2-fa84f037a2db6757", "one onion-eefb212c97a4e74c", "scissors guard cover-0880d4396cb17028", "scissors-71ca5a5e668d4493", "another garlic clove-8c188aee4bae3cb3", "one garlic clove-5dccb4aa10df781a", "garlic pulp-d88829ce5c941361", "mesh of onions-8bddee00e3a058e9", "can opener-7d6f7980f515c26b", "chopping board2-1dddb247ed4076ae", "chopping board-b43c436faa839b48", "chicken stock cube-c7f49be3dc0b05d7", "packaging of meat-b96d7221c1da4827", "chicken stock cube packaging-5438b04a72c67012", "box of chicken stock cubes-632ae16822563d5c", "knife-9e007c9ad3a4ae17", "garlic piece3-188fe3e716ae5e8c", "garlic piece2-fe0794c82a598c90", "garlic piece1-1681159d8efe0802", "bottle of vegetable oil-8f37f20c9984b5c5", "large bowl2-4ccf0fdff4b2e903", "chicken piece5-2e242ddd02c4ac85", "chicken piece4-800aba2339a15b1d", "chicken piece3-03abf52571bb29be", "chicken piece2-0d7345a28bf70924", "chicken piece1-cd350488e54fba12", "cling film-fb8d988440446d77", "baking tray-783309f74e7cd029", "tupperware-11d17e118b8d1a46", "water filter jug-222231917b2f66da", "large bowl-f0bc9ffbb1831abe", "washing up liquid bottle-011cf12411c66663", "sponge-de576ed0ce7ac77e", "left glove-fb2d6fb4138d2350", "right glove-ec4db5c45409608d", "food processor lid-5c9b90c05feab9b2", "strainer-32b82b308bd95a1e" ], "reasoning": "1. I look for the time the user touches the second knob of the oven .\n2. At 00:06, the user presses the knob with their middle finger.\n3. I look for the time just before the user cuts the first chicken stock cube (chicken stock cube-c7f49be3dc0b05d7), and find it at 05:36.\n4. I notice that, just before this, the user holds the knife between their pointer and middle finger.\n5. Since the middle finger is used in both of these situations, the answer is the middle finger.", "referenced_object_masks": [ "chicken stock cube-c7f49be3dc0b05d7" ] }, { "key": "P01-20240204-142301:7860d9466051ddb4872c962359db6f0b211694df", "video_id": "P01-20240204-142301", "question": "If twice the amount of time had passed between me emptying the rice from the white bowl and putting it beside the stove, approximately how much time would have passed?", "answer_choice_0": "30 seconds", "answer_choice_1": "50 seconds", "answer_choice_2": "40 seconds", "answer_choice_3": "10 seconds", "answer_choice_4": "20 seconds", "answer_id": 1, "answer": "50 seconds", "question_type": [ "Object Recognition", "Action Recognition", "Egocentric & Spatial Perception", "State Changes", "Situational Awareness", "Temporal Reasoning", "Numerical Reasoning", "Counterfactual" ], "objects_of_interest": [ "food waste bin-2ec33e268849da1c", "pot lid-f632e97c78eb5302", "juicer central part-0ef9446b6b32000e", "pint glass-82a00cd50f5e4611", "orange4 half-4fff2a83e7f88844", "orange3 half-a8423a6289658c5f", "orange mesh-3aa1e0fffbe6c2df", "orange2 half-a70502769b2a278d", "orange1 half-1b3a7815da8b768b", "food processor plug-47b5d020e71c6f14", "wrong plug-bd4a484bf4e5c893", "knife3-95697c2a2bbd5ddb", "chicken stock cube2-46dcbf252def833e", "teaspoon-89dcd301e47966d8", "chicken stock cube packaging2-8b7aecbb71521156", "mug-86debf886202916a", "plastic bag-8a541364d1e004fa", "kettle-37cc90555dd7bf83", "flask-ef4405e57730661c", "garlic presser-10dd11fd7a42a745", "pot-4f03adbe4f0f4713", "orange4-c365a9eef4feec35", "orange3-71cb26d5ab3aa6e3", "orange2-a732115852b6d410", "orange1-f7f6f1e812a63898", "wooden spatula-9592527836440d66", "scale-dc8c680d89d8b622", "juicer part-d49ba76878cb8dcc", "food processor bowl-bcb3dc03f39509ca", "heat diffuser-59aea36aad36ece6", "pan lid6-8671d1eaa61cebd2", "pan lid5-912af158ada88380", "pan lid4-9bbade45672fe4e7", "pan lid3-142f890ea752a459", "pan lid2-7a76675708c86623", "pan lid-55a6df77c6737a16", "pan-98bd4c0c7ef13902", "saucepan-fd9b979eb149cf10", "wooden spoon-7be668b519476e5e", "onion piece-aacc24f8849a666f", "half onion-3e17212325230128", "knife2-fa84f037a2db6757", "one onion-eefb212c97a4e74c", "scissors guard cover-0880d4396cb17028", "scissors-71ca5a5e668d4493", "another garlic clove-8c188aee4bae3cb3", "one garlic clove-5dccb4aa10df781a", "garlic pulp-d88829ce5c941361", "mesh of onions-8bddee00e3a058e9", "can opener-7d6f7980f515c26b", "chopping board2-1dddb247ed4076ae", "chopping board-b43c436faa839b48", "chicken stock cube-c7f49be3dc0b05d7", "packaging of meat-b96d7221c1da4827", "chicken stock cube packaging-5438b04a72c67012", "box of chicken stock cubes-632ae16822563d5c", "knife-9e007c9ad3a4ae17", "garlic piece3-188fe3e716ae5e8c", "garlic piece2-fe0794c82a598c90", "garlic piece1-1681159d8efe0802", "bottle of vegetable oil-8f37f20c9984b5c5", "large bowl2-4ccf0fdff4b2e903", "chicken piece5-2e242ddd02c4ac85", "chicken piece4-800aba2339a15b1d", "chicken piece3-03abf52571bb29be", "chicken piece2-0d7345a28bf70924", "chicken piece1-cd350488e54fba12", "cling film-fb8d988440446d77", "baking tray-783309f74e7cd029", "tupperware-11d17e118b8d1a46", "water filter jug-222231917b2f66da", "large bowl-f0bc9ffbb1831abe", "washing up liquid bottle-011cf12411c66663", "sponge-de576ed0ce7ac77e", "left glove-fb2d6fb4138d2350", "right glove-ec4db5c45409608d", "food processor lid-5c9b90c05feab9b2", "strainer-32b82b308bd95a1e" ], "reasoning": "1. I watched the video and found the white bowl (large bowl-f0bc9ffbb1831abe) at 00:36.\n2. I saw the user empty the rice from the white bowl (large bowl-f0bc9ffbb1831abe) and into the strainer (strainer-32b82b308bd95a1e) at 02:44.\n3. I saw the user put the strainer (strainer-32b82b308bd95a1e) of rice inside the white bowl (large bowl-f0bc9ffbb1831abe) at 03:05.\n4. I saw the user put the white bowl (large bowl-f0bc9ffbb1831abe) with the strainer (strainer-32b82b308bd95a1e) of rice beside the stove at 03:08.\n5. I calculated 03:08-02:44=24.\n6. I calculated 24*2=48, which approximates to 50.\n7. Therefore, if twice the amount of time had passed between the user emptying the rice from the white bowl (large bowl-f0bc9ffbb1831abe) and putting it beside the stove, approximately 50 seconds would have passed.", "referenced_object_masks": [ "large bowl-f0bc9ffbb1831abe", "strainer-32b82b308bd95a1e" ] }, { "key": "P01-20240204-142301:8a5a98615fc5f87620bc5acb5c42bfc155f7b4c6", "video_id": "P01-20240204-142301", "question": "What direction would I have turned to look at the stove if it were in the opposite direction while I was at the sink?", "answer_choice_0": "45 degrees upward", "answer_choice_1": "90 degrees to the left", "answer_choice_2": "180 degrees behind", "answer_choice_3": "90 degrees to the right", "answer_choice_4": "45 degrees downward", "answer_id": 3, "answer": "90 degrees to the right", "question_type": [ "Situational Awareness", "Action Recognition", "Egocentric & Spatial Perception", "Temporal Reasoning", "Counterfactual" ], "objects_of_interest": [ "food waste bin-2ec33e268849da1c", "pot lid-f632e97c78eb5302", "juicer central part-0ef9446b6b32000e", "pint glass-82a00cd50f5e4611", "orange4 half-4fff2a83e7f88844", "orange3 half-a8423a6289658c5f", "orange mesh-3aa1e0fffbe6c2df", "orange2 half-a70502769b2a278d", "orange1 half-1b3a7815da8b768b", "food processor plug-47b5d020e71c6f14", "wrong plug-bd4a484bf4e5c893", "knife3-95697c2a2bbd5ddb", "chicken stock cube2-46dcbf252def833e", "teaspoon-89dcd301e47966d8", "chicken stock cube packaging2-8b7aecbb71521156", "mug-86debf886202916a", "plastic bag-8a541364d1e004fa", "kettle-37cc90555dd7bf83", "flask-ef4405e57730661c", "garlic presser-10dd11fd7a42a745", "pot-4f03adbe4f0f4713", "orange4-c365a9eef4feec35", "orange3-71cb26d5ab3aa6e3", "orange2-a732115852b6d410", "orange1-f7f6f1e812a63898", "wooden spatula-9592527836440d66", "scale-dc8c680d89d8b622", "juicer part-d49ba76878cb8dcc", "food processor bowl-bcb3dc03f39509ca", "heat diffuser-59aea36aad36ece6", "pan lid6-8671d1eaa61cebd2", "pan lid5-912af158ada88380", "pan lid4-9bbade45672fe4e7", "pan lid3-142f890ea752a459", "pan lid2-7a76675708c86623", "pan lid-55a6df77c6737a16", "pan-98bd4c0c7ef13902", "saucepan-fd9b979eb149cf10", "wooden spoon-7be668b519476e5e", "onion piece-aacc24f8849a666f", "half onion-3e17212325230128", "knife2-fa84f037a2db6757", "one onion-eefb212c97a4e74c", "scissors guard cover-0880d4396cb17028", "scissors-71ca5a5e668d4493", "another garlic clove-8c188aee4bae3cb3", "one garlic clove-5dccb4aa10df781a", "garlic pulp-d88829ce5c941361", "mesh of onions-8bddee00e3a058e9", "can opener-7d6f7980f515c26b", "chopping board2-1dddb247ed4076ae", "chopping board-b43c436faa839b48", "chicken stock cube-c7f49be3dc0b05d7", "packaging of meat-b96d7221c1da4827", "chicken stock cube packaging-5438b04a72c67012", "box of chicken stock cubes-632ae16822563d5c", "knife-9e007c9ad3a4ae17", "garlic piece3-188fe3e716ae5e8c", "garlic piece2-fe0794c82a598c90", "garlic piece1-1681159d8efe0802", "bottle of vegetable oil-8f37f20c9984b5c5", "large bowl2-4ccf0fdff4b2e903", "chicken piece5-2e242ddd02c4ac85", "chicken piece4-800aba2339a15b1d", "chicken piece3-03abf52571bb29be", "chicken piece2-0d7345a28bf70924", "chicken piece1-cd350488e54fba12", "cling film-fb8d988440446d77", "baking tray-783309f74e7cd029", "tupperware-11d17e118b8d1a46", "water filter jug-222231917b2f66da", "large bowl-f0bc9ffbb1831abe", "washing up liquid bottle-011cf12411c66663", "sponge-de576ed0ce7ac77e", "left glove-fb2d6fb4138d2350", "right glove-ec4db5c45409608d", "food processor lid-5c9b90c05feab9b2", "strainer-32b82b308bd95a1e" ], "reasoning": "1. I watched the video and found the sink at 00:13.\n2. At 03:08, I saw the stove when the user turned 90 degrees to the left to look from the sink and at the stove while putting the white bowl (large bowl-f0bc9ffbb1831abe) with the strainer (strainer-32b82b308bd95a1e) of rice beside the stove.\n3. The opposite of \"90 degrees to the left\" is \"90 degrees to the right.\"\n4. Therefore, the user would have turned 90 degrees to the right to look at the stove if it were in the opposite direction while they were at the sink.", "referenced_object_masks": [ "large bowl-f0bc9ffbb1831abe", "strainer-32b82b308bd95a1e" ] }, { "key": "P01-20240204-142301:c444a838e362eda0f4e68c2589de467e854826aa", "video_id": "P01-20240204-142301", "question": "How did I cut the onion after slicing it in half and before dicing it?", "answer_choice_0": "Horizontally", "answer_choice_1": "Randomly", "answer_choice_2": "Vertically", "answer_choice_3": "Diagonally", "answer_choice_4": "Radially", "answer_id": 2, "answer": "Vertically", "question_type": [ "Object Recognition", "Action Recognition", "State Changes", "Temporal Reasoning", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "food waste bin-2ec33e268849da1c", "pot lid-f632e97c78eb5302", "juicer central part-0ef9446b6b32000e", "pint glass-82a00cd50f5e4611", "orange4 half-4fff2a83e7f88844", "orange3 half-a8423a6289658c5f", "orange mesh-3aa1e0fffbe6c2df", "orange2 half-a70502769b2a278d", "orange1 half-1b3a7815da8b768b", "food processor plug-47b5d020e71c6f14", "wrong plug-bd4a484bf4e5c893", "knife3-95697c2a2bbd5ddb", "chicken stock cube2-46dcbf252def833e", "teaspoon-89dcd301e47966d8", "chicken stock cube packaging2-8b7aecbb71521156", "mug-86debf886202916a", "plastic bag-8a541364d1e004fa", "kettle-37cc90555dd7bf83", "flask-ef4405e57730661c", "garlic presser-10dd11fd7a42a745", "pot-4f03adbe4f0f4713", "orange4-c365a9eef4feec35", "orange3-71cb26d5ab3aa6e3", "orange2-a732115852b6d410", "orange1-f7f6f1e812a63898", "wooden spatula-9592527836440d66", "scale-dc8c680d89d8b622", "juicer part-d49ba76878cb8dcc", "food processor bowl-bcb3dc03f39509ca", "heat diffuser-59aea36aad36ece6", "pan lid6-8671d1eaa61cebd2", "pan lid5-912af158ada88380", "pan lid4-9bbade45672fe4e7", "pan lid3-142f890ea752a459", "pan lid2-7a76675708c86623", "pan lid-55a6df77c6737a16", "pan-98bd4c0c7ef13902", "saucepan-fd9b979eb149cf10", "wooden spoon-7be668b519476e5e", "onion piece-aacc24f8849a666f", "half onion-3e17212325230128", "knife2-fa84f037a2db6757", "one onion-eefb212c97a4e74c", "scissors guard cover-0880d4396cb17028", "scissors-71ca5a5e668d4493", "another garlic clove-8c188aee4bae3cb3", "one garlic clove-5dccb4aa10df781a", "garlic pulp-d88829ce5c941361", "mesh of onions-8bddee00e3a058e9", "can opener-7d6f7980f515c26b", "chopping board2-1dddb247ed4076ae", "chopping board-b43c436faa839b48", "chicken stock cube-c7f49be3dc0b05d7", "packaging of meat-b96d7221c1da4827", "chicken stock cube packaging-5438b04a72c67012", "box of chicken stock cubes-632ae16822563d5c", "knife-9e007c9ad3a4ae17", "garlic piece3-188fe3e716ae5e8c", "garlic piece2-fe0794c82a598c90", "garlic piece1-1681159d8efe0802", "bottle of vegetable oil-8f37f20c9984b5c5", "large bowl2-4ccf0fdff4b2e903", "chicken piece5-2e242ddd02c4ac85", "chicken piece4-800aba2339a15b1d", "chicken piece3-03abf52571bb29be", "chicken piece2-0d7345a28bf70924", "chicken piece1-cd350488e54fba12", "cling film-fb8d988440446d77", "baking tray-783309f74e7cd029", "tupperware-11d17e118b8d1a46", "water filter jug-222231917b2f66da", "large bowl-f0bc9ffbb1831abe", "washing up liquid bottle-011cf12411c66663", "sponge-de576ed0ce7ac77e", "left glove-fb2d6fb4138d2350", "right glove-ec4db5c45409608d", "food processor lid-5c9b90c05feab9b2", "strainer-32b82b308bd95a1e" ], "reasoning": "1. I watched the video and found the onion (one onion-eefb212c97a4e74c) at 07:06.\n2. I saw the user cut the onion (one onion-eefb212c97a4e74c) in half with a knife (knife2-fa84f037a2db6757) at 07:40, resulting in two onion halves (half onion-3e17212325230128).\n3. I saw the user dice an onion half (half onion-3e17212325230128) at 10:00.\n4. After 07:40 and before 10:00, I saw the user cut the onion (half onion-3e17212325230128) vertically at 09:28-09:58.\n5. Therefore, the user cut the onion (one onion-eefb212c97a4e74c) vertically after slicing it in half and before dicing it.", "referenced_object_masks": [ "knife2-fa84f037a2db6757", "half onion-3e17212325230128", "one onion-eefb212c97a4e74c" ] }, { "key": "P01-20240204-142301:d70582684e45c0d113184042cb2a7c3fc3ab9912", "video_id": "P01-20240204-142301", "question": "Where did I put the second item I held and ran under the faucet after I poured water out from it?", "answer_choice_0": "Into the dishwasher", "answer_choice_1": "Onto a counter", "answer_choice_2": "Into the refrigerator", "answer_choice_3": "Onto the stove", "answer_choice_4": "Into the sink", "answer_id": 4, "answer": "Into the sink", "question_type": [ "Counting", "Action Recognition", "Situational Awareness", "Object Recognition", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "food waste bin-2ec33e268849da1c", "pot lid-f632e97c78eb5302", "juicer central part-0ef9446b6b32000e", "pint glass-82a00cd50f5e4611", "orange4 half-4fff2a83e7f88844", "orange3 half-a8423a6289658c5f", "orange mesh-3aa1e0fffbe6c2df", "orange2 half-a70502769b2a278d", "orange1 half-1b3a7815da8b768b", "food processor plug-47b5d020e71c6f14", "wrong plug-bd4a484bf4e5c893", "knife3-95697c2a2bbd5ddb", "chicken stock cube2-46dcbf252def833e", "teaspoon-89dcd301e47966d8", "chicken stock cube packaging2-8b7aecbb71521156", "mug-86debf886202916a", "plastic bag-8a541364d1e004fa", "kettle-37cc90555dd7bf83", "flask-ef4405e57730661c", "garlic presser-10dd11fd7a42a745", "pot-4f03adbe4f0f4713", "orange4-c365a9eef4feec35", "orange3-71cb26d5ab3aa6e3", "orange2-a732115852b6d410", "orange1-f7f6f1e812a63898", "wooden spatula-9592527836440d66", "scale-dc8c680d89d8b622", "juicer part-d49ba76878cb8dcc", "food processor bowl-bcb3dc03f39509ca", "heat diffuser-59aea36aad36ece6", "pan lid6-8671d1eaa61cebd2", "pan lid5-912af158ada88380", "pan lid4-9bbade45672fe4e7", "pan lid3-142f890ea752a459", "pan lid2-7a76675708c86623", "pan lid-55a6df77c6737a16", "pan-98bd4c0c7ef13902", "saucepan-fd9b979eb149cf10", "wooden spoon-7be668b519476e5e", "onion piece-aacc24f8849a666f", "half onion-3e17212325230128", "knife2-fa84f037a2db6757", "one onion-eefb212c97a4e74c", "scissors guard cover-0880d4396cb17028", "scissors-71ca5a5e668d4493", "another garlic clove-8c188aee4bae3cb3", "one garlic clove-5dccb4aa10df781a", "garlic pulp-d88829ce5c941361", "mesh of onions-8bddee00e3a058e9", "can opener-7d6f7980f515c26b", "chopping board2-1dddb247ed4076ae", "chopping board-b43c436faa839b48", "chicken stock cube-c7f49be3dc0b05d7", "packaging of meat-b96d7221c1da4827", "chicken stock cube packaging-5438b04a72c67012", "box of chicken stock cubes-632ae16822563d5c", "knife-9e007c9ad3a4ae17", "garlic piece3-188fe3e716ae5e8c", "garlic piece2-fe0794c82a598c90", "garlic piece1-1681159d8efe0802", "bottle of vegetable oil-8f37f20c9984b5c5", "large bowl2-4ccf0fdff4b2e903", "chicken piece5-2e242ddd02c4ac85", "chicken piece4-800aba2339a15b1d", "chicken piece3-03abf52571bb29be", "chicken piece2-0d7345a28bf70924", "chicken piece1-cd350488e54fba12", "cling film-fb8d988440446d77", "baking tray-783309f74e7cd029", "tupperware-11d17e118b8d1a46", "water filter jug-222231917b2f66da", "large bowl-f0bc9ffbb1831abe", "washing up liquid bottle-011cf12411c66663", "sponge-de576ed0ce7ac77e", "left glove-fb2d6fb4138d2350", "right glove-ec4db5c45409608d", "food processor lid-5c9b90c05feab9b2", "strainer-32b82b308bd95a1e" ], "reasoning": "1. I watched the video and counted each item the user held and ran under the faucet in the sink.\n2. At 00:44, I saw the user hold and run the first item under the faucet in the sink, which was a white bowl (large bowl-f0bc9ffbb1831abe).\n3. At 01:14, I saw the user hold and run the second item under the faucet in the sink, which was a piece of plastic tupperware (tupperware-11d17e118b8d1a46).\n4. At 01:20, the user pours water out from the plastic tupperware (tupperware-11d17e118b8d1a46).\n5. At 01:30, I saw the user put the plastic tupperware (tupperware-11d17e118b8d1a46) back into the sink.\n6. Therefore, the user put the second item they held and ran under the faucet back into the sink.", "referenced_object_masks": [ "large bowl-f0bc9ffbb1831abe", "tupperware-11d17e118b8d1a46" ] }, { "key": "P01-20240204-142301:e5ee57d3c746ff47f9c266c1647ad4bd9699aa6a", "video_id": "P01-20240204-142301", "question": "Which color in the following list does not match any of the ingredients I prepared or used in the video?", "answer_choice_0": "Yellow", "answer_choice_1": "Orange", "answer_choice_2": "Black", "answer_choice_3": "Red", "answer_choice_4": "White", "answer_id": 2, "answer": "Black", "question_type": [ "Object Recognition", "Event Occurence" ], "objects_of_interest": [ "food waste bin-2ec33e268849da1c", "pot lid-f632e97c78eb5302", "juicer central part-0ef9446b6b32000e", "pint glass-82a00cd50f5e4611", "orange4 half-4fff2a83e7f88844", "orange3 half-a8423a6289658c5f", "orange mesh-3aa1e0fffbe6c2df", "orange2 half-a70502769b2a278d", "orange1 half-1b3a7815da8b768b", "food processor plug-47b5d020e71c6f14", "wrong plug-bd4a484bf4e5c893", "knife3-95697c2a2bbd5ddb", "chicken stock cube2-46dcbf252def833e", "teaspoon-89dcd301e47966d8", "chicken stock cube packaging2-8b7aecbb71521156", "mug-86debf886202916a", "plastic bag-8a541364d1e004fa", "kettle-37cc90555dd7bf83", "flask-ef4405e57730661c", "garlic presser-10dd11fd7a42a745", "pot-4f03adbe4f0f4713", "orange4-c365a9eef4feec35", "orange3-71cb26d5ab3aa6e3", "orange2-a732115852b6d410", "orange1-f7f6f1e812a63898", "wooden spatula-9592527836440d66", "scale-dc8c680d89d8b622", "juicer part-d49ba76878cb8dcc", "food processor bowl-bcb3dc03f39509ca", "heat diffuser-59aea36aad36ece6", "pan lid6-8671d1eaa61cebd2", "pan lid5-912af158ada88380", "pan lid4-9bbade45672fe4e7", "pan lid3-142f890ea752a459", "pan lid2-7a76675708c86623", "pan lid-55a6df77c6737a16", "pan-98bd4c0c7ef13902", "saucepan-fd9b979eb149cf10", "wooden spoon-7be668b519476e5e", "onion piece-aacc24f8849a666f", "half onion-3e17212325230128", "knife2-fa84f037a2db6757", "one onion-eefb212c97a4e74c", "scissors guard cover-0880d4396cb17028", "scissors-71ca5a5e668d4493", "another garlic clove-8c188aee4bae3cb3", "one garlic clove-5dccb4aa10df781a", "garlic pulp-d88829ce5c941361", "mesh of onions-8bddee00e3a058e9", "can opener-7d6f7980f515c26b", "chopping board2-1dddb247ed4076ae", "chopping board-b43c436faa839b48", "chicken stock cube-c7f49be3dc0b05d7", "packaging of meat-b96d7221c1da4827", "chicken stock cube packaging-5438b04a72c67012", "box of chicken stock cubes-632ae16822563d5c", "knife-9e007c9ad3a4ae17", "garlic piece3-188fe3e716ae5e8c", "garlic piece2-fe0794c82a598c90", "garlic piece1-1681159d8efe0802", "bottle of vegetable oil-8f37f20c9984b5c5", "large bowl2-4ccf0fdff4b2e903", "chicken piece5-2e242ddd02c4ac85", "chicken piece4-800aba2339a15b1d", "chicken piece3-03abf52571bb29be", "chicken piece2-0d7345a28bf70924", "chicken piece1-cd350488e54fba12", "cling film-fb8d988440446d77", "baking tray-783309f74e7cd029", "tupperware-11d17e118b8d1a46", "water filter jug-222231917b2f66da", "large bowl-f0bc9ffbb1831abe", "washing up liquid bottle-011cf12411c66663", "sponge-de576ed0ce7ac77e", "left glove-fb2d6fb4138d2350", "right glove-ec4db5c45409608d", "food processor lid-5c9b90c05feab9b2", "strainer-32b82b308bd95a1e" ], "reasoning": "1. I look for used ingredients that do not match the colors presented in the list.\n2. At 01:08, the user handles white rice .\n3. From 05:35-06:18, the user slices a yellow chicken stock cube (chicken stock cube-c7f49be3dc0b05d7).\n4. From 12:55-13:06 the user slices some red ground beef (packaging of meat-b96d7221c1da4827).\n5. From 24:06-24:08, the user slices an orange orange (orange1-f7f6f1e812a63898).\n6. The user doesn't make use of any black ingredients throughout the video.\n7. Therefore, black is the answer.", "referenced_object_masks": [ "packaging of meat-b96d7221c1da4827", "orange1-f7f6f1e812a63898", "chicken stock cube-c7f49be3dc0b05d7" ] }, { "key": "P01-20240204-142301:f36972b5eea6e6710b9b847b3dad6189b13b03f3", "video_id": "P01-20240204-142301", "question": "For what percentage of time, rounded to the nearest whole number, while I am in the midst of looking for a pan lid, do I briefly gaze out the window?", "answer_choice_0": "10%", "answer_choice_1": "I do not look out the window", "answer_choice_2": "3%", "answer_choice_3": "60%", "answer_choice_4": "1%", "answer_id": 2, "answer": "3%", "question_type": [ "Egocentric & Spatial Perception", "Temporal Reasoning", "Action Recognition", "Numerical Reasoning" ], "objects_of_interest": [ "food waste bin-2ec33e268849da1c", "pot lid-f632e97c78eb5302", "juicer central part-0ef9446b6b32000e", "pint glass-82a00cd50f5e4611", "orange4 half-4fff2a83e7f88844", "orange3 half-a8423a6289658c5f", "orange mesh-3aa1e0fffbe6c2df", "orange2 half-a70502769b2a278d", "orange1 half-1b3a7815da8b768b", "food processor plug-47b5d020e71c6f14", "wrong plug-bd4a484bf4e5c893", "knife3-95697c2a2bbd5ddb", "chicken stock cube2-46dcbf252def833e", "teaspoon-89dcd301e47966d8", "chicken stock cube packaging2-8b7aecbb71521156", "mug-86debf886202916a", "plastic bag-8a541364d1e004fa", "kettle-37cc90555dd7bf83", "flask-ef4405e57730661c", "garlic presser-10dd11fd7a42a745", "pot-4f03adbe4f0f4713", "orange4-c365a9eef4feec35", "orange3-71cb26d5ab3aa6e3", "orange2-a732115852b6d410", "orange1-f7f6f1e812a63898", "wooden spatula-9592527836440d66", "scale-dc8c680d89d8b622", "juicer part-d49ba76878cb8dcc", "food processor bowl-bcb3dc03f39509ca", "heat diffuser-59aea36aad36ece6", "pan lid6-8671d1eaa61cebd2", "pan lid5-912af158ada88380", "pan lid4-9bbade45672fe4e7", "pan lid3-142f890ea752a459", "pan lid2-7a76675708c86623", "pan lid-55a6df77c6737a16", "pan-98bd4c0c7ef13902", "saucepan-fd9b979eb149cf10", "wooden spoon-7be668b519476e5e", "onion piece-aacc24f8849a666f", "half onion-3e17212325230128", "knife2-fa84f037a2db6757", "one onion-eefb212c97a4e74c", "scissors guard cover-0880d4396cb17028", "scissors-71ca5a5e668d4493", "another garlic clove-8c188aee4bae3cb3", "one garlic clove-5dccb4aa10df781a", "garlic pulp-d88829ce5c941361", "mesh of onions-8bddee00e3a058e9", "can opener-7d6f7980f515c26b", "chopping board2-1dddb247ed4076ae", "chopping board-b43c436faa839b48", "chicken stock cube-c7f49be3dc0b05d7", "packaging of meat-b96d7221c1da4827", "chicken stock cube packaging-5438b04a72c67012", "box of chicken stock cubes-632ae16822563d5c", "knife-9e007c9ad3a4ae17", "garlic piece3-188fe3e716ae5e8c", "garlic piece2-fe0794c82a598c90", "garlic piece1-1681159d8efe0802", "bottle of vegetable oil-8f37f20c9984b5c5", "large bowl2-4ccf0fdff4b2e903", "chicken piece5-2e242ddd02c4ac85", "chicken piece4-800aba2339a15b1d", "chicken piece3-03abf52571bb29be", "chicken piece2-0d7345a28bf70924", "chicken piece1-cd350488e54fba12", "cling film-fb8d988440446d77", "baking tray-783309f74e7cd029", "tupperware-11d17e118b8d1a46", "water filter jug-222231917b2f66da", "large bowl-f0bc9ffbb1831abe", "washing up liquid bottle-011cf12411c66663", "sponge-de576ed0ce7ac77e", "left glove-fb2d6fb4138d2350", "right glove-ec4db5c45409608d", "food processor lid-5c9b90c05feab9b2", "strainer-32b82b308bd95a1e" ], "reasoning": "1. I look for the time the user is looking for a pan lid (pan lid6-8671d1eaa61cebd2), and I find the segment from 11:38-12:15.\n2. The time spent looking comes to a total of 37 seconds.\n3. I look for a moment where the user stops to look out the window, and I find it from 12:00-12:01.\n4. The time the user spent looking out the window comes to a total of 1 second.\n5. I calculate 1/37, convert into a percentage, and get 2.7%.\n6. Rounding up, therefore, the answer is 3%.", "referenced_object_masks": [ "pan lid6-8671d1eaa61cebd2" ] }, { "key": "P01-20240204-145458:08c495e5b1750c41c742eb6d83a4236057db0952", "video_id": "P01-20240204-145458", "question": "If I took 2 steps backwards and turned left 90 degrees from where I drank juice for the third time, what would be in front of me?", "answer_choice_0": "The sink", "answer_choice_1": "The refrigerator", "answer_choice_2": "The dishwasher", "answer_choice_3": "The window", "answer_choice_4": "The garbage can", "answer_id": 1, "answer": "The refrigerator", "question_type": [ "Counting", "Action Recognition", "Egocentric & Spatial Perception", "Object Recognition", "Counterfactual" ], "objects_of_interest": [ "plastic piece round from juicer-afc4616e3bb9ebbd", "lid of food processor-61e7d409c57f74e6", "plastic lid-4b1bc1eed311477d", "third glass-e69d89c461ecb12a", "another glass-202aeb4488d6fd08", "glass-1e14b7e2aacc0554", "two parts of blade-7b371e22fa33bffe", "plastic piece-b4cbd19cffa9f5b6", "glove-1088ee85095ad2b0", "pot-628939386fd7ac72", "empty tupperware-cc7332248940d9cd", "brush-b9e00831d20cdb08", "blade of knife from sink-aa9aaad30e336a6d", "knife from sink-83bd1e72ec233647", "brown plate in dishwasher-3fbc776aa47f4db6", "plate from the sink-227fb374c212530c", "third bowl from the sink-1846c34c89453d95", "second bowl from sink-07424256b4ca0344", "bowl from sink-c5a5203e8911c8c6", "another white bowl-c150ba35717a3622", "another chopping board-f3d26d9a73db3988", "knife-9a354016f1d55a0d", "sponge-49fbc5073665000c", "white bowl-cd9ee625800e3e08", "tray-32bfe3d0b7019d90", "lid in dishwasher-9da3ea69065b1fea", "thermal flask lid-710454a7145847bf", "frying pan-16a00a5ac557ee9d", "thermal flask-383e93abe1fded86", "chopping board-0b3afccd4a17df2a", "empty glass-f9dcaa9af3835888", "scale-d41d55f1740cd4ce", "notepad-6fbf559cda7fc95a", "pen-872591fb42387dd7", "center piece-30966619ffe37d42", "centerpiece-77b907cc12d8e928", "center plastic piece-1005a622c53a895b", "food processor plug-e1b933f854a32061", "second spice jar-d95e5db2026b8936", "spice jar lid-3def4a95fe316d7f", "milk frother-ff653eee5f5cdd71", "spice jar-a113d8d9fad594f7", "salt bottle-f27fd204cbb4b41c", "kettle-bffd0bd346550cd1", "chicken stock cube-3abcb27b403a6f0a", "box of chicken cube stock-2e8ab5f46dc046c6", "wooden spoon 2-4f98a808afbbea60", "small spoon-1ec7f68ebbddbddb", "designer bowl-371a1b7f0a798584", "strainer-9ee450a932677732", "mug-36bececaeb54d3d7", "plastic packaging-48e351ac4da6813c", "plastic bag-14489cf41e6cb931", "bottle of vegetable oil-0539dc57c39570b0", "one piece of kitchen towel-f72f93cb09996e6f", "roll of kitchen towels-5b75526039e13cd5", "white saucepan lid-4a3120f382200b1c", "small black saucepan-2a3202d4e21f2a18", "whitw saucepan-89cb36901c54794b", "saucepan-13c1292ab79727c0", "wooden spoon-31d79ef33080751f", "lid-6b782523989b0081", "glass of orange juice-88a7d132cb47cbe0", "food trash bin-4f7fec57ba7c5c4c", "new garlic presser-c508fc81f27ce7c0" ], "reasoning": "1. I watched until I saw the user drink from their cup of juice (glass of orange juice-88a7d132cb47cbe0) for the third time.\n2. I saw the user drink from a glass (glass of orange juice-88a7d132cb47cbe0) first at 00:10, then at 01:31, and at 03:29 for the third time.\n3. Just before the user drank for the third time, I could see that they turned in place 180 degrees from directly in front of the stove to pick up the glass (glass of orange juice-88a7d132cb47cbe0) from the opposite counter.\n4. The user was facing towards the sink when they drank at 03:29. If they stepped backwards, they would get closer to the refrigerator and garbage can .\n5. If the user turned left 90 degrees, they would be facing the refrigerator.", "referenced_object_masks": [ "glass of orange juice-88a7d132cb47cbe0" ] }, { "key": "P01-20240204-145458:103a13cc867f95c460cdd944c68f6f46e527ee6e", "video_id": "P01-20240204-145458", "question": "After I add in water to the saucepan for the final time and close it with a lid, what is the 3rd item that I pick up after I lift the lid from the saucepan for the 3rd time?", "answer_choice_0": "A wooden spoon", "answer_choice_1": "A knife", "answer_choice_2": "A cutting board", "answer_choice_3": "A thermal flask", "answer_choice_4": "A frying pan", "answer_id": 3, "answer": "A thermal flask", "question_type": [ "Action Recognition", "Counting", "Object Recognition" ], "objects_of_interest": [ "plastic piece round from juicer-afc4616e3bb9ebbd", "lid of food processor-61e7d409c57f74e6", "plastic lid-4b1bc1eed311477d", "third glass-e69d89c461ecb12a", "another glass-202aeb4488d6fd08", "glass-1e14b7e2aacc0554", "two parts of blade-7b371e22fa33bffe", "plastic piece-b4cbd19cffa9f5b6", "glove-1088ee85095ad2b0", "pot-628939386fd7ac72", "empty tupperware-cc7332248940d9cd", "brush-b9e00831d20cdb08", "blade of knife from sink-aa9aaad30e336a6d", "knife from sink-83bd1e72ec233647", "brown plate in dishwasher-3fbc776aa47f4db6", "plate from the sink-227fb374c212530c", "third bowl from the sink-1846c34c89453d95", "second bowl from sink-07424256b4ca0344", "bowl from sink-c5a5203e8911c8c6", "another white bowl-c150ba35717a3622", "another chopping board-f3d26d9a73db3988", "knife-9a354016f1d55a0d", "sponge-49fbc5073665000c", "white bowl-cd9ee625800e3e08", "tray-32bfe3d0b7019d90", "lid in dishwasher-9da3ea69065b1fea", "thermal flask lid-710454a7145847bf", "frying pan-16a00a5ac557ee9d", "thermal flask-383e93abe1fded86", "chopping board-0b3afccd4a17df2a", "empty glass-f9dcaa9af3835888", "scale-d41d55f1740cd4ce", "notepad-6fbf559cda7fc95a", "pen-872591fb42387dd7", "center piece-30966619ffe37d42", "centerpiece-77b907cc12d8e928", "center plastic piece-1005a622c53a895b", "food processor plug-e1b933f854a32061", "second spice jar-d95e5db2026b8936", "spice jar lid-3def4a95fe316d7f", "milk frother-ff653eee5f5cdd71", "spice jar-a113d8d9fad594f7", "salt bottle-f27fd204cbb4b41c", "kettle-bffd0bd346550cd1", "chicken stock cube-3abcb27b403a6f0a", "box of chicken cube stock-2e8ab5f46dc046c6", "wooden spoon 2-4f98a808afbbea60", "small spoon-1ec7f68ebbddbddb", "designer bowl-371a1b7f0a798584", "strainer-9ee450a932677732", "mug-36bececaeb54d3d7", "plastic packaging-48e351ac4da6813c", "plastic bag-14489cf41e6cb931", "bottle of vegetable oil-0539dc57c39570b0", "one piece of kitchen towel-f72f93cb09996e6f", "roll of kitchen towels-5b75526039e13cd5", "white saucepan lid-4a3120f382200b1c", "small black saucepan-2a3202d4e21f2a18", "whitw saucepan-89cb36901c54794b", "saucepan-13c1292ab79727c0", "wooden spoon-31d79ef33080751f", "lid-6b782523989b0081", "glass of orange juice-88a7d132cb47cbe0", "food trash bin-4f7fec57ba7c5c4c", "new garlic presser-c508fc81f27ce7c0" ], "reasoning": "1. I watched the video and looked for the time that the user added water to the saucepan (whitw saucepan-89cb36901c54794b) the final time. I found this moment at 07:22, when the user added water.\n2. I noticed the user stir from 07:24-07:27, and then at 07:33 the user replaced the lid (white saucepan lid-4a3120f382200b1c) on the saucepan (whitw saucepan-89cb36901c54794b).\n3. From this point, I watched to see when the user lifted the lid (white saucepan lid-4a3120f382200b1c) for the third time. I found three successive moments at 09:23, 10:00, and 10:19. Each time, the user picked up the lid (white saucepan lid-4a3120f382200b1c) very briefly and then immediately replaced it on top of the saucepan (whitw saucepan-89cb36901c54794b).\n4. Next, I looked for the objects that the user touched after 10:19. At 10:25, I noticed the user touched and picked up a cutting board (chopping board-0b3afccd4a17df2a). At the same time, they also touched a knife which was sitting on top of it. I counted these as the 1st and 2nd items to be touched.\n5. The next item the user touched was a thermal flask (thermal flask-383e93abe1fded86. I counted this as the 3rd item and thus, the final answer.", "referenced_object_masks": [ "white saucepan lid-4a3120f382200b1c", "whitw saucepan-89cb36901c54794b", "chopping board-0b3afccd4a17df2a" ] }, { "key": "P01-20240204-145458:6e38b87714be66bcfb4541abb850775c36b392d8", "video_id": "P01-20240204-145458", "question": "What was the 9th item that I moved into the dishwasher after it was sitting in the sink?", "answer_choice_0": "A bowl", "answer_choice_1": "A knife", "answer_choice_2": "A strainer", "answer_choice_3": "A wooden spoon", "answer_choice_4": "A cutting board", "answer_id": 3, "answer": "A wooden spoon", "question_type": [ "Action Recognition", "Counting", "Egocentric & Spatial Perception", "Situational Awareness", "Object Recognition" ], "objects_of_interest": [ "plastic piece round from juicer-afc4616e3bb9ebbd", "lid of food processor-61e7d409c57f74e6", "plastic lid-4b1bc1eed311477d", "third glass-e69d89c461ecb12a", "another glass-202aeb4488d6fd08", "glass-1e14b7e2aacc0554", "two parts of blade-7b371e22fa33bffe", "plastic piece-b4cbd19cffa9f5b6", "glove-1088ee85095ad2b0", "pot-628939386fd7ac72", "empty tupperware-cc7332248940d9cd", "brush-b9e00831d20cdb08", "blade of knife from sink-aa9aaad30e336a6d", "knife from sink-83bd1e72ec233647", "brown plate in dishwasher-3fbc776aa47f4db6", "plate from the sink-227fb374c212530c", "third bowl from the sink-1846c34c89453d95", "second bowl from sink-07424256b4ca0344", "bowl from sink-c5a5203e8911c8c6", "another white bowl-c150ba35717a3622", "another chopping board-f3d26d9a73db3988", "knife-9a354016f1d55a0d", "sponge-49fbc5073665000c", "white bowl-cd9ee625800e3e08", "tray-32bfe3d0b7019d90", "lid in dishwasher-9da3ea69065b1fea", "thermal flask lid-710454a7145847bf", "frying pan-16a00a5ac557ee9d", "thermal flask-383e93abe1fded86", "chopping board-0b3afccd4a17df2a", "empty glass-f9dcaa9af3835888", "scale-d41d55f1740cd4ce", "notepad-6fbf559cda7fc95a", "pen-872591fb42387dd7", "center piece-30966619ffe37d42", "centerpiece-77b907cc12d8e928", "center plastic piece-1005a622c53a895b", "food processor plug-e1b933f854a32061", "second spice jar-d95e5db2026b8936", "spice jar lid-3def4a95fe316d7f", "milk frother-ff653eee5f5cdd71", "spice jar-a113d8d9fad594f7", "salt bottle-f27fd204cbb4b41c", "kettle-bffd0bd346550cd1", "chicken stock cube-3abcb27b403a6f0a", "box of chicken cube stock-2e8ab5f46dc046c6", "wooden spoon 2-4f98a808afbbea60", "small spoon-1ec7f68ebbddbddb", "designer bowl-371a1b7f0a798584", "strainer-9ee450a932677732", "mug-36bececaeb54d3d7", "plastic packaging-48e351ac4da6813c", "plastic bag-14489cf41e6cb931", "bottle of vegetable oil-0539dc57c39570b0", "one piece of kitchen towel-f72f93cb09996e6f", "roll of kitchen towels-5b75526039e13cd5", "white saucepan lid-4a3120f382200b1c", "small black saucepan-2a3202d4e21f2a18", "whitw saucepan-89cb36901c54794b", "saucepan-13c1292ab79727c0", "wooden spoon-31d79ef33080751f", "lid-6b782523989b0081", "glass of orange juice-88a7d132cb47cbe0", "food trash bin-4f7fec57ba7c5c4c", "new garlic presser-c508fc81f27ce7c0" ], "reasoning": "1. I watched the video and looked for the time when the user started moving items into the dishwasher from the sink . I found this at 11:37, when I saw the user place a strainer (strainer-9ee450a932677732) into the dishwasher .\n2. I moved back a bit in the video and saw that the user had grabbed the strainer (strainer-9ee450a932677732) from the counter at 11:27, and it never went into the sink. I continued watching to find the 1st item that fit the criteria for the question.\n3. At 12:09, I saw the user add a knife to the dishwasher which had been sitting in the sink . I counted this as the 1st item.\n4. Counting up to 9 items, I observed the following items move from the sink to the dishwasher : a cutting board (chopping board-0b3afccd4a17df2a) at 12:27, another cutting board (another chopping board-f3d26d9a73db3988) at 12:40, a bowl (another white bowl-c150ba35717a3622) at 12:58, a small bowl (bowl from sink-c5a5203e8911c8c6) at 13:08, another small bowl (second bowl from sink-07424256b4ca0344) at 13:16, a 3rd small bowl (third bowl from the sink-1846c34c89453d95) at 13:20, a plate (plate from the sink-227fb374c212530c) at 13:30, and finally a wooden spoon at 13:45.\n5. Since this wooden spoon was the 9th item to be added to the dishwasher after it was in the sink , I determined that was the answer.", "referenced_object_masks": [ "bowl from sink-c5a5203e8911c8c6", "strainer-9ee450a932677732", "second bowl from sink-07424256b4ca0344", "third bowl from the sink-1846c34c89453d95", "plate from the sink-227fb374c212530c", "chopping board-0b3afccd4a17df2a", "another white bowl-c150ba35717a3622", "another chopping board-f3d26d9a73db3988" ] }, { "key": "P01-20240204-145458:73fd448558e46f83a1a658a64260bb0b0721278f", "video_id": "P01-20240204-145458", "question": "Excluding the compost bin I walked in with, what do the second and last objects I touched with my left hand have in common?", "answer_choice_0": "They're both made of wood", "answer_choice_1": "I washed them both", "answer_choice_2": "They're both glasses", "answer_choice_3": "They both hold spices", "answer_choice_4": "They're both lids", "answer_id": 4, "answer": "They're both lids", "question_type": [ "Object Recognition", "Egocentric & Spatial Perception", "Temporal Reasoning" ], "objects_of_interest": [ "plastic piece round from juicer-afc4616e3bb9ebbd", "lid of food processor-61e7d409c57f74e6", "plastic lid-4b1bc1eed311477d", "third glass-e69d89c461ecb12a", "another glass-202aeb4488d6fd08", "glass-1e14b7e2aacc0554", "two parts of blade-7b371e22fa33bffe", "plastic piece-b4cbd19cffa9f5b6", "glove-1088ee85095ad2b0", "pot-628939386fd7ac72", "empty tupperware-cc7332248940d9cd", "brush-b9e00831d20cdb08", "blade of knife from sink-aa9aaad30e336a6d", "knife from sink-83bd1e72ec233647", "brown plate in dishwasher-3fbc776aa47f4db6", "plate from the sink-227fb374c212530c", "third bowl from the sink-1846c34c89453d95", "second bowl from sink-07424256b4ca0344", "bowl from sink-c5a5203e8911c8c6", "another white bowl-c150ba35717a3622", "another chopping board-f3d26d9a73db3988", "knife-9a354016f1d55a0d", "sponge-49fbc5073665000c", "white bowl-cd9ee625800e3e08", "tray-32bfe3d0b7019d90", "lid in dishwasher-9da3ea69065b1fea", "thermal flask lid-710454a7145847bf", "frying pan-16a00a5ac557ee9d", "thermal flask-383e93abe1fded86", "chopping board-0b3afccd4a17df2a", "empty glass-f9dcaa9af3835888", "scale-d41d55f1740cd4ce", "notepad-6fbf559cda7fc95a", "pen-872591fb42387dd7", "center piece-30966619ffe37d42", "centerpiece-77b907cc12d8e928", "center plastic piece-1005a622c53a895b", "food processor plug-e1b933f854a32061", "second spice jar-d95e5db2026b8936", "spice jar lid-3def4a95fe316d7f", "milk frother-ff653eee5f5cdd71", "spice jar-a113d8d9fad594f7", "salt bottle-f27fd204cbb4b41c", "kettle-bffd0bd346550cd1", "chicken stock cube-3abcb27b403a6f0a", "box of chicken cube stock-2e8ab5f46dc046c6", "wooden spoon 2-4f98a808afbbea60", "small spoon-1ec7f68ebbddbddb", "designer bowl-371a1b7f0a798584", "strainer-9ee450a932677732", "mug-36bececaeb54d3d7", "plastic packaging-48e351ac4da6813c", "plastic bag-14489cf41e6cb931", "bottle of vegetable oil-0539dc57c39570b0", "one piece of kitchen towel-f72f93cb09996e6f", "roll of kitchen towels-5b75526039e13cd5", "white saucepan lid-4a3120f382200b1c", "small black saucepan-2a3202d4e21f2a18", "whitw saucepan-89cb36901c54794b", "saucepan-13c1292ab79727c0", "wooden spoon-31d79ef33080751f", "lid-6b782523989b0081", "glass of orange juice-88a7d132cb47cbe0", "food trash bin-4f7fec57ba7c5c4c", "new garlic presser-c508fc81f27ce7c0" ], "reasoning": "1. I watched until I saw the user touch an object with their left hand for the second time.\n2. The user first touched a glass (glass of orange juice-88a7d132cb47cbe0) with their left hand at 00:07, and then a lid (lid-6b782523989b0081) from a pan (saucepan-13c1292ab79727c0) at 00:20.\n3. I continued watching until I saw the user touch the last item with their left hand, another lid (white saucepan lid-4a3120f382200b1c), at 18:03.\n4. Therefore both items were lids .", "referenced_object_masks": [ "white saucepan lid-4a3120f382200b1c", "lid-6b782523989b0081", "glass of orange juice-88a7d132cb47cbe0", "saucepan-13c1292ab79727c0" ] }, { "key": "P01-20240204-145458:8c721e72b6c1a36ab20dfd3ea3e783ec875c6ea5", "video_id": "P01-20240204-145458", "question": "Rounded to the nearest tenth, approximately how many times do I stir the contents of the saucepan per second right after adding rice to it?", "answer_choice_0": "1.3 stirs per seconds", "answer_choice_1": "2.3 stirs per second", "answer_choice_2": "1.9 stirs per second", "answer_choice_3": "2.8 stirs per second", "answer_choice_4": "0.9 stirs per seconds", "answer_id": 2, "answer": "1.9 stirs per second", "question_type": [ "Action Recognition", "Counting", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "plastic piece round from juicer-afc4616e3bb9ebbd", "lid of food processor-61e7d409c57f74e6", "plastic lid-4b1bc1eed311477d", "third glass-e69d89c461ecb12a", "another glass-202aeb4488d6fd08", "glass-1e14b7e2aacc0554", "two parts of blade-7b371e22fa33bffe", "plastic piece-b4cbd19cffa9f5b6", "glove-1088ee85095ad2b0", "pot-628939386fd7ac72", "empty tupperware-cc7332248940d9cd", "brush-b9e00831d20cdb08", "blade of knife from sink-aa9aaad30e336a6d", "knife from sink-83bd1e72ec233647", "brown plate in dishwasher-3fbc776aa47f4db6", "plate from the sink-227fb374c212530c", "third bowl from the sink-1846c34c89453d95", "second bowl from sink-07424256b4ca0344", "bowl from sink-c5a5203e8911c8c6", "another white bowl-c150ba35717a3622", "another chopping board-f3d26d9a73db3988", "knife-9a354016f1d55a0d", "sponge-49fbc5073665000c", "white bowl-cd9ee625800e3e08", "tray-32bfe3d0b7019d90", "lid in dishwasher-9da3ea69065b1fea", "thermal flask lid-710454a7145847bf", "frying pan-16a00a5ac557ee9d", "thermal flask-383e93abe1fded86", "chopping board-0b3afccd4a17df2a", "empty glass-f9dcaa9af3835888", "scale-d41d55f1740cd4ce", "notepad-6fbf559cda7fc95a", "pen-872591fb42387dd7", "center piece-30966619ffe37d42", "centerpiece-77b907cc12d8e928", "center plastic piece-1005a622c53a895b", "food processor plug-e1b933f854a32061", "second spice jar-d95e5db2026b8936", "spice jar lid-3def4a95fe316d7f", "milk frother-ff653eee5f5cdd71", "spice jar-a113d8d9fad594f7", "salt bottle-f27fd204cbb4b41c", "kettle-bffd0bd346550cd1", "chicken stock cube-3abcb27b403a6f0a", "box of chicken cube stock-2e8ab5f46dc046c6", "wooden spoon 2-4f98a808afbbea60", "small spoon-1ec7f68ebbddbddb", "designer bowl-371a1b7f0a798584", "strainer-9ee450a932677732", "mug-36bececaeb54d3d7", "plastic packaging-48e351ac4da6813c", "plastic bag-14489cf41e6cb931", "bottle of vegetable oil-0539dc57c39570b0", "one piece of kitchen towel-f72f93cb09996e6f", "roll of kitchen towels-5b75526039e13cd5", "white saucepan lid-4a3120f382200b1c", "small black saucepan-2a3202d4e21f2a18", "whitw saucepan-89cb36901c54794b", "saucepan-13c1292ab79727c0", "wooden spoon-31d79ef33080751f", "lid-6b782523989b0081", "glass of orange juice-88a7d132cb47cbe0", "food trash bin-4f7fec57ba7c5c4c", "new garlic presser-c508fc81f27ce7c0" ], "reasoning": "1. I watched the video and saw that the user added rice to the saucepan (whitw saucepan-89cb36901c54794b) from 05:52-05:53.\n2. I saw the user pick up a wooden spoon (wooden spoon 2-4f98a808afbbea60) right afterward at 05:56, and then they started stirring the contents.\n3. I saw the user stir the saucepan (whitw saucepan-89cb36901c54794b) with the wooden spoon (wooden spoon 2-4f98a808afbbea60) from 05:56-06:20, for a total of 24 seconds.\n4. From 05:56-06:20, I noted that the user stirred the saucepan (whitw saucepan-89cb36901c54794b) 45 times.\n5. I divided 45 stirs by 24 seconds (45/24) to reach a total of 1.875, which I rounded up to reach the approximate total of 1.9 stirs per second.", "referenced_object_masks": [ "wooden spoon 2-4f98a808afbbea60", "whitw saucepan-89cb36901c54794b" ] }, { "key": "P01-20240204-145458:b37580b8c3c5fcea0ce4a2887adfb8bdaae19362", "video_id": "P01-20240204-145458", "question": "How long would it have taken me to pick up the food trash bin after I set it down on the counter if I took half as much time to do so?", "answer_choice_0": "9 minutes and 15 seconds", "answer_choice_1": "7 minutes and 23 seconds", "answer_choice_2": "8 minutes and 24 seconds", "answer_choice_3": "6 minutes and 4 seconds", "answer_choice_4": "5 minutes and 12 seconds", "answer_id": 2, "answer": "8 minutes and 24 seconds", "question_type": [ "Action Recognition", "Object Recognition", "Counterfactual" ], "objects_of_interest": [ "plastic piece round from juicer-afc4616e3bb9ebbd", "lid of food processor-61e7d409c57f74e6", "plastic lid-4b1bc1eed311477d", "third glass-e69d89c461ecb12a", "another glass-202aeb4488d6fd08", "glass-1e14b7e2aacc0554", "two parts of blade-7b371e22fa33bffe", "plastic piece-b4cbd19cffa9f5b6", "glove-1088ee85095ad2b0", "pot-628939386fd7ac72", "empty tupperware-cc7332248940d9cd", "brush-b9e00831d20cdb08", "blade of knife from sink-aa9aaad30e336a6d", "knife from sink-83bd1e72ec233647", "brown plate in dishwasher-3fbc776aa47f4db6", "plate from the sink-227fb374c212530c", "third bowl from the sink-1846c34c89453d95", "second bowl from sink-07424256b4ca0344", "bowl from sink-c5a5203e8911c8c6", "another white bowl-c150ba35717a3622", "another chopping board-f3d26d9a73db3988", "knife-9a354016f1d55a0d", "sponge-49fbc5073665000c", "white bowl-cd9ee625800e3e08", "tray-32bfe3d0b7019d90", "lid in dishwasher-9da3ea69065b1fea", "thermal flask lid-710454a7145847bf", "frying pan-16a00a5ac557ee9d", "thermal flask-383e93abe1fded86", "chopping board-0b3afccd4a17df2a", "empty glass-f9dcaa9af3835888", "scale-d41d55f1740cd4ce", "notepad-6fbf559cda7fc95a", "pen-872591fb42387dd7", "center piece-30966619ffe37d42", "centerpiece-77b907cc12d8e928", "center plastic piece-1005a622c53a895b", "food processor plug-e1b933f854a32061", "second spice jar-d95e5db2026b8936", "spice jar lid-3def4a95fe316d7f", "milk frother-ff653eee5f5cdd71", "spice jar-a113d8d9fad594f7", "salt bottle-f27fd204cbb4b41c", "kettle-bffd0bd346550cd1", "chicken stock cube-3abcb27b403a6f0a", "box of chicken cube stock-2e8ab5f46dc046c6", "wooden spoon 2-4f98a808afbbea60", "small spoon-1ec7f68ebbddbddb", "designer bowl-371a1b7f0a798584", "strainer-9ee450a932677732", "mug-36bececaeb54d3d7", "plastic packaging-48e351ac4da6813c", "plastic bag-14489cf41e6cb931", "bottle of vegetable oil-0539dc57c39570b0", "one piece of kitchen towel-f72f93cb09996e6f", "roll of kitchen towels-5b75526039e13cd5", "white saucepan lid-4a3120f382200b1c", "small black saucepan-2a3202d4e21f2a18", "whitw saucepan-89cb36901c54794b", "saucepan-13c1292ab79727c0", "wooden spoon-31d79ef33080751f", "lid-6b782523989b0081", "glass of orange juice-88a7d132cb47cbe0", "food trash bin-4f7fec57ba7c5c4c", "new garlic presser-c508fc81f27ce7c0" ], "reasoning": "1. I watched the video in order to see how long it took the user to pick up the food trash bin (food trash bin-4f7fec57ba7c5c4c). I noticed that the user walked into the kitchen with the food trash bin (food trash bin-4f7fec57ba7c5c4c) immediately at 00:00.\n2. At 00:06, I saw the user set the food trash bin (food trash bin-4f7fec57ba7c5c4c) down on the counter next to the food processor (centerpiece-77b907cc12d8e928). Then, I watched to see how long it took for the user to pick up the food trash bin (food trash bin-4f7fec57ba7c5c4c).\n3. At 16:54, I noticed the user finally picked up the food trash bin (food trash bin-4f7fec57ba7c5c4c) from its position on the counter.\n4. I subtracted 00:06 from 16:54 (16:54-00:06) to get a total of 16:48.\n5. I halved 16:48 (16:48/2) to get a final answer of 8 minutes and 24 seconds.", "referenced_object_masks": [ "food trash bin-4f7fec57ba7c5c4c", "centerpiece-77b907cc12d8e928" ] }, { "key": "P01-20240204-145458:bb63cadf059080fc1aa7109051e242b4de8ce9f1", "video_id": "P01-20240204-145458", "question": "Which container that I touch holds something different from its label?", "answer_choice_0": "The noodle package", "answer_choice_1": "The Nutella jar", "answer_choice_2": "The cooking oil bottle", "answer_choice_3": "The cinnamon jar", "answer_choice_4": "The salt bottle", "answer_id": 1, "answer": "The Nutella jar", "question_type": [ "Object Recognition", "Reading" ], "objects_of_interest": [ "plastic piece round from juicer-afc4616e3bb9ebbd", "lid of food processor-61e7d409c57f74e6", "plastic lid-4b1bc1eed311477d", "third glass-e69d89c461ecb12a", "another glass-202aeb4488d6fd08", "glass-1e14b7e2aacc0554", "two parts of blade-7b371e22fa33bffe", "plastic piece-b4cbd19cffa9f5b6", "glove-1088ee85095ad2b0", "pot-628939386fd7ac72", "empty tupperware-cc7332248940d9cd", "brush-b9e00831d20cdb08", "blade of knife from sink-aa9aaad30e336a6d", "knife from sink-83bd1e72ec233647", "brown plate in dishwasher-3fbc776aa47f4db6", "plate from the sink-227fb374c212530c", "third bowl from the sink-1846c34c89453d95", "second bowl from sink-07424256b4ca0344", "bowl from sink-c5a5203e8911c8c6", "another white bowl-c150ba35717a3622", "another chopping board-f3d26d9a73db3988", "knife-9a354016f1d55a0d", "sponge-49fbc5073665000c", "white bowl-cd9ee625800e3e08", "tray-32bfe3d0b7019d90", "lid in dishwasher-9da3ea69065b1fea", "thermal flask lid-710454a7145847bf", "frying pan-16a00a5ac557ee9d", "thermal flask-383e93abe1fded86", "chopping board-0b3afccd4a17df2a", "empty glass-f9dcaa9af3835888", "scale-d41d55f1740cd4ce", "notepad-6fbf559cda7fc95a", "pen-872591fb42387dd7", "center piece-30966619ffe37d42", "centerpiece-77b907cc12d8e928", "center plastic piece-1005a622c53a895b", "food processor plug-e1b933f854a32061", "second spice jar-d95e5db2026b8936", "spice jar lid-3def4a95fe316d7f", "milk frother-ff653eee5f5cdd71", "spice jar-a113d8d9fad594f7", "salt bottle-f27fd204cbb4b41c", "kettle-bffd0bd346550cd1", "chicken stock cube-3abcb27b403a6f0a", "box of chicken cube stock-2e8ab5f46dc046c6", "wooden spoon 2-4f98a808afbbea60", "small spoon-1ec7f68ebbddbddb", "designer bowl-371a1b7f0a798584", "strainer-9ee450a932677732", "mug-36bececaeb54d3d7", "plastic packaging-48e351ac4da6813c", "plastic bag-14489cf41e6cb931", "bottle of vegetable oil-0539dc57c39570b0", "one piece of kitchen towel-f72f93cb09996e6f", "roll of kitchen towels-5b75526039e13cd5", "white saucepan lid-4a3120f382200b1c", "small black saucepan-2a3202d4e21f2a18", "whitw saucepan-89cb36901c54794b", "saucepan-13c1292ab79727c0", "wooden spoon-31d79ef33080751f", "lid-6b782523989b0081", "glass of orange juice-88a7d132cb47cbe0", "food trash bin-4f7fec57ba7c5c4c", "new garlic presser-c508fc81f27ce7c0" ], "reasoning": "1. I looked for instances of containers holding something other than what appeared on their labels.\n2. I saw the user pull a Nutella jar (spice jar-a113d8d9fad594f7) out of a cupboard at 07:45. From 07:48-08:04, they open the jar (spice jar-a113d8d9fad594f7) and reach inside to grab a few pinches of a powdered spice and sprinkle it into a pan (saucepan-13c1292ab79727c0).\n3. The jar contained spice and not Nutella , therefore the answer is the Nutella jar (spice jar-a113d8d9fad594f7).", "referenced_object_masks": [ "spice jar-a113d8d9fad594f7", "saucepan-13c1292ab79727c0" ] }, { "key": "P01-20240204-145458:c1ad9f69172eb0677211fb0be4e3c1a2998ca287", "video_id": "P01-20240204-145458", "question": "What's the difference between the number of items I load into the dishwasher with my right hand versus my left hand?", "answer_choice_0": "6", "answer_choice_1": "10", "answer_choice_2": "8", "answer_choice_3": ", 7", "answer_choice_4": "9", "answer_id": 2, "answer": "8", "question_type": [ "Action Recognition", "Counting", "Egocentric & Spatial Perception", "Numerical Reasoning", "Object Recognition" ], "objects_of_interest": [ "plastic piece round from juicer-afc4616e3bb9ebbd", "lid of food processor-61e7d409c57f74e6", "plastic lid-4b1bc1eed311477d", "third glass-e69d89c461ecb12a", "another glass-202aeb4488d6fd08", "glass-1e14b7e2aacc0554", "two parts of blade-7b371e22fa33bffe", "plastic piece-b4cbd19cffa9f5b6", "glove-1088ee85095ad2b0", "pot-628939386fd7ac72", "empty tupperware-cc7332248940d9cd", "brush-b9e00831d20cdb08", "blade of knife from sink-aa9aaad30e336a6d", "knife from sink-83bd1e72ec233647", "brown plate in dishwasher-3fbc776aa47f4db6", "plate from the sink-227fb374c212530c", "third bowl from the sink-1846c34c89453d95", "second bowl from sink-07424256b4ca0344", "bowl from sink-c5a5203e8911c8c6", "another white bowl-c150ba35717a3622", "another chopping board-f3d26d9a73db3988", "knife-9a354016f1d55a0d", "sponge-49fbc5073665000c", "white bowl-cd9ee625800e3e08", "tray-32bfe3d0b7019d90", "lid in dishwasher-9da3ea69065b1fea", "thermal flask lid-710454a7145847bf", "frying pan-16a00a5ac557ee9d", "thermal flask-383e93abe1fded86", "chopping board-0b3afccd4a17df2a", "empty glass-f9dcaa9af3835888", "scale-d41d55f1740cd4ce", "notepad-6fbf559cda7fc95a", "pen-872591fb42387dd7", "center piece-30966619ffe37d42", "centerpiece-77b907cc12d8e928", "center plastic piece-1005a622c53a895b", "food processor plug-e1b933f854a32061", "second spice jar-d95e5db2026b8936", "spice jar lid-3def4a95fe316d7f", "milk frother-ff653eee5f5cdd71", "spice jar-a113d8d9fad594f7", "salt bottle-f27fd204cbb4b41c", "kettle-bffd0bd346550cd1", "chicken stock cube-3abcb27b403a6f0a", "box of chicken cube stock-2e8ab5f46dc046c6", "wooden spoon 2-4f98a808afbbea60", "small spoon-1ec7f68ebbddbddb", "designer bowl-371a1b7f0a798584", "strainer-9ee450a932677732", "mug-36bececaeb54d3d7", "plastic packaging-48e351ac4da6813c", "plastic bag-14489cf41e6cb931", "bottle of vegetable oil-0539dc57c39570b0", "one piece of kitchen towel-f72f93cb09996e6f", "roll of kitchen towels-5b75526039e13cd5", "white saucepan lid-4a3120f382200b1c", "small black saucepan-2a3202d4e21f2a18", "whitw saucepan-89cb36901c54794b", "saucepan-13c1292ab79727c0", "wooden spoon-31d79ef33080751f", "lid-6b782523989b0081", "glass of orange juice-88a7d132cb47cbe0", "food trash bin-4f7fec57ba7c5c4c", "new garlic presser-c508fc81f27ce7c0" ], "reasoning": "1. I looked for a segment where the user loads items into the dishwasher to compare the number of items loaded with each hand.\n2. I first saw the user open the dishwasher to load it at 11:35.\n3. I counted the items they loaded with their right hand: a strainer (strainer-9ee450a932677732) at 11:37, a white bowl (white bowl-cd9ee625800e3e08) at 11:56, a knife at 12:09, another white bowl (another white bowl-c150ba35717a3622) at 12:56, a ceramic bowl (bowl from sink-c5a5203e8911c8c6) at 13:08, a wooden spatula (blade of knife from sink-aa9aaad30e336a6d) at 13:45, another knife (knife from sink-83bd1e72ec233647) at 13:46, a plastic tupperware container (empty tupperware-cc7332248940d9cd) at 14:10, a pot (pot-628939386fd7ac72) at 14:31, a glass mug at 15:14, a glass (glass-1e14b7e2aacc0554) at 15:20, another glass (another glass-202aeb4488d6fd08) at 15:33, yet another glass (third glass-e69d89c461ecb12a) at 15:37, and a final glass (glass of orange juice-88a7d132cb47cbe0) at 17:31 for a total of 14.\n4. I then counted the items they loaded with their left hand: a knife (knife-9a354016f1d55a0d) at 12:15, a cutting board (chopping board-0b3afccd4a17df2a) at 12:27, another cutting board (another chopping board-f3d26d9a73db3988) at 12:40, a ceramic bowl (second bowl from sink-07424256b4ca0344) at 13:15, another ceramic bowl (third bowl from the sink-1846c34c89453d95) at 13:20, and a plate (plate from the sink-227fb374c212530c) at 13:29 for a total of 6.\n5. I subtracted 6 from 14 to reach 8.", "referenced_object_masks": [ "glass of orange juice-88a7d132cb47cbe0", "plate from the sink-227fb374c212530c", "chopping board-0b3afccd4a17df2a", "pot-628939386fd7ac72", "another glass-202aeb4488d6fd08", "glass-1e14b7e2aacc0554", "bowl from sink-c5a5203e8911c8c6", "third bowl from the sink-1846c34c89453d95", "third glass-e69d89c461ecb12a", "empty tupperware-cc7332248940d9cd", "another white bowl-c150ba35717a3622", "another chopping board-f3d26d9a73db3988", "knife from sink-83bd1e72ec233647", "strainer-9ee450a932677732", "knife-9a354016f1d55a0d", "white bowl-cd9ee625800e3e08", "second bowl from sink-07424256b4ca0344", "blade of knife from sink-aa9aaad30e336a6d" ] }, { "key": "P01-20240204-145458:ce5595dd8cf7a961699760dddc68bdc4e7c3120f", "video_id": "P01-20240204-145458", "question": "What would the total be if I added the number of ingredients I added to the pot to the number of times I drank from a glass?", "answer_choice_0": "5", "answer_choice_1": "7", "answer_choice_2": "6", "answer_choice_3": "8", "answer_choice_4": "9", "answer_id": 3, "answer": "8", "question_type": [ "Counterfactual", "Counting", "Numerical Reasoning", "Object Recognition", "Action Recognition", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "plastic piece round from juicer-afc4616e3bb9ebbd", "lid of food processor-61e7d409c57f74e6", "plastic lid-4b1bc1eed311477d", "third glass-e69d89c461ecb12a", "another glass-202aeb4488d6fd08", "glass-1e14b7e2aacc0554", "two parts of blade-7b371e22fa33bffe", "plastic piece-b4cbd19cffa9f5b6", "glove-1088ee85095ad2b0", "pot-628939386fd7ac72", "empty tupperware-cc7332248940d9cd", "brush-b9e00831d20cdb08", "blade of knife from sink-aa9aaad30e336a6d", "knife from sink-83bd1e72ec233647", "brown plate in dishwasher-3fbc776aa47f4db6", "plate from the sink-227fb374c212530c", "third bowl from the sink-1846c34c89453d95", "second bowl from sink-07424256b4ca0344", "bowl from sink-c5a5203e8911c8c6", "another white bowl-c150ba35717a3622", "another chopping board-f3d26d9a73db3988", "knife-9a354016f1d55a0d", "sponge-49fbc5073665000c", "white bowl-cd9ee625800e3e08", "tray-32bfe3d0b7019d90", "lid in dishwasher-9da3ea69065b1fea", "thermal flask lid-710454a7145847bf", "frying pan-16a00a5ac557ee9d", "thermal flask-383e93abe1fded86", "chopping board-0b3afccd4a17df2a", "empty glass-f9dcaa9af3835888", "scale-d41d55f1740cd4ce", "notepad-6fbf559cda7fc95a", "pen-872591fb42387dd7", "center piece-30966619ffe37d42", "centerpiece-77b907cc12d8e928", "center plastic piece-1005a622c53a895b", "food processor plug-e1b933f854a32061", "second spice jar-d95e5db2026b8936", "spice jar lid-3def4a95fe316d7f", "milk frother-ff653eee5f5cdd71", "spice jar-a113d8d9fad594f7", "salt bottle-f27fd204cbb4b41c", "kettle-bffd0bd346550cd1", "chicken stock cube-3abcb27b403a6f0a", "box of chicken cube stock-2e8ab5f46dc046c6", "wooden spoon 2-4f98a808afbbea60", "small spoon-1ec7f68ebbddbddb", "designer bowl-371a1b7f0a798584", "strainer-9ee450a932677732", "mug-36bececaeb54d3d7", "plastic packaging-48e351ac4da6813c", "plastic bag-14489cf41e6cb931", "bottle of vegetable oil-0539dc57c39570b0", "one piece of kitchen towel-f72f93cb09996e6f", "roll of kitchen towels-5b75526039e13cd5", "white saucepan lid-4a3120f382200b1c", "small black saucepan-2a3202d4e21f2a18", "whitw saucepan-89cb36901c54794b", "saucepan-13c1292ab79727c0", "wooden spoon-31d79ef33080751f", "lid-6b782523989b0081", "glass of orange juice-88a7d132cb47cbe0", "food trash bin-4f7fec57ba7c5c4c", "new garlic presser-c508fc81f27ce7c0" ], "reasoning": "1. I watched for the user adding cooking ingredients to a pot (whitw saucepan-89cb36901c54794b) and drinking from a glass (glass of orange juice-88a7d132cb47cbe0).\n2. I saw them add cooking oil (bottle of vegetable oil-0539dc57c39570b0) at 01:49, crushed noodles at 04:15, cooked rice at 05:51, a stock cube at 06:28, and water at 06:54 for a total of 5 ingredients.\n3. I saw the user drink from a glass (glass of orange juice-88a7d132cb47cbe0) at 00:10, 01:31, and 03:29 for a total of 3 times.\n4. I added 5 and 3 to reach 8.", "referenced_object_masks": [ "glass of orange juice-88a7d132cb47cbe0", "whitw saucepan-89cb36901c54794b", "bottle of vegetable oil-0539dc57c39570b0" ] }, { "key": "P01-20240204-145458:df6430058091c41a317dc7e88174e90e3af032b7", "video_id": "P01-20240204-145458", "question": "If I wanted to take the 4th unique glass that I touched in the video and place it where I found the chicken stock cubes, what actions would I need to take?", "answer_choice_0": "Turn completely around, walk forward, and turn right", "answer_choice_1": "Turn left, walk forward, and turn right", "answer_choice_2": "Turn right, walk forward, and turn right again", "answer_choice_3": "Turn completely around and walk forward", "answer_choice_4": "Turn left, walk forward, and turn left again", "answer_id": 2, "answer": "Turn right, walk forward, and turn right again", "question_type": [ "Counterfactual", "Action Recognition", "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "plastic piece round from juicer-afc4616e3bb9ebbd", "lid of food processor-61e7d409c57f74e6", "plastic lid-4b1bc1eed311477d", "third glass-e69d89c461ecb12a", "another glass-202aeb4488d6fd08", "glass-1e14b7e2aacc0554", "two parts of blade-7b371e22fa33bffe", "plastic piece-b4cbd19cffa9f5b6", "glove-1088ee85095ad2b0", "pot-628939386fd7ac72", "empty tupperware-cc7332248940d9cd", "brush-b9e00831d20cdb08", "blade of knife from sink-aa9aaad30e336a6d", "knife from sink-83bd1e72ec233647", "brown plate in dishwasher-3fbc776aa47f4db6", "plate from the sink-227fb374c212530c", "third bowl from the sink-1846c34c89453d95", "second bowl from sink-07424256b4ca0344", "bowl from sink-c5a5203e8911c8c6", "another white bowl-c150ba35717a3622", "another chopping board-f3d26d9a73db3988", "knife-9a354016f1d55a0d", "sponge-49fbc5073665000c", "white bowl-cd9ee625800e3e08", "tray-32bfe3d0b7019d90", "lid in dishwasher-9da3ea69065b1fea", "thermal flask lid-710454a7145847bf", "frying pan-16a00a5ac557ee9d", "thermal flask-383e93abe1fded86", "chopping board-0b3afccd4a17df2a", "empty glass-f9dcaa9af3835888", "scale-d41d55f1740cd4ce", "notepad-6fbf559cda7fc95a", "pen-872591fb42387dd7", "center piece-30966619ffe37d42", "centerpiece-77b907cc12d8e928", "center plastic piece-1005a622c53a895b", "food processor plug-e1b933f854a32061", "second spice jar-d95e5db2026b8936", "spice jar lid-3def4a95fe316d7f", "milk frother-ff653eee5f5cdd71", "spice jar-a113d8d9fad594f7", "salt bottle-f27fd204cbb4b41c", "kettle-bffd0bd346550cd1", "chicken stock cube-3abcb27b403a6f0a", "box of chicken cube stock-2e8ab5f46dc046c6", "wooden spoon 2-4f98a808afbbea60", "small spoon-1ec7f68ebbddbddb", "designer bowl-371a1b7f0a798584", "strainer-9ee450a932677732", "mug-36bececaeb54d3d7", "plastic packaging-48e351ac4da6813c", "plastic bag-14489cf41e6cb931", "bottle of vegetable oil-0539dc57c39570b0", "one piece of kitchen towel-f72f93cb09996e6f", "roll of kitchen towels-5b75526039e13cd5", "white saucepan lid-4a3120f382200b1c", "small black saucepan-2a3202d4e21f2a18", "whitw saucepan-89cb36901c54794b", "saucepan-13c1292ab79727c0", "wooden spoon-31d79ef33080751f", "lid-6b782523989b0081", "glass of orange juice-88a7d132cb47cbe0", "food trash bin-4f7fec57ba7c5c4c", "new garlic presser-c508fc81f27ce7c0" ], "reasoning": "1. I watched the video to determine where the 4th glass was that the user touched in the video. At 00:08, I saw the user pick up a glass of orange juice (glass of orange juice-88a7d132cb47cbe0), so I counted this as the 1st glass.\n2. At 15:17, I noticed the user touched another glass (glass-1e14b7e2aacc0554). This was followed soon afterward with a similar glass (another glass-202aeb4488d6fd08) at 15:31, and a 3rd glass (third glass-e69d89c461ecb12a) at 15:35. I counted this as the 4th unique glass overall, and noted that the user was standing between the sink and the dishwasher , oriented with their focus to the right of the sink in the corner. \n3. Armed with this knowledge, I went back and attempted to locate the chicken stock cubes (box of chicken cube stock-2e8ab5f46dc046c6). I found the package (box of chicken cube stock-2e8ab5f46dc046c6) inside the fridge at 03:47.\n4. When I saw the user pass by the fridge on the left at 00:02, I noted that the aforementioned corner to the right of the sink was straight ahead and slightly to the right.\n5. So, if the user wanted to take the 4th glass that they touched in the video and place it where they found the chicken stock cubes (box of chicken cube stock-2e8ab5f46dc046c6, they would need to turn right, walk forward, and then turn right again to be facing the fridge .", "referenced_object_masks": [ "another glass-202aeb4488d6fd08", "glass-1e14b7e2aacc0554", "glass of orange juice-88a7d132cb47cbe0", "box of chicken cube stock-2e8ab5f46dc046c6", "third glass-e69d89c461ecb12a" ] }, { "key": "P01-20240204-152537:1dd94e8efd7204b317e90ae6309a4b0637647b4b", "video_id": "P01-20240204-152537", "question": "Which is greater, and by how much (rounding to the nearest half minute): the time I spent wearing both cleaning gloves or the time that the rice was cooking before meat was added?", "answer_choice_0": "The time spent wearing both cleaning gloves--by 1 and a half minutes", "answer_choice_1": "Neither--they were approximately the same amount of time", "answer_choice_2": "The time the rice was cooking before the meat was added--by 1 minute", "answer_choice_3": "The time spent wearing both cleaning gloves--by 2 and a half minutes", "answer_choice_4": "The time the rice was cooking before the meat was added--by 2 minutes", "answer_id": 4, "answer": "The time the rice was cooking before the meat was added--by 2 minutes", "question_type": [ "Temporal Reasoning", "Object Recognition", "Numerical Reasoning" ], "objects_of_interest": [ "pan-c00c32ceb2e6e55f", "glass lid-392e6e6827aed2c2", "blue cloth-284ec7328b009cb2", "wire-14cede2dcafc00e7", "metalic piece-1671f16379855e48", "pen-33130383fc92e82b", "notepad-a70879259fdc3436", "food processor piece-7b32507df8dbeb1b", "food processor's motor-5d778431c8960fc8", "toaster tray-56b4b0efc7a483d6", "cloth-ad354c3e8b76b320", "sponge-08f5036d110e0c0a", "strainer-e64063f8842166f1", "food waste bin-83a972efd91c3a57", "one bag-d523f2667e875f41", "green bag's roll-d1b1eb9719d07839", "spoon-2acd48b16642f21b", "left glove-76b4b89b099376ab", "glove-b7274b383e51662d", "pot-9d284daf50934007", "wooden spoon-437d8b44615fe26e", "lid-2e3906b1e729f2a5" ], "reasoning": "1. I watched the video and noted each segment in which the user was wearing both cleaning gloves . This occurred from 01:09-01:27 and from 01:50-05:23. \n2. I looked for when the rice began cooking. The first time I saw the user interact with the pot of rice (pot-9d284daf50934007) was when they took the pot lid (lid-2e3906b1e729f2a5) off from 00:05-00:06. At 00:06, I saw steam come out of the pot of rice (pot-9d284daf50934007), and determined that this meant the food was already cooking. Therefore, I treated the start of the video as the start of when the pot of rice (pot-9d284daf50934007) began cooking. \n3. I looked for when meat was added to the pot of rice (pot-9d284daf50934007). At 05:59, I saw the user begin to transfer the meat from the pan (pan-c00c32ceb2e6e55f) into the pot of rice (pot-9d284daf50934007). \n4. I found the duration of the first time wearing both cleaning gloves by subtracting the start time from the end time: 01:27-01:09=00:18. \n5. I found the duration of the second time wearing both cleaning gloves by subtracting the start time from the end time: 05:23-01:50=03:33\n6. I added the two durations together: 00:18+03:33=03:51\n7. I found the duration that the pot of rice (pot-9d284daf50934007) was cooking before the meat was added by subtracting 05:59-00:00=05:59\n8. I found the difference between the 2 times (05:59-03:51) and saw that the time the pot of rice (pot-9d284daf50934007) was cooking before the meat was added was 2 minutes and 8 seconds longer than the total duration the user spent wearing both cleaning gloves , which would be roughly 2 minutes longer when rounding to the nearest half minute.", "referenced_object_masks": [ "pan-c00c32ceb2e6e55f", "lid-2e3906b1e729f2a5", "pot-9d284daf50934007" ] }, { "key": "P01-20240204-152537:2bfb34bdd63adfc4c2356150a1ae614b2906f42a", "video_id": "P01-20240204-152537", "question": "When I scraped the strainer for the first time, approximately how many degrees would I have to turn right if I wanted to grab the wooden spoon I used to stir the rice?", "answer_choice_0": "180", "answer_choice_1": "225", "answer_choice_2": "315", "answer_choice_3": "270", "answer_choice_4": "135", "answer_id": 0, "answer": "180", "question_type": [ "Temporal Reasoning", "Object Recognition", "Action Recognition", "Numerical Reasoning", "Egocentric & Spatial Perception", "Situational Awareness" ], "objects_of_interest": [ "pan-c00c32ceb2e6e55f", "glass lid-392e6e6827aed2c2", "blue cloth-284ec7328b009cb2", "wire-14cede2dcafc00e7", "metalic piece-1671f16379855e48", "pen-33130383fc92e82b", "notepad-a70879259fdc3436", "food processor piece-7b32507df8dbeb1b", "food processor's motor-5d778431c8960fc8", "toaster tray-56b4b0efc7a483d6", "cloth-ad354c3e8b76b320", "sponge-08f5036d110e0c0a", "strainer-e64063f8842166f1", "food waste bin-83a972efd91c3a57", "one bag-d523f2667e875f41", "green bag's roll-d1b1eb9719d07839", "spoon-2acd48b16642f21b", "left glove-76b4b89b099376ab", "glove-b7274b383e51662d", "pot-9d284daf50934007", "wooden spoon-437d8b44615fe26e", "lid-2e3906b1e729f2a5" ], "reasoning": "1. I watched the video and noted the layout of the kitchen as the user walked in from 00:00-00:03. I noted that two counters ran parallel to each other, with a stovetop on the left side. When the user stepped into the kitchen at 00:03, they were in between the two counters and facing the sink at the far end. \n2. I looked for when the user first scraped the strainer (strainer-e64063f8842166f1). From 02:15-02:17 I saw the user scrape the (strainer-e64063f8842166f1) with their finger over the food waste bin (food waste bin-83a972efd91c3a57). \n4. I observed the area from 02:15-02:17 and saw that the user was right by the sink, facing about 45 degrees right. \n5. I looked for the wooden spoon used to stir the rice. From 00:09-00:31 I saw the user stir the pot of rice (pot-9d284daf50934007) with a wooden spoon (wooden spoon-437d8b44615fe26e). At 00:32, I saw the user place the wooden spoon (wooden spoon-437d8b44615fe26e) just to the right of the stove. \n6. I watched the video from 00:32-02:17 and confirmed that the wooden spoon (wooden spoon-437d8b44615fe26e) was not moved in that time frame. \n7. At 00:58 I saw the user turn from facing the oven to facing the sink. I noted that there was also a dishwasher between the oven and the sink on the left. \n8. Since I couldn't tell exactly how far the wooden spoon (wooden spoon-437d8b44615fe26e) was in relation to the sink, I broke the problem down into smaller chunks to find the most likely answer. If the user was facing 45 degrees to the right of the sink from 02:15-02:17, turning another 45 degrees would have them facing the right counter. Turning another 90 degrees would have them facing away from the sink (45+90=135 total). If the user turned another 45 degrees (180 total), the user would likely be facing the right side of the stovetop where the wooden spoon (wooden spoon (wooden spoon-437d8b44615fe26e) was placed. In contrast, turning another 90 degrees after being opposite the sink (270 total) would likely have them facing the dishwasher. Therefore, I determined the most likely answer was 180 degrees.", "referenced_object_masks": [ "strainer-e64063f8842166f1", "pot-9d284daf50934007", "wooden spoon-437d8b44615fe26e", "food waste bin-83a972efd91c3a57" ] }, { "key": "P01-20240204-152537:4c8c252645151136b71d9170138b492dd17fb42d", "video_id": "P01-20240204-152537", "question": "In which order would one place the following: The amount of minutes (rounding to the nearest minute) that the pot on the stove was uncovered, twice the number of times I squeezed the dishrag, and 1 plus the number of times I threw something away in the trash - from highest to lowest?", "answer_choice_0": "Minutes that the pot was uncovered; twice the number of times the dishrag was squeezed; 1 plus the number of times something was thrown in the trash bin", "answer_choice_1": "1 plus the number of times something was thrown in the trash bin; minutes that the pot was uncovered; twice the number of times the dishrag was squeezed", "answer_choice_2": "1 plus the number of times something was thrown in the trash bin; twice the number of times the dishrag was squeezed; minutes that the pot was uncovered", "answer_choice_3": "Minutes that the pot was uncovered; 1 plus the number of times something was thrown in the trash bin; twice the number of times the dishrag was squeezed", "answer_choice_4": "Twice the number of times the dishrag was squeezed; 1 plus the number of times something was thrown in the trash bin; minutes that the pot was uncovered", "answer_id": 4, "answer": "Twice the number of times the dishrag was squeezed; 1 plus the number of times something was thrown in the trash bin; minutes that the pot was uncovered", "question_type": [ "Temporal Reasoning", "Counting", "Object Recognition", "Action Recognition", "Numerical Reasoning" ], "objects_of_interest": [ "pan-c00c32ceb2e6e55f", "glass lid-392e6e6827aed2c2", "blue cloth-284ec7328b009cb2", "wire-14cede2dcafc00e7", "metalic piece-1671f16379855e48", "pen-33130383fc92e82b", "notepad-a70879259fdc3436", "food processor piece-7b32507df8dbeb1b", "food processor's motor-5d778431c8960fc8", "toaster tray-56b4b0efc7a483d6", "cloth-ad354c3e8b76b320", "sponge-08f5036d110e0c0a", "strainer-e64063f8842166f1", "food waste bin-83a972efd91c3a57", "one bag-d523f2667e875f41", "green bag's roll-d1b1eb9719d07839", "spoon-2acd48b16642f21b", "left glove-76b4b89b099376ab", "glove-b7274b383e51662d", "pot-9d284daf50934007", "wooden spoon-437d8b44615fe26e", "lid-2e3906b1e729f2a5" ], "reasoning": "1. I watched the video and noted each segment in which the pot of rice (pot-9d284daf50934007) was not covered by the pot lid (lid-2e3906b1e729f2a5). This occurred from 00:06-00:34 and 05:27-07:05. \n2. I counted each time the user squeezed the dishrag (cloth-ad354c3e8b76b320). This occurred at 03:57, 05:09, and 05:11, making a total of 3 times. \n3. I counted each time the user threw something away in the trash bin . This occurred at 03:48 and 04:58, making a total of 2 times. \n4. I subtracted the end times from the start times for the 2 occurrences in which the pot of rice (pot-9d284daf50934007) was not covered by the pot lid (lid-2e3906b1e729f2a5). 00:34-00:06=00:28. 07:05-05:27=01:38. \n5. I added the two durations together: 00:28+01:38=02:06\n6. I multiplied the number of times the user squeezed the dishrag (cloth-ad354c3e8b76b320) by 2: 3*2=6.\n7. I added 1 to the number of times the user threw something in the trash bin : 2+1=3", "referenced_object_masks": [ "lid-2e3906b1e729f2a5", "pot-9d284daf50934007", "cloth-ad354c3e8b76b320" ] }, { "key": "P01-20240204-152537:ab61d955ffd909591e89e1073568107972973d73", "video_id": "P01-20240204-152537", "question": "When I moved the paper, what would I have had to do to place that paper under a magnet on the fridge?", "answer_choice_0": "Turn around and step right", "answer_choice_1": "Step left", "answer_choice_2": "Turn right and step forward", "answer_choice_3": "Step right", "answer_choice_4": "Turn around and step left", "answer_id": 4, "answer": "Turn around and step left", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness", "Action Recognition", "Counterfactual" ], "objects_of_interest": [ "pan-c00c32ceb2e6e55f", "glass lid-392e6e6827aed2c2", "blue cloth-284ec7328b009cb2", "wire-14cede2dcafc00e7", "metalic piece-1671f16379855e48", "pen-33130383fc92e82b", "notepad-a70879259fdc3436", "food processor piece-7b32507df8dbeb1b", "food processor's motor-5d778431c8960fc8", "toaster tray-56b4b0efc7a483d6", "cloth-ad354c3e8b76b320", "sponge-08f5036d110e0c0a", "strainer-e64063f8842166f1", "food waste bin-83a972efd91c3a57", "one bag-d523f2667e875f41", "green bag's roll-d1b1eb9719d07839", "spoon-2acd48b16642f21b", "left glove-76b4b89b099376ab", "glove-b7274b383e51662d", "pot-9d284daf50934007", "wooden spoon-437d8b44615fe26e", "lid-2e3906b1e729f2a5" ], "reasoning": "1. I watched the video and noted the layout of the kitchen as the user walked in from 00:00-00:03. I noted that there was a long counter separating the kitchen from another room. Opposite that counter was the refrigerator . When the user stepped fully into the kitchen at 00:03, I saw the sink at the far end of the kitchen.\n2. I looked for the time when the user moved a piece of paper (notepad-a70879259fdc3436) and saw this happen from 03:19-03:20.\n3. I observed the surroundings at 03:19-03:20 and noted that the user was facing the counter that separated the kitchen from another room.", "referenced_object_masks": [ "notepad-a70879259fdc3436" ] }, { "key": "P01-20240204-152537:d862cf575c38aa3434bbc6b592d291be31da761e", "video_id": "P01-20240204-152537", "question": "What would the result be if I divided the number of times I touched a pot or saucepan lid by the number of times I opened the oven?", "answer_choice_0": "4", "answer_choice_1": "5", "answer_choice_2": "2", "answer_choice_3": "3", "answer_choice_4": "6", "answer_id": 4, "answer": "6", "question_type": [ "Numerical Reasoning", "Counting", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "pan-c00c32ceb2e6e55f", "glass lid-392e6e6827aed2c2", "blue cloth-284ec7328b009cb2", "wire-14cede2dcafc00e7", "metalic piece-1671f16379855e48", "pen-33130383fc92e82b", "notepad-a70879259fdc3436", "food processor piece-7b32507df8dbeb1b", "food processor's motor-5d778431c8960fc8", "toaster tray-56b4b0efc7a483d6", "cloth-ad354c3e8b76b320", "sponge-08f5036d110e0c0a", "strainer-e64063f8842166f1", "food waste bin-83a972efd91c3a57", "one bag-d523f2667e875f41", "green bag's roll-d1b1eb9719d07839", "spoon-2acd48b16642f21b", "left glove-76b4b89b099376ab", "glove-b7274b383e51662d", "pot-9d284daf50934007", "wooden spoon-437d8b44615fe26e", "lid-2e3906b1e729f2a5" ], "reasoning": "1. I watched the video and noted each time the user touched the pot lid (lid-2e3906b1e729f2a5). This occurred at 00:05, 00:33, 05:27, and 07:04, making a total of 4 times. \n2. I noted each time the user touched the saucepan lid (glass lid-392e6e6827aed2c2). This occurred at 05:53 and 07:07, making a total of 2 times. \n3. I noted each time the user opened the oven door . This occurred at 00:39, making a total of 1 time.\n4. I added the number of times the pot lid (lid-2e3906b1e729f2a5) was touched to the number of times the saucepan lid (glass lid-392e6e6827aed2c2) was touched: 4+2=6. \n5. I divided 6 by the number of times the oven door was opened: 6/1=6.", "referenced_object_masks": [ "lid-2e3906b1e729f2a5", "glass lid-392e6e6827aed2c2" ] }, { "key": "P01-20240204-160230:24ba5535cb6efd1c01b762bd2de9957406a6301c", "video_id": "P01-20240204-160230", "question": "In what direction am I gazing when I retrieve the plate from the cupboard and also when I put back the oven mitts?", "answer_choice_0": "Left", "answer_choice_1": "Up", "answer_choice_2": "Down", "answer_choice_3": "Behind me", "answer_choice_4": "Forward", "answer_id": 4, "answer": "Forward", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Situational Awareness" ], "objects_of_interest": [ "fork-57d58200a58d127f", "lid-bf9499af77f007d2", "plastic spoon-a2dfb736e54a1734", "food processor bowl-817281af5dc8e4e0", "plate-ec2ad0fdebd30f3e", "pile of plates-3047df83bd662fd3", "tray-4c6cd9ca2adc8795", "oven glove2-0c036ab16c7caf7a", "oven glove-2378b070ff4b77d3", "heat diffuser-a03d37b5e13ded05", "pot-37d41a4b03309eca" ], "reasoning": "1. First, I determine when the user retrieves the plate (plate-ec2ad0fdebd30f3e), which occurs at 01:07. I see that the user is gazing up towards the cupboard. \n2. Next, I locate when the user puts back the oven mitts . The user puts back the first mitt (oven glove-2378b070ff4b77d3) at 00:57 and the second mitt (oven glove2-0c036ab16c7caf7a) at 01:01. I see that the user is gazing up towards the oven mitts . \n3. In both instances, the user is looking up, making this the answer.", "referenced_object_masks": [ "oven glove-2378b070ff4b77d3", "oven glove2-0c036ab16c7caf7a", "plate-ec2ad0fdebd30f3e" ] }, { "key": "P01-20240204-160230:4eb8e7c4675e25adc44906a456a39fb4be3a5e24", "video_id": "P01-20240204-160230", "question": "What is the next to last object I hold with my right hand before the second time I look in a drawer to the left of the stove?", "answer_choice_0": "Saucepan", "answer_choice_1": "Tongs", "answer_choice_2": "Bottle of olive oil", "answer_choice_3": "Pot lid", "answer_choice_4": "Baking sheet", "answer_id": 3, "answer": "Pot lid", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Counting", "Action Recognition", "Situational Awareness" ], "objects_of_interest": [ "fork-57d58200a58d127f", "lid-bf9499af77f007d2", "plastic spoon-a2dfb736e54a1734", "food processor bowl-817281af5dc8e4e0", "plate-ec2ad0fdebd30f3e", "pile of plates-3047df83bd662fd3", "tray-4c6cd9ca2adc8795", "oven glove2-0c036ab16c7caf7a", "oven glove-2378b070ff4b77d3", "heat diffuser-a03d37b5e13ded05", "pot-37d41a4b03309eca" ], "reasoning": "1. I found the first moment that the user looks into a drawer to the left of the stove at 01:13. \n2. Next, at 02:08, I saw the user look in a drawer directly above the first drawer they opened, which is also to the left of the stove. I counted this as the second instance. \n3. I then went back to find the last thing the user held with their right hand before this moment. At 02:02, the user sets down a black spoon (plastic spoon-a2dfb736e54a1734) which is the last thing they held before opening the drawer for the second time. \n4. I continued going back, and saw the user grab the lid (lid-bf9499af77f007d2) of a pot with their right hand at 01:48. I counted the lid (lid-bf9499af77f007d2) as the next to last thing the user holds with their right hand before opening a drawer to the left of the stove for a second time.", "referenced_object_masks": [ "lid-bf9499af77f007d2", "plastic spoon-a2dfb736e54a1734" ] }, { "key": "P01-20240204-160230:5cb5c0ecc5e13b1a9a54e083913c4ac8a24a937d", "video_id": "P01-20240204-160230", "question": "Where are the red oven mitts in relation to me when I first move the pot?", "answer_choice_0": "2 o'clock", "answer_choice_1": "8 o'clock", "answer_choice_2": "10 o'clock", "answer_choice_3": "7 o'clock", "answer_choice_4": "12 o'clock", "answer_id": 0, "answer": "2 o'clock", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness", "Event Occurence" ], "objects_of_interest": [ "fork-57d58200a58d127f", "lid-bf9499af77f007d2", "plastic spoon-a2dfb736e54a1734", "food processor bowl-817281af5dc8e4e0", "plate-ec2ad0fdebd30f3e", "pile of plates-3047df83bd662fd3", "tray-4c6cd9ca2adc8795", "oven glove2-0c036ab16c7caf7a", "oven glove-2378b070ff4b77d3", "heat diffuser-a03d37b5e13ded05", "pot-37d41a4b03309eca" ], "reasoning": "1. First, I determine the location of the red oven mitts . I spot them hanging on the wall to the right of the sink, visible at 00:05 when the user enters the kitchen.\n2. Next, I determine when the user first moves the pot (pot-37d41a4b03309eca). This occurs at 00:09, where they are facing the stove on the left side of the kitchen. \n3. The user is standing at the stove moving the pot (pot-37d41a4b03309eca) assuming this is 12 o'clock. So, the red oven mitts are to their right, on the right side of the sink which would be about 2 o'clock.", "referenced_object_masks": [ "pot-37d41a4b03309eca" ] }, { "key": "P01-20240204-160230:d3c189517e9ee99ffe0a99159b10d85f70befe65", "video_id": "P01-20240204-160230", "question": "Where is my apron in relation to me when I'm making my plate?", "answer_choice_0": "Hanging on the stove door", "answer_choice_1": "On the wall behind me", "answer_choice_2": "On my left", "answer_choice_3": "Hanging over the sink", "answer_choice_4": "On my right", "answer_id": 2, "answer": "On my left", "question_type": [ "Object Recognition", "Egocentric & Spatial Perception", "Action Recognition", "Situational Awareness" ], "objects_of_interest": [ "fork-57d58200a58d127f", "lid-bf9499af77f007d2", "plastic spoon-a2dfb736e54a1734", "food processor bowl-817281af5dc8e4e0", "plate-ec2ad0fdebd30f3e", "pile of plates-3047df83bd662fd3", "tray-4c6cd9ca2adc8795", "oven glove2-0c036ab16c7caf7a", "oven glove-2378b070ff4b77d3", "heat diffuser-a03d37b5e13ded05", "pot-37d41a4b03309eca" ], "reasoning": "1. First, I determine where the apron is located. I notice it for the first time at 00:21, hanging on the far left wall, opposite the window. The apron remains in this position throughout the entirety of the video. \n2. Next, I locate the time stamp when the user makes their plate (plate-ec2ad0fdebd30f3e). This occurs from 01:33-02:01 as the user stands at the stove , positioned on the left side of the kitchen.\n3. Therefore, the apron hangs on the user's left while they make their plate (plate-ec2ad0fdebd30f3e).", "referenced_object_masks": [ "plate-ec2ad0fdebd30f3e" ] }, { "key": "P01-20240204-160230:f16950ead97e8a49e7fe45b5d21ca793048c4a44", "video_id": "P01-20240204-160230", "question": "Where is the skillet in relation to me when I'm retrieving the roasted chicken from the oven?", "answer_choice_0": "Below me in the oven", "answer_choice_1": "In front of me on the right", "answer_choice_2": "Behind me on the left", "answer_choice_3": "In front of me on the left", "answer_choice_4": "Behind me on the right", "answer_id": 2, "answer": "Behind me on the left", "question_type": [ "Object Recognition", "Egocentric & Spatial Perception", "Action Recognition", "Situational Awareness" ], "objects_of_interest": [ "fork-57d58200a58d127f", "lid-bf9499af77f007d2", "plastic spoon-a2dfb736e54a1734", "food processor bowl-817281af5dc8e4e0", "plate-ec2ad0fdebd30f3e", "pile of plates-3047df83bd662fd3", "tray-4c6cd9ca2adc8795", "oven glove2-0c036ab16c7caf7a", "oven glove-2378b070ff4b77d3", "heat diffuser-a03d37b5e13ded05", "pot-37d41a4b03309eca" ], "reasoning": "1. First, I determine where the skillet is positioned. I discover the skillet is visible at 00:01, positioned on the right corner edge of the counter when the user walks into the kitchen. \n2. When the user turns the corner at 00:04, they walk to the stove on the left side of the kitchen. \n3. The user removes the sheet (tray-4c6cd9ca2adc8795) of roasted chicken from the oven , on the left, at 00:41. \n4. Since the skillet is sitting on the counter on the right side of the kitchen, and the user is facing the stove on the left, the skillet is behind the user on their left.", "referenced_object_masks": [ "tray-4c6cd9ca2adc8795" ] }, { "key": "P02-20240209-184316:1ee6428b2a4c366209f8f8f050d54d99b17daed8", "video_id": "P02-20240209-184316", "question": "Where am I when I can see the brown doormat with hearts at 07:41?", "answer_choice_0": "On the other side of the kitchen counter where the user was cutting up food", "answer_choice_1": "At the sink", "answer_choice_2": "In front of the microwave", "answer_choice_3": "In front of the stove", "answer_choice_4": "At the kitchen table", "answer_id": 0, "answer": "On the other side of the kitchen counter where the user was cutting up food", "question_type": [ "Egocentric & Spatial Perception", "Temporal Reasoning", "Situational Awareness" ], "objects_of_interest": [ "chopping board-fd0b21986c8318e2'" ], "reasoning": "1. I watched the video up to 07:41.\n2. At 07:36, the user walked around the kitchen counter.\n3. The user got a kitchen towel (towel-c11fae041c90ed76) and is now facing the door out of the kitchen.\n4. The user can see the brown mat with hearts directly ahead.", "referenced_object_masks": [ "towel-c11fae041c90ed76" ] }, { "key": "P02-20240209-184316:5b904f8cc42f703220a20c0ee14d249bd30748f9", "video_id": "P02-20240209-184316", "question": "During the sequence in which I do the following - place the trash bin on the floor, pick food off the spatula, season the potatoes, and squeeze out tomato paste - when does squeezing tomato paste occur in relation to the other actions?", "answer_choice_0": "Before placing trash bin on floor, but after seasoning potatoes and picking food off spatula", "answer_choice_1": "Before seasoning the potatoes and picking food off spatula, but after placing trash bin on floor", "answer_choice_2": "Before picking food off spatula, but after placing the trash bin on the floor and seasoning the potatoes", "answer_choice_3": "Before placing trash bin on floor and picking food off spatula, but after seasoning potatoes", "answer_choice_4": "Before seasoning the potatoes, picking food off spatula, and placing the trash bin on the floor", "answer_id": 0, "answer": "Before placing trash bin on floor, but after seasoning potatoes and picking food off spatula", "question_type": [ "Temporal Reasoning", "Action Recognition", "Event Occurence" ], "objects_of_interest": [ "paper-1c912d32b23d8742", "chopping board-fd0b21986c8318e2", "chef's knife-d51820ef3d0fc285", "bread knife-ef7048b3bcfa2633", "paring knife-e17adc5cf13704da", "bag of onions-c54c6523f7ea9787", "tomatoes-d70aa11c6f134c33", "bag of apples-5e14641248a273fc", "garlic-e2dc647353903ed7", "mushroom punnet-ae8cbe4975d05506", "bag of potatoes-936e0a9449d8206f", "potato-ae119b0a5dfdd67f", "second potato-581f3d668e4642b6", "kitchen towel holder-bd9c559b24c8072c", "kitchen towel-f7353d73ce5950f4", "kitchen towel2-3ddc671a4a4a1a5c", "covered potato-90939352d92c5c40", "tomato1-8b3eaf5994214ee8", "tomato2-454398a4d4f7015f", "tomato3-5b9e7804fd31ba8c", "tomato4-d2d0fd3b00c35649", "bowl-65784c9756ef77cd", "new track 1-90d758fb2f27d09b", "towel-c11fae041c90ed76", "frying pan-106bd6c72dd6f296", "empty milk carton-c609f299e89314ea", "bottle of oil-52021f4f6730725c", "remaining cloves of garlic-228cd8aadb05f056", "garlic clove-b39fcd995b2bae4a", "food waste bin-ef6bec4e52821490", "onion-0b9c20489b7b37aa", "onion2-2f87572c6fc40f9c", "bottle of salt-6def12b6cae73951", "spatula-077aa8d5f974e8ed", "pepper mill-617c5230a0f5ca3e", "cap of pepper mill-c28cf8065116bb73", "covered potato2-4e78a81106145a7c", "part of potato skin-ffea09fb11b74020", "slice of potato for discarding-2baa2823d329534b", "cut off part of potato-06a92a6b09b360f8", "small frying pan-43670be45bc916a8", "cover of punnet of mushrooms-1ee5ec40c8db6279", "mushroom-a66ffbeef537c960", "mushroom2-e40ea5785f052001", "mushroom3-0686f90c902dfc5c", "mushroom4-ec3b5636533251bc", "teaspoon-b823de64f41945e9", "tomato puree tube-e6264106ead4f241", "cap of tomato puree tube-229f19ee26b12eca", "mushrooms-6a5678f8535bb4f9", "mushrooms2-87b1d4dd5934233a", "garage door key-d8784f7084b38e1f", "bag of food-9c08520d9c1b75e3", "bag of sweet corn-0503e31ae7b52f73", "red knife-06950cab98093c8d", "scissors-12fe4953ff45336f", "plastic bag clip-a4951fa262bfe5e7", "worcestershire sauce bottle-b6c27bcef04a781b", "jar of garlic puree-282741fbb57f705e", "apple cider vinegar-4dfd380326f92e73", "plate1-cd74e0a7dbe5b3bb", "plate2-7af93510060bc894", "bowl2-2e24124d0a731542", "kitchen towen piece-1796873e1653b5d4", "plastic black spoon-4a58ba8ac6936eff", "plate3-f9e21088a5227ecd", "washing up liquid bottle-b0d98cab31c8be87", "kitchen towel3-dcdaa621f32ec68d" ], "reasoning": "1. I first watch the video to discover when I notice the first action in this sequence of actions - I identify it at 18:03, when the user uses their left hand to pick a speck of crusted food off of the spatula (spatula-077aa8d5f974e8ed).\n2. I continue watching to identify the next event in the sequence - it occurs from 18:12-18:14, when the user seasons the cubed potatoes in the small frying pan (small frying pan-43670be45bc916a8) with salt (bottle of salt-6def12b6cae73951), and continues from 18:16-18:29, when the user uses the pepper grinder to continue seasoning.\n3. I continue watching to identify the next event in the sequence - it occurs from 18:42-18:50, when the user unscrews the cap (cap of tomato puree tube-229f19ee26b12eca) off the tomato paste tube (tomato puree tube-e6264106ead4f241), and squeezes the tomato paste into the large pan (frying pan-106bd6c72dd6f296) full of onion and tomato chunks . \n4. I continue watching to identify the next event in the sequence - it occurs from 19:11-19:14, when the user takes the trash bin (food waste bin-ef6bec4e52821490) off the countertop and places it on the floor.\n5. With knowledge of the chronological order in which these events occur, I compare that order to the answer choices, and determine that only D aligns with this sequence, making D the correct answer.", "referenced_object_masks": [ "bottle of salt-6def12b6cae73951", "food waste bin-ef6bec4e52821490", "tomato puree tube-e6264106ead4f241", "frying pan-106bd6c72dd6f296", "spatula-077aa8d5f974e8ed", "small frying pan-43670be45bc916a8", "cap of tomato puree tube-229f19ee26b12eca" ] }, { "key": "P02-20240209-184316:7592f26cbe555e78f06235b3dd136475e9b14829", "video_id": "P02-20240209-184316", "question": "To the nearest whole number, what percentage of cored tomato wedges have I not yet sliced into when the microwave beeps to signal doneness for the first time?", "answer_choice_0": "69%", "answer_choice_1": "31%", "answer_choice_2": "75%", "answer_choice_3": "38%", "answer_choice_4": "25%", "answer_id": 0, "answer": "69%", "question_type": [ "Counting", "Numerical Reasoning", "Object Recognition", "Audio Reasoning" ], "objects_of_interest": [ "paper-1c912d32b23d8742", "chopping board-fd0b21986c8318e2", "chef's knife-d51820ef3d0fc285", "bread knife-ef7048b3bcfa2633", "paring knife-e17adc5cf13704da", "bag of onions-c54c6523f7ea9787", "tomatoes-d70aa11c6f134c33", "bag of apples-5e14641248a273fc", "garlic-e2dc647353903ed7", "mushroom punnet-ae8cbe4975d05506", "bag of potatoes-936e0a9449d8206f", "potato-ae119b0a5dfdd67f", "second potato-581f3d668e4642b6", "kitchen towel holder-bd9c559b24c8072c", "kitchen towel-f7353d73ce5950f4", "kitchen towel2-3ddc671a4a4a1a5c", "covered potato-90939352d92c5c40", "tomato1-8b3eaf5994214ee8", "tomato2-454398a4d4f7015f", "tomato3-5b9e7804fd31ba8c", "tomato4-d2d0fd3b00c35649", "bowl-65784c9756ef77cd", "new track 1-90d758fb2f27d09b", "towel-c11fae041c90ed76", "frying pan-106bd6c72dd6f296", "empty milk carton-c609f299e89314ea", "bottle of oil-52021f4f6730725c", "remaining cloves of garlic-228cd8aadb05f056", "garlic clove-b39fcd995b2bae4a", "food waste bin-ef6bec4e52821490", "onion-0b9c20489b7b37aa", "onion2-2f87572c6fc40f9c", "bottle of salt-6def12b6cae73951", "spatula-077aa8d5f974e8ed", "pepper mill-617c5230a0f5ca3e", "cap of pepper mill-c28cf8065116bb73", "covered potato2-4e78a81106145a7c", "part of potato skin-ffea09fb11b74020", "slice of potato for discarding-2baa2823d329534b", "cut off part of potato-06a92a6b09b360f8", "small frying pan-43670be45bc916a8", "cover of punnet of mushrooms-1ee5ec40c8db6279", "mushroom-a66ffbeef537c960", "mushroom2-e40ea5785f052001", "mushroom3-0686f90c902dfc5c", "mushroom4-ec3b5636533251bc", "teaspoon-b823de64f41945e9", "tomato puree tube-e6264106ead4f241", "cap of tomato puree tube-229f19ee26b12eca", "mushrooms-6a5678f8535bb4f9", "mushrooms2-87b1d4dd5934233a", "garage door key-d8784f7084b38e1f", "bag of food-9c08520d9c1b75e3", "bag of sweet corn-0503e31ae7b52f73", "red knife-06950cab98093c8d", "scissors-12fe4953ff45336f", "plastic bag clip-a4951fa262bfe5e7", "worcestershire sauce bottle-b6c27bcef04a781b", "jar of garlic puree-282741fbb57f705e", "apple cider vinegar-4dfd380326f92e73", "plate1-cd74e0a7dbe5b3bb", "plate2-7af93510060bc894", "bowl2-2e24124d0a731542", "kitchen towen piece-1796873e1653b5d4", "plastic black spoon-4a58ba8ac6936eff", "plate3-f9e21088a5227ecd", "washing up liquid bottle-b0d98cab31c8be87", "kitchen towel3-dcdaa621f32ec68d" ], "reasoning": "1. As background to determine when the microwave first beeps to signal doneness, I first identify the point in the video where the microwave is first put to use - this occurs from 01:12-01:18, when the user puts two covered potatoes (covered potato-90939352d92c5c40, covered potato2-4e78a81106145a7c) into the microwave and presses a series of buttons to begin heating them. The microwave beeps at this point, but these are beeps at the start of the sequence, distinct from those that would signal the potatoes being done. \n2. I then listen for more beeping from the microwave , which I hear many minutes later, from 06:19-06:23. This series of beeps, and the passage of time between them and the start of the cooking, plus the hum of the microwave that took place in the interim and stopped at 06:19, indicate to me that these beeps signal that the microwave is done cooking.\n3. With 06:19 as my frame of reference, I must then determine the percentage of cored tomato wedges that the user has cut by the time of the beep. To do this, I first determine the point in the video where the user finishes coring the tomato wedges - this occurs from 05:37-5:40, when the user slices the last tomato core from the last wedge .\n4. At this point, 05:40, I determine the total number of cored tomato wedges by counting the ones on the cutting board (chopping board-fd0b21986c8318e2) - at 05:40, I count 16 cored wedges, making this the denominator of my equation.\n5. From 05:40-06:19 (my frame of reference for microwave beeping), I count how many of these cored tomato wedges the user slices or begins to slice. I count a total of 5 - the first wedge is sliced from 05:46-05:50, the second wedge is sliced from 05:55-06:00, the third wedge is sliced from 06:01-06:07, the fourth wedge is sliced from 06:07-06:12, and the fifth wedge is sliced from 06:13-06:19, when the microwave beeps.\n6. Having now determined that the user sliced 5 out of the 16 total wedges by the time of the mircowave beep, I can therefore determine that 11 total wedges have not yet been sliced, through the equation 16 - 5 = 11. I now have my numerator, 11, for my final equation.\n7. I do the equation 11 / 16 = 0.6875, and express it as a percentage to the nearest whole number, which would be 69% of tomato wedges not yet sliced when the microwave beeped to signal doneness.", "referenced_object_masks": [ "chopping board-fd0b21986c8318e2", "covered potato-90939352d92c5c40", "covered potato2-4e78a81106145a7c" ] }, { "key": "P02-20240209-184316:78d9d8b735a541e95c02028764085c745d6f3b76", "video_id": "P02-20240209-184316", "question": "How many food items did I cut on the cutting board?", "answer_choice_0": "4", "answer_choice_1": "6", "answer_choice_2": "3", "answer_choice_3": "5", "answer_choice_4": "1", "answer_id": 3, "answer": "5", "question_type": [ "Action Recognition", "Object Recognition", "Counting", "Temporal Reasoning", "Event Occurence" ], "objects_of_interest": [ "chopping board-fd0b21986c8318e2'" ], "reasoning": "1. I watched the video and the first food item the user cut up on the cutting board (chopping board-fd0b21986c8318e2) were 4 red tomatoes (tomato4-d2d0fd3b00c35649) starting at 01:49.\n2. I continued to watch the video to see the second food item to be 2 onions (onion2-2f87572c6fc40f9c) starting at 09:08.\n3. Continuing to watch the video, the third food item was 2 pieces of fresh garlic (garlic clove-b39fcd995b2bae4a) starting at 11:36.\n4. Still watching the video, the fourth food item I saw was 2 potatoes (part of potato skin-ffea09fb11b74020) starting at 14:01.\n5. I watched the video to the end and the fifth and last food item to be cut up were mushrooms (mushroom3-0686f90c902dfc5c) starting at 16:31.\n6. Therefore, the total number of food items the user cut on the cutting board was 5.", "referenced_object_masks": [ "garlic clove-b39fcd995b2bae4a", "tomato4-d2d0fd3b00c35649", "onion2-2f87572c6fc40f9c", "part of potato skin-ffea09fb11b74020", "chopping board-fd0b21986c8318e2", "mushroom3-0686f90c902dfc5c" ] }, { "key": "P02-20240209-184316:912221b81f533ce9ee0ea83bbb09f804a3e76715", "video_id": "P02-20240209-184316", "question": "During the second-to-last minute of the video, where is the fridge in relation to me for the longest amount of time?", "answer_choice_0": "Diagonally behind and to my left", "answer_choice_1": "Diagonally in front and to my right", "answer_choice_2": "To my left", "answer_choice_3": "Behind me", "answer_choice_4": "In front of me", "answer_id": 4, "answer": "In front of me", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Numerical Reasoning", "Counting", "Temporal Reasoning" ], "objects_of_interest": [ "paper-1c912d32b23d8742", "chopping board-fd0b21986c8318e2", "chef's knife-d51820ef3d0fc285", "bread knife-ef7048b3bcfa2633", "paring knife-e17adc5cf13704da", "bag of onions-c54c6523f7ea9787", "tomatoes-d70aa11c6f134c33", "bag of apples-5e14641248a273fc", "garlic-e2dc647353903ed7", "mushroom punnet-ae8cbe4975d05506", "bag of potatoes-936e0a9449d8206f", "potato-ae119b0a5dfdd67f", "second potato-581f3d668e4642b6", "kitchen towel holder-bd9c559b24c8072c", "kitchen towel-f7353d73ce5950f4", "kitchen towel2-3ddc671a4a4a1a5c", "covered potato-90939352d92c5c40", "tomato1-8b3eaf5994214ee8", "tomato2-454398a4d4f7015f", "tomato3-5b9e7804fd31ba8c", "tomato4-d2d0fd3b00c35649", "bowl-65784c9756ef77cd", "new track 1-90d758fb2f27d09b", "towel-c11fae041c90ed76", "frying pan-106bd6c72dd6f296", "empty milk carton-c609f299e89314ea", "bottle of oil-52021f4f6730725c", "remaining cloves of garlic-228cd8aadb05f056", "garlic clove-b39fcd995b2bae4a", "food waste bin-ef6bec4e52821490", "onion-0b9c20489b7b37aa", "onion2-2f87572c6fc40f9c", "bottle of salt-6def12b6cae73951", "spatula-077aa8d5f974e8ed", "pepper mill-617c5230a0f5ca3e", "cap of pepper mill-c28cf8065116bb73", "covered potato2-4e78a81106145a7c", "part of potato skin-ffea09fb11b74020", "slice of potato for discarding-2baa2823d329534b", "cut off part of potato-06a92a6b09b360f8", "small frying pan-43670be45bc916a8", "cover of punnet of mushrooms-1ee5ec40c8db6279", "mushroom-a66ffbeef537c960", "mushroom2-e40ea5785f052001", "mushroom3-0686f90c902dfc5c", "mushroom4-ec3b5636533251bc", "teaspoon-b823de64f41945e9", "tomato puree tube-e6264106ead4f241", "cap of tomato puree tube-229f19ee26b12eca", "mushrooms-6a5678f8535bb4f9", "mushrooms2-87b1d4dd5934233a", "garage door key-d8784f7084b38e1f", "bag of food-9c08520d9c1b75e3", "bag of sweet corn-0503e31ae7b52f73", "red knife-06950cab98093c8d", "scissors-12fe4953ff45336f", "plastic bag clip-a4951fa262bfe5e7", "worcestershire sauce bottle-b6c27bcef04a781b", "jar of garlic puree-282741fbb57f705e", "apple cider vinegar-4dfd380326f92e73", "plate1-cd74e0a7dbe5b3bb", "plate2-7af93510060bc894", "bowl2-2e24124d0a731542", "kitchen towen piece-1796873e1653b5d4", "plastic black spoon-4a58ba8ac6936eff", "plate3-f9e21088a5227ecd", "washing up liquid bottle-b0d98cab31c8be87", "kitchen towel3-dcdaa621f32ec68d" ], "reasoning": "1. To determine this answer, I watch the second-to-last minute of the video: since the video is 29:48 long, I watch from 27:48-28:48.\n2. From 27:48-27:51, I watch the user scrape out the large frying pan (frying pan-106bd6c72dd6f296) on the counter space, with the fridge directly in front of them, for a total of 3 seconds.\n3. From 27:51-27:57 (6 seconds) I watch the user turn around 180 degrees and scrape the frying pan (frying pan-106bd6c72dd6f296) over the stovetop, with the fridge directly behind them.\n4. From 27:57-28:13 (16 seconds) I watched the user turn around 180 degrees again and and scrape out the frying pan (frying pan-106bd6c72dd6f296) with the fridge directly in front of them.\n5. From 28:13-28:15 (2 seconds), I watched the user turn around 180 degrees again toward the stovetop , facing directly away from the fridge .\n6. From 28:15-28:32 (17 seconds) I watched the user turn around 180 degrees again and finish scooping food out from the large frying pan (frying pan-106bd6c72dd6f296), facing the fridge again.\n7. From 28:32-28:38, I watched the user turn around with the empty pan (frying pan-106bd6c72dd6f296) toward the stovetop , facing away from the fridge for 6 seconds.\n8. From 28:38-28:42, 4 seconds, I watched the user face the sink , with the fridge to their right.\n9. From 28:42-28:48, I watched the user face the corner of the counter between the sink and stovetop , with the fridge diagonally behind them and to their right.\n10. I recognize that the user spent the most time out of this minute directly facing the fridge by adding the numbers from steps 2, 4, and 6 with the equation 3 +16 + 17 = 36, during this timespan, making A the correct answer.", "referenced_object_masks": [ "frying pan-106bd6c72dd6f296" ] }, { "key": "P02-20240209-184316:b77880a24a72397ebf92520f646c40188a6a0c7a", "video_id": "P02-20240209-184316", "question": "Where can the third unique type of edible foodstuff I touch after I put the sliced tomatoes in the pan be found when I touch it?", "answer_choice_0": "To the right of the stove", "answer_choice_1": "To the left of the cutting board", "answer_choice_2": "Behind the cutting board", "answer_choice_3": "To the right of the cutting board", "answer_choice_4": "Behind the stove", "answer_id": 1, "answer": "To the left of the cutting board", "question_type": [ "Counting", "Temporal Reasoning", "Action Recognition", "Object Recognition", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "paper-1c912d32b23d8742", "chopping board-fd0b21986c8318e2", "chef's knife-d51820ef3d0fc285", "bread knife-ef7048b3bcfa2633", "paring knife-e17adc5cf13704da", "bag of onions-c54c6523f7ea9787", "tomatoes-d70aa11c6f134c33", "bag of apples-5e14641248a273fc", "garlic-e2dc647353903ed7", "mushroom punnet-ae8cbe4975d05506", "bag of potatoes-936e0a9449d8206f", "potato-ae119b0a5dfdd67f", "second potato-581f3d668e4642b6", "kitchen towel holder-bd9c559b24c8072c", "kitchen towel-f7353d73ce5950f4", "kitchen towel2-3ddc671a4a4a1a5c", "covered potato-90939352d92c5c40", "tomato1-8b3eaf5994214ee8", "tomato2-454398a4d4f7015f", "tomato3-5b9e7804fd31ba8c", "tomato4-d2d0fd3b00c35649", "bowl-65784c9756ef77cd", "new track 1-90d758fb2f27d09b", "towel-c11fae041c90ed76", "frying pan-106bd6c72dd6f296", "empty milk carton-c609f299e89314ea", "bottle of oil-52021f4f6730725c", "remaining cloves of garlic-228cd8aadb05f056", "garlic clove-b39fcd995b2bae4a", "food waste bin-ef6bec4e52821490", "onion-0b9c20489b7b37aa", "onion2-2f87572c6fc40f9c", "bottle of salt-6def12b6cae73951", "spatula-077aa8d5f974e8ed", "pepper mill-617c5230a0f5ca3e", "cap of pepper mill-c28cf8065116bb73", "covered potato2-4e78a81106145a7c", "part of potato skin-ffea09fb11b74020", "slice of potato for discarding-2baa2823d329534b", "cut off part of potato-06a92a6b09b360f8", "small frying pan-43670be45bc916a8", "cover of punnet of mushrooms-1ee5ec40c8db6279", "mushroom-a66ffbeef537c960", "mushroom2-e40ea5785f052001", "mushroom3-0686f90c902dfc5c", "mushroom4-ec3b5636533251bc", "teaspoon-b823de64f41945e9", "tomato puree tube-e6264106ead4f241", "cap of tomato puree tube-229f19ee26b12eca", "mushrooms-6a5678f8535bb4f9", "mushrooms2-87b1d4dd5934233a", "garage door key-d8784f7084b38e1f", "bag of food-9c08520d9c1b75e3", "bag of sweet corn-0503e31ae7b52f73", "red knife-06950cab98093c8d", "scissors-12fe4953ff45336f", "plastic bag clip-a4951fa262bfe5e7", "worcestershire sauce bottle-b6c27bcef04a781b", "jar of garlic puree-282741fbb57f705e", "apple cider vinegar-4dfd380326f92e73", "plate1-cd74e0a7dbe5b3bb", "plate2-7af93510060bc894", "bowl2-2e24124d0a731542", "kitchen towen piece-1796873e1653b5d4", "plastic black spoon-4a58ba8ac6936eff", "plate3-f9e21088a5227ecd", "washing up liquid bottle-b0d98cab31c8be87", "kitchen towel3-dcdaa621f32ec68d" ], "reasoning": "1. To determine this answer, I first find when the user places the sliced tomatoes into the large pan (frying pan-106bd6c72dd6f296). I watch the user slide the sliced tomatoes off the cutting board (chopping board-fd0b21986c8318e2) and into the large pan (frying pan-106bd6c72dd6f296) from 08:36-08:39.\n2. From 08:39 onward, I observe what the user does with their hands, and begin to count the unqiue foodstuffs they touch in order: the first food item I see the user touch is salt , when they pour it from the bottle of salt (bottle of salt-6def12b6cae73951) onto their left hand from 08:49-08:50. I count this as the first item.\n3. I continue watching and observe the user touch an onion (onion2-2f87572c6fc40f9c) at 09:07, which they begin to chop. I count this as the second item. \n4. At 09:49, the user grabs the other onion (onion-0b9c20489b7b37aa) and begins to chop it, but since this another onion, and thus not a unique type of food from any that have already been touched by the user, I do not count it. I make the same distinction for a third onion which is cut at 10:30. \n5. I continue watching until I see the user touch another unique type of food item: this occurs at 11:34, when the user grabs a garlic clove (garlic clove-b39fcd995b2bae4a) that sits to the left of the cutting board (chopping board-fd0b21986c8318e2) and begins to chop it. I count this as the third item, and, noting its location, therefore obtain D as the correct answer.", "referenced_object_masks": [ "garlic clove-b39fcd995b2bae4a", "bottle of salt-6def12b6cae73951", "frying pan-106bd6c72dd6f296", "onion2-2f87572c6fc40f9c", "onion-0b9c20489b7b37aa", "chopping board-fd0b21986c8318e2" ] }, { "key": "P02-20240209-184316:d9f45c8cce809a0ac9c9d3c05113085fdf4aa19d", "video_id": "P02-20240209-184316", "question": "What is the total number of tomato slices I scoop out the flesh of each slice?", "answer_choice_0": "10", "answer_choice_1": "8", "answer_choice_2": "12", "answer_choice_3": "4", "answer_choice_4": "16", "answer_id": 4, "answer": "16", "question_type": [ "Counting", "Object Recognition", "Temporal Reasoning", "Action Recognition", "Numerical Reasoning" ], "objects_of_interest": [ "chopping board-fd0b21986c8318e2'" ], "reasoning": "1. The user gets 4 tomatoes (tomato4-d2d0fd3b00c35649) out of the refrigerator at 00:26.\n2. From 01:49 to 05:45, the user scoops out the flesh of each tomato slice (new track 1-90d758fb2f27d09b).\n3. The user had 4 tomatoes (tomato4-d2d0fd3b00c35649) and cut each tomato (new track 1-90d758fb2f27d09b) into 4 slices.\n4. 4*4 = 16 slices", "referenced_object_masks": [ "new track 1-90d758fb2f27d09b", "tomato4-d2d0fd3b00c35649" ] }, { "key": "P02-20240209-184316:db1f10537381eb9b82b97ae832da1a109a1ea977", "video_id": "P02-20240209-184316", "question": "Where is the stove in relation to me when I'm at the cutting board?", "answer_choice_0": "On the other side of the counter where the user is cutting up food", "answer_choice_1": "To the left of the user", "answer_choice_2": "Behind the user", "answer_choice_3": "In front of the user", "answer_choice_4": "To the right of the user", "answer_id": 2, "answer": "Behind the user", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "chopping board-fd0b21986c8318e2'" ], "reasoning": "1. I watched the video and saw at 07:46, the user turned around to put a frying pan (frying pan-106bd6c72dd6f296) on the stove .\n2. The stove is directly behind the user when the user is at the counter using the cutting board (chopping board-fd0b21986c8318e2).", "referenced_object_masks": [ "chopping board-fd0b21986c8318e2", "frying pan-106bd6c72dd6f296" ] }, { "key": "P02-20240209-184316:f880c7cd1813093329ed9172a707b3180f1df40b", "video_id": "P02-20240209-184316", "question": "What is the ratio of the total number of cuts I made to the mushrooms to the total number of cuts I made to the first potato I cut when each is on the cutting board?", "answer_choice_0": "27:13", "answer_choice_1": "29:11", "answer_choice_2": "29:13", "answer_choice_3": "13:27", "answer_choice_4": "11:29", "answer_id": 2, "answer": "29:13", "question_type": [ "Counting", "Numerical Reasoning", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "paper-1c912d32b23d8742", "chopping board-fd0b21986c8318e2", "chef's knife-d51820ef3d0fc285", "bread knife-ef7048b3bcfa2633", "paring knife-e17adc5cf13704da", "bag of onions-c54c6523f7ea9787", "tomatoes-d70aa11c6f134c33", "bag of apples-5e14641248a273fc", "garlic-e2dc647353903ed7", "mushroom punnet-ae8cbe4975d05506", "bag of potatoes-936e0a9449d8206f", "potato-ae119b0a5dfdd67f", "second potato-581f3d668e4642b6", "kitchen towel holder-bd9c559b24c8072c", "kitchen towel-f7353d73ce5950f4", "kitchen towel2-3ddc671a4a4a1a5c", "covered potato-90939352d92c5c40", "tomato1-8b3eaf5994214ee8", "tomato2-454398a4d4f7015f", "tomato3-5b9e7804fd31ba8c", "tomato4-d2d0fd3b00c35649", "bowl-65784c9756ef77cd", "new track 1-90d758fb2f27d09b", "towel-c11fae041c90ed76", "frying pan-106bd6c72dd6f296", "empty milk carton-c609f299e89314ea", "bottle of oil-52021f4f6730725c", "remaining cloves of garlic-228cd8aadb05f056", "garlic clove-b39fcd995b2bae4a", "food waste bin-ef6bec4e52821490", "onion-0b9c20489b7b37aa", "onion2-2f87572c6fc40f9c", "bottle of salt-6def12b6cae73951", "spatula-077aa8d5f974e8ed", "pepper mill-617c5230a0f5ca3e", "cap of pepper mill-c28cf8065116bb73", "covered potato2-4e78a81106145a7c", "part of potato skin-ffea09fb11b74020", "slice of potato for discarding-2baa2823d329534b", "cut off part of potato-06a92a6b09b360f8", "small frying pan-43670be45bc916a8", "cover of punnet of mushrooms-1ee5ec40c8db6279", "mushroom-a66ffbeef537c960", "mushroom2-e40ea5785f052001", "mushroom3-0686f90c902dfc5c", "mushroom4-ec3b5636533251bc", "teaspoon-b823de64f41945e9", "tomato puree tube-e6264106ead4f241", "cap of tomato puree tube-229f19ee26b12eca", "mushrooms-6a5678f8535bb4f9", "mushrooms2-87b1d4dd5934233a", "garage door key-d8784f7084b38e1f", "bag of food-9c08520d9c1b75e3", "bag of sweet corn-0503e31ae7b52f73", "red knife-06950cab98093c8d", "scissors-12fe4953ff45336f", "plastic bag clip-a4951fa262bfe5e7", "worcestershire sauce bottle-b6c27bcef04a781b", "jar of garlic puree-282741fbb57f705e", "apple cider vinegar-4dfd380326f92e73", "plate1-cd74e0a7dbe5b3bb", "plate2-7af93510060bc894", "bowl2-2e24124d0a731542", "kitchen towen piece-1796873e1653b5d4", "plastic black spoon-4a58ba8ac6936eff", "plate3-f9e21088a5227ecd", "washing up liquid bottle-b0d98cab31c8be87", "kitchen towel3-dcdaa621f32ec68d" ], "reasoning": "1. To determine this ratio, I first identify the point in the video where the user cuts either a mushroom or potato on the cutting board (chopping board-fd0b21986c8318e2) for the first time. I see this happen at 14:09 and 14:13, respectively, when the user makes 2 cuts to the first potato (potato-ae119b0a5dfdd67f), cutting off each end on either side.\n2. After a short interim of cutting the second potato (second potato-581f3d668e4642b6) from 14:23-14:29, I watch the user return to cutting the first potato (potato-ae119b0a5dfdd67f) from 14:33-14:51. During this span, I count the number of cuts that the user makes to the first potato (potato-ae119b0a5dfdd67f): 1 at 14:33, 1 at 14:39, 1 at 14:40, 1 at 14:42, 1 at 14:43, 1 at 14:44, 1 at 14:46, 1 at 14:47, 2 at 14:49, 1 at 14:50, or 11 cuts in all.\n3. I continue watching as the user cuts the second potato (second potato-581f3d668e4642b6) from 14:53-15:04 to ensure that the user makes no more cuts to the first potato (potato-ae119b0a5dfdd67f) while it's on the cutting board (chopping board-fd0b21986c8318e2), and after successfully determing this at 16:43 when the user clears the cutting board (chopping board-fd0b21986c8318e2) of the potato slices (cut off part of potato-06a92a6b09b360f8s) without making any further cuts to the first potato (potato-ae119b0a5dfdd67f), I add 11 and 2 to find that the user cut the first potato (potato-ae119b0a5dfdd67f) 13 times while it was on the cutting board.\n4. I continue watching the video to find when the user cuts the mushrooms (mushroom-a66ffbeef537c960, mushroom2-e40ea5785f052001, mushroom3-0686f90c902dfc5c, mushroom4-ec3b5636533251bc), which I observe when they are sliced on the cutting board (chopping board-fd0b21986c8318e2). From 16:51-17:24, I count the number of cuts that the user makes to these mushrooms (mushroom-a66ffbeef537c960, mushroom2-e40ea5785f052001, mushroom3-0686f90c902dfc5c, mushroom4-ec3b5636533251bc): 1 at 16:51, 1 at 16:53, 1 at 16:54, 1 at 16:55, 1 at 16:56, 1 at 16:58, 1 at 17:01, 1 at 17:02, 1 at 17:03, 1 at 17:04, 2 at 17:07, 2 at 17:08, 1 at 17:09, 1 at 17:11, 1 at 17:14, 2 at 17:15, 1 at 17:16, 1 at 17:17, 1 at 17:18, 1 at 17:20, 1 at 17:21, 2 at 17:22, 1 at 17:24, or 27 cuts in all.\n5. I continue watching the video to ensure that no more cuts are made to the mushrooms (mushroom-a66ffbeef537c960, mushroom2-e40ea5785f052001, mushroom3-0686f90c902dfc5c, mushroom4-ec3b5636533251bc) while they are on the chopping board (chopping board-fd0b21986c8318e2), but do observe 2 more cuts at 18:58 and 19:00, respectively, before the user clears the mushroom slices (mushrooms-6a5678f8535bb4f9, mushrooms2-87b1d4dd5934233a) off the cutting board (chopping board-fd0b21986c8318e2). Therefore, I add 2 to the original 27 for a total of 29 cuts.\n6. I place these final numbers from step 3 and step 5 into a ratio, per the order of the question, to obtain a final answer of 29:13.", "referenced_object_masks": [ "mushrooms-6a5678f8535bb4f9", "mushrooms2-87b1d4dd5934233a", "mushroom-a66ffbeef537c960", "mushroom2-e40ea5785f052001", "mushroom3-0686f90c902dfc5c", "mushroom4-ec3b5636533251bc", "cut off part of potato-06a92a6b09b360f8", "chopping board-fd0b21986c8318e2", "potato-ae119b0a5dfdd67f", "second potato-581f3d668e4642b6" ] }, { "key": "P02-20240209-184316:f97d746706f8d1f43b8f4ffa7fed1f799c9de7be", "video_id": "P02-20240209-184316", "question": "How many items are on the second shelf above the second drawer opened in the refrigerator?", "answer_choice_0": "3", "answer_choice_1": "4", "answer_choice_2": "2", "answer_choice_3": "0", "answer_choice_4": "1", "answer_id": 2, "answer": "2", "question_type": [ "Action Recognition", "Counting", "Event Occurence", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "chopping board-fd0b21986c8318e2'" ], "reasoning": "1. I watched the video and the second drawer in the refrigerator was opened by the user at 00:36.\n2. The first shelf above the second drawer is empty.\n3. Two shelves above the drawer has 2 items on it.", "referenced_object_masks": [] }, { "key": "P02-20240209-194141:09e1d2b96d1d76b53eb9c2b97e00903f04b2562b", "video_id": "P02-20240209-194141", "question": "Assuming that each of the following dishes/cookware that I wash in the video were a different shape--plates = squares, bowls = triangles, pans = hexagons--, what number would I get if I multiplied the number of angles of clean dishes/cookware by the number of angles of dirty dishes/cookware before I do the dishes?", "answer_choice_0": "374", "answer_choice_1": "272", "answer_choice_2": "330", "answer_choice_3": "407", "answer_choice_4": "341", "answer_id": 0, "answer": "374", "question_type": [ "Action Recognition", "Counterfactual", "Counting", "Numerical Reasoning", "Object Recognition", "State Changes" ], "objects_of_interest": [ "Small thing to throw-ff936c7fde6cdd7b", "paper-cddf0363575a77bf", "knife-78f51ce9495d0bc5", "dirty plate-2fcaa07234d783ec", "bin lid-29369059ecdf8218", "another plate-214990ea27e29c82", "plate-c73b80b857dda267", "milk bottle-5cfb7518e14a1974", "glass-e1e5c13db45111a6", "scissors-5c52e5b121baf315", "brush-74776df3b4f98c37", "fork-6fa21391be43c692", "knife and fork-768aa635c0cb2387", "spoon-1c45c389db9047c3", "spatula-cf57787fc724fef1", "bowl-3c0f2e545387189d", "washing soup-acbb30a683107bfd", "plastic-5dae67113ef4ab31", "small bowl-e17f2059959db602", "grater-471f4305a01489f2", "box-115ed01fcd5adaa1", "red thing-de4868f3da0f5b24", "dirty cutlery-ff57979fc12181e1", "new fork-fde20363341988a5", "dirty cutlery 2-e697e6673ba34267", "dirty cutlery 4-cda04cfaa2728606", "no clue-7b6aa74be938d105", "no clue2-63307e0b6e11978d", "bowl3-3d9a8057cfd77698", "frying pan-602ae1d27ad60cd2", "food bin-2fcc7254de1cdc3f", "sponge-dd06dd3546c214a5", "paring knife-3c4389caa8961a89", "washing up brush-5c929c5eb60a0d23", "chef's knife-2a4212a9e675e83a", "bread knife-ac6b5668dfb756b0", "chopping board-30ce9207f6d26b7b", "remaining frying pan-32da07cf6e756f93", "lid of food bin-53008f884eb070b3", "new thing-063bef0e819e7cf1", "new fork 2-bb1ca842e7021484", "new knife-0dac74fb3bfba0f7", "tupperware lid-31f8ee3b8aaca4e9", "olive oil bottle-816c865ac4cdd983", "pepper mill-f662f67bc5bc2bb5", "pan support-3e2c8a56a6352636", "both grill covers-4f2680dee04bc05e", "plastic microwave cover-553f1dcc3bbab873", "second pan support-295d5dbcb869109e", "mat-8d314d89c4fb8691", "roll of drawer liners-48cb1485799139be", "other glass countertop protector-4bfc29e7deb6a9ab", "drain protector-2640ae3f60f2a571", "new glass-15c25cecf7ae49ec", "bottle of squash-b1eb2675d7ff6e6e" ], "reasoning": "1. I began by looking for a clear view of all dishes/cookware that the user either intends to wash or has washed already. At 00:02 I saw 1 dirty plate (dirty plate-2fcaa07234d783ec)(4) and 1 bowl on the dining room table. Since the user never washes the bowl, it does not count toward the total number of angles to be calculated and can thus be disregarded. At 00:19 the user picks up 2 clean plates (another plate-214990ea27e29c82, plate-c73b80b857dda267)(8) to put away. At 01:16 the user puts away a clean bowl (small bowl-e17f2059959db602)(3). At 01:41 the user begins to wash a dirty bowl (bowl3-3d9a8057cfd77698)(3). At 01:48 the user picks up a pan that will be washed (frying plan-602ae1)(6). At 01:56 there is a good view of the remaining dishes that will be washed: 1 additional bowl (3), 1 additional plate (4) and 1 additional pan (remaining frying pan-32da07cf6e756f93)(6). At 03:47 the user picks up 1 dirty plate (4) from the left of the stove to wash. At 03:50 the user picks up 1 more dirty plate (4) to wash. These are all of the dishes/cookware that come into consideration.\n2. From here, I calculated the sum of all clean angles: 8+3=11.\n3. I then calculated the sum of all dirty/pre-washed angles: 4+3+6+3+4+6+4+4=34.\n4. Finally, I calculated the product of clean angles (11) and pre-washed angles (34): 11*34=374.", "referenced_object_masks": [ "remaining frying pan-32da07cf6e756f93", "small bowl-e17f2059959db602", "bowl3-3d9a8057cfd77698", "another plate-214990ea27e29c82", "plate-c73b80b857dda267", "dirty plate-2fcaa07234d783ec" ] }, { "key": "P02-20240209-194141:0f2e4f7e790543af8fdb4f95a3242a2899843ebd", "video_id": "P02-20240209-194141", "question": "How many plates did I handle before I turned the lights on?", "answer_choice_0": "5", "answer_choice_1": "3", "answer_choice_2": "2", "answer_choice_3": "4", "answer_choice_4": "1", "answer_id": 1, "answer": "3", "question_type": [ "Counting", "Action Recognition", "Egocentric & Spatial Perception", "Object Recognition", "State Changes" ], "objects_of_interest": [ "Small thing to throw-ff936c7fde6cdd7b", "paper-cddf0363575a77bf", "knife-78f51ce9495d0bc5", "dirty plate-2fcaa07234d783ec", "bin lid-29369059ecdf8218", "another plate-214990ea27e29c82", "plate-c73b80b857dda267", "milk bottle-5cfb7518e14a1974", "glass-e1e5c13db45111a6", "scissors-5c52e5b121baf315", "brush-74776df3b4f98c37", "fork-6fa21391be43c692", "knife and fork-768aa635c0cb2387", "spoon-1c45c389db9047c3", "spatula-cf57787fc724fef1", "bowl-3c0f2e545387189d", "washing soup-acbb30a683107bfd", "plastic-5dae67113ef4ab31", "small bowl-e17f2059959db602", "grater-471f4305a01489f2", "box-115ed01fcd5adaa1", "red thing-de4868f3da0f5b24", "dirty cutlery-ff57979fc12181e1", "new fork-fde20363341988a5", "dirty cutlery 2-e697e6673ba34267", "dirty cutlery 4-cda04cfaa2728606", "no clue-7b6aa74be938d105", "no clue2-63307e0b6e11978d", "bowl3-3d9a8057cfd77698", "frying pan-602ae1d27ad60cd2", "food bin-2fcc7254de1cdc3f", "sponge-dd06dd3546c214a5", "paring knife-3c4389caa8961a89", "washing up brush-5c929c5eb60a0d23", "chef's knife-2a4212a9e675e83a", "bread knife-ac6b5668dfb756b0", "chopping board-30ce9207f6d26b7b", "remaining frying pan-32da07cf6e756f93", "lid of food bin-53008f884eb070b3", "new thing-063bef0e819e7cf1", "new fork 2-bb1ca842e7021484", "new knife-0dac74fb3bfba0f7", "tupperware lid-31f8ee3b8aaca4e9", "olive oil bottle-816c865ac4cdd983", "pepper mill-f662f67bc5bc2bb5", "pan support-3e2c8a56a6352636", "both grill covers-4f2680dee04bc05e", "plastic microwave cover-553f1dcc3bbab873", "second pan support-295d5dbcb869109e", "mat-8d314d89c4fb8691", "roll of drawer liners-48cb1485799139be", "other glass countertop protector-4bfc29e7deb6a9ab", "drain protector-2640ae3f60f2a571", "new glass-15c25cecf7ae49ec", "bottle of squash-b1eb2675d7ff6e6e" ], "reasoning": "1. I watched until I heard the user flick a light switch and saw the lighting in the room brighten at 00:28.\n2. I went back in the video and counted the number of plates the user handled from 00:00-00:28.\n3. I saw the user pick up the first plate (dirty plate-2fcaa07234d783ec) at 00:05, and then grab 2 plates from the dishrack at 00:19 for a total of 3 plates.", "referenced_object_masks": [ "dirty plate-2fcaa07234d783ec" ] }, { "key": "P02-20240209-194141:7fb1d9cece0964f5d487f24bb34255339e2b11ee", "video_id": "P02-20240209-194141", "question": "If I were to multiply the total number of cupboards and drawers in my kitchen by the number of hearts on the floor mat by the stove, and then divide it by the total number of shoes on my kitchen floor at any given time during the video, what number would I get?", "answer_choice_0": "37.5", "answer_choice_1": "75", "answer_choice_2": "60", "answer_choice_3": "30", "answer_choice_4": "82.5", "answer_id": 4, "answer": "82.5", "question_type": [ "Counting", "Egocentric & Spatial Perception", "Numerical Reasoning", "Object Recognition" ], "objects_of_interest": [ "Small thing to throw-ff936c7fde6cdd7b", "paper-cddf0363575a77bf", "knife-78f51ce9495d0bc5", "dirty plate-2fcaa07234d783ec", "bin lid-29369059ecdf8218", "another plate-214990ea27e29c82", "plate-c73b80b857dda267", "milk bottle-5cfb7518e14a1974", "glass-e1e5c13db45111a6", "scissors-5c52e5b121baf315", "brush-74776df3b4f98c37", "fork-6fa21391be43c692", "knife and fork-768aa635c0cb2387", "spoon-1c45c389db9047c3", "spatula-cf57787fc724fef1", "bowl-3c0f2e545387189d", "washing soup-acbb30a683107bfd", "plastic-5dae67113ef4ab31", "small bowl-e17f2059959db602", "grater-471f4305a01489f2", "box-115ed01fcd5adaa1", "red thing-de4868f3da0f5b24", "dirty cutlery-ff57979fc12181e1", "new fork-fde20363341988a5", "dirty cutlery 2-e697e6673ba34267", "dirty cutlery 4-cda04cfaa2728606", "no clue-7b6aa74be938d105", "no clue2-63307e0b6e11978d", "bowl3-3d9a8057cfd77698", "frying pan-602ae1d27ad60cd2", "food bin-2fcc7254de1cdc3f", "sponge-dd06dd3546c214a5", "paring knife-3c4389caa8961a89", "washing up brush-5c929c5eb60a0d23", "chef's knife-2a4212a9e675e83a", "bread knife-ac6b5668dfb756b0", "chopping board-30ce9207f6d26b7b", "remaining frying pan-32da07cf6e756f93", "lid of food bin-53008f884eb070b3", "new thing-063bef0e819e7cf1", "new fork 2-bb1ca842e7021484", "new knife-0dac74fb3bfba0f7", "tupperware lid-31f8ee3b8aaca4e9", "olive oil bottle-816c865ac4cdd983", "pepper mill-f662f67bc5bc2bb5", "pan support-3e2c8a56a6352636", "both grill covers-4f2680dee04bc05e", "plastic microwave cover-553f1dcc3bbab873", "second pan support-295d5dbcb869109e", "mat-8d314d89c4fb8691", "roll of drawer liners-48cb1485799139be", "other glass countertop protector-4bfc29e7deb6a9ab", "drain protector-2640ae3f60f2a571", "new glass-15c25cecf7ae49ec", "bottle of squash-b1eb2675d7ff6e6e" ], "reasoning": "1. I began by looking for a good view of the total number of kitchen cupboards and drawers, the hearts on the floor mat , and any and all shoes lying around the kitchen or on the user's feet.\n2. At 00:29 I got a clear view of most of the cupboards and drawers in the kitchen: 8 in all. At 15:18 there is a clear view of 3 additional drawers to the right of the dishwasher: 8+3=11.\n3. At 01:25 I could clearly see that the floor mat in question had 15 stars on it.\n4. At 01:12, 2 shoes are visible by the trash (food bin-2fcc7254de1cdc3f).\n5. At 00:12, it is clear that the user is not wearing any shoes, making the total number of shoes on the kitchen floor during the video 2.\n6. From here I calculated the product of the cupboards and hearts (11*15=165) and divided it by the number of shoes: 165/2=82.5.", "referenced_object_masks": [ "food bin-2fcc7254de1cdc3f" ] }, { "key": "P02-20240209-194141:95c4d8ac998d428e9b5a00ccd738ba556a913174", "video_id": "P02-20240209-194141", "question": "What's behind me when I pour a red liquid into a glass?", "answer_choice_0": "The dining table", "answer_choice_1": "The radiator", "answer_choice_2": "A trashcan", "answer_choice_3": "The sink", "answer_choice_4": "The toaster", "answer_id": 4, "answer": "The toaster", "question_type": [ "Egocentric & Spatial Perception", "Action Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "Small thing to throw-ff936c7fde6cdd7b", "paper-cddf0363575a77bf", "knife-78f51ce9495d0bc5", "dirty plate-2fcaa07234d783ec", "bin lid-29369059ecdf8218", "another plate-214990ea27e29c82", "plate-c73b80b857dda267", "milk bottle-5cfb7518e14a1974", "glass-e1e5c13db45111a6", "scissors-5c52e5b121baf315", "brush-74776df3b4f98c37", "fork-6fa21391be43c692", "knife and fork-768aa635c0cb2387", "spoon-1c45c389db9047c3", "spatula-cf57787fc724fef1", "bowl-3c0f2e545387189d", "washing soup-acbb30a683107bfd", "plastic-5dae67113ef4ab31", "small bowl-e17f2059959db602", "grater-471f4305a01489f2", "box-115ed01fcd5adaa1", "red thing-de4868f3da0f5b24", "dirty cutlery-ff57979fc12181e1", "new fork-fde20363341988a5", "dirty cutlery 2-e697e6673ba34267", "dirty cutlery 4-cda04cfaa2728606", "no clue-7b6aa74be938d105", "no clue2-63307e0b6e11978d", "bowl3-3d9a8057cfd77698", "frying pan-602ae1d27ad60cd2", "food bin-2fcc7254de1cdc3f", "sponge-dd06dd3546c214a5", "paring knife-3c4389caa8961a89", "washing up brush-5c929c5eb60a0d23", "chef's knife-2a4212a9e675e83a", "bread knife-ac6b5668dfb756b0", "chopping board-30ce9207f6d26b7b", "remaining frying pan-32da07cf6e756f93", "lid of food bin-53008f884eb070b3", "new thing-063bef0e819e7cf1", "new fork 2-bb1ca842e7021484", "new knife-0dac74fb3bfba0f7", "tupperware lid-31f8ee3b8aaca4e9", "olive oil bottle-816c865ac4cdd983", "pepper mill-f662f67bc5bc2bb5", "pan support-3e2c8a56a6352636", "both grill covers-4f2680dee04bc05e", "plastic microwave cover-553f1dcc3bbab873", "second pan support-295d5dbcb869109e", "mat-8d314d89c4fb8691", "roll of drawer liners-48cb1485799139be", "other glass countertop protector-4bfc29e7deb6a9ab", "drain protector-2640ae3f60f2a571", "new glass-15c25cecf7ae49ec", "bottle of squash-b1eb2675d7ff6e6e" ], "reasoning": "1. I watched until I saw the user pour a red liquid into a glass (new glass-15c25cecf7ae49ec) at 14:55.\n2. I noted that the user was standing at the counter between two cutting boards and facing the dining room when they were pouring.\n3. I went back in the video to 14:47 where I could see the user approaching the counter from the dining room.\n4. I looked behind the spot between the two cutting boards and saw the toaster directly behind on the counter on the opposite side of the kitchen.", "referenced_object_masks": [ "new glass-15c25cecf7ae49ec" ] }, { "key": "P02-20240209-194141:9eaa4a604f2c156b7f883444395c8b258f95b4f5", "video_id": "P02-20240209-194141", "question": "What individual pieces of silverware am I holding in my right hand at 00:44?", "answer_choice_0": "2 spoons, 1 fork and 2 knives", "answer_choice_1": "1 spoon, 2 forks and 2 knives", "answer_choice_2": "3 forks and 2 knives", "answer_choice_3": "2 spoons, 2 forks and 2 knives", "answer_choice_4": "2 forks and 3 knives", "answer_id": 1, "answer": "1 spoon, 2 forks and 2 knives", "question_type": [ "Action Recognition", "Counting", "Egocentric & Spatial Perception", "Object Recognition" ], "objects_of_interest": [ "Small thing to throw-ff936c7fde6cdd7b", "paper-cddf0363575a77bf", "knife-78f51ce9495d0bc5", "dirty plate-2fcaa07234d783ec", "bin lid-29369059ecdf8218", "another plate-214990ea27e29c82", "plate-c73b80b857dda267", "milk bottle-5cfb7518e14a1974", "glass-e1e5c13db45111a6", "scissors-5c52e5b121baf315", "brush-74776df3b4f98c37", "fork-6fa21391be43c692", "knife and fork-768aa635c0cb2387", "spoon-1c45c389db9047c3", "spatula-cf57787fc724fef1", "bowl-3c0f2e545387189d", "washing soup-acbb30a683107bfd", "plastic-5dae67113ef4ab31", "small bowl-e17f2059959db602", "grater-471f4305a01489f2", "box-115ed01fcd5adaa1", "red thing-de4868f3da0f5b24", "dirty cutlery-ff57979fc12181e1", "new fork-fde20363341988a5", "dirty cutlery 2-e697e6673ba34267", "dirty cutlery 4-cda04cfaa2728606", "no clue-7b6aa74be938d105", "no clue2-63307e0b6e11978d", "bowl3-3d9a8057cfd77698", "frying pan-602ae1d27ad60cd2", "food bin-2fcc7254de1cdc3f", "sponge-dd06dd3546c214a5", "paring knife-3c4389caa8961a89", "washing up brush-5c929c5eb60a0d23", "chef's knife-2a4212a9e675e83a", "bread knife-ac6b5668dfb756b0", "chopping board-30ce9207f6d26b7b", "remaining frying pan-32da07cf6e756f93", "lid of food bin-53008f884eb070b3", "new thing-063bef0e819e7cf1", "new fork 2-bb1ca842e7021484", "new knife-0dac74fb3bfba0f7", "tupperware lid-31f8ee3b8aaca4e9", "olive oil bottle-816c865ac4cdd983", "pepper mill-f662f67bc5bc2bb5", "pan support-3e2c8a56a6352636", "both grill covers-4f2680dee04bc05e", "plastic microwave cover-553f1dcc3bbab873", "second pan support-295d5dbcb869109e", "mat-8d314d89c4fb8691", "roll of drawer liners-48cb1485799139be", "other glass countertop protector-4bfc29e7deb6a9ab", "drain protector-2640ae3f60f2a571", "new glass-15c25cecf7ae49ec", "bottle of squash-b1eb2675d7ff6e6e" ], "reasoning": "1. I began by going to the specified timestamp to see which pieces of silverware the user was holding in his right hand. Since it wasn't clear, I continued to watch.\n2. At 00:47, the user begins to put said silverware away. At 00:48 the user puts 1 clean spoon (spoon-1c45c389db9047c3) in the drawer, followed by 1 clean fork (fork-6fa21391be43c692) at 00:50, 1 clean knife and 1 clean fork (knife and fork-768aa635c0cb2387) at 00:50 and finally 1 clean knife at 00:51. These are all the items that were in the user's hand at 00:44.\n3. I then counted each item and item type to get 1 spoon (spoon-1c45c389db9047c3), 2 forks (fork-6fa21391be43c692, knife and fork-768aa635c0cb2387) and 2 knives (knife and fork-768aa635c0cb2387).", "referenced_object_masks": [ "fork-6fa21391be43c692", "knife and fork-768aa635c0cb2387", "spoon-1c45c389db9047c3" ] }, { "key": "P02-20240209-194141:a1dde3642390e6e3fba3f7af77da783d5ee01480", "video_id": "P02-20240209-194141", "question": "How many unique red objects do I touch with my hands?", "answer_choice_0": "5", "answer_choice_1": "4", "answer_choice_2": "6", "answer_choice_3": "3", "answer_choice_4": "2", "answer_id": 3, "answer": "3", "question_type": [ "Counting", "Object Recognition", "Action Recognition", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "Small thing to throw-ff936c7fde6cdd7b", "paper-cddf0363575a77bf", "knife-78f51ce9495d0bc5", "dirty plate-2fcaa07234d783ec", "bin lid-29369059ecdf8218", "another plate-214990ea27e29c82", "plate-c73b80b857dda267", "milk bottle-5cfb7518e14a1974", "glass-e1e5c13db45111a6", "scissors-5c52e5b121baf315", "brush-74776df3b4f98c37", "fork-6fa21391be43c692", "knife and fork-768aa635c0cb2387", "spoon-1c45c389db9047c3", "spatula-cf57787fc724fef1", "bowl-3c0f2e545387189d", "washing soup-acbb30a683107bfd", "plastic-5dae67113ef4ab31", "small bowl-e17f2059959db602", "grater-471f4305a01489f2", "box-115ed01fcd5adaa1", "red thing-de4868f3da0f5b24", "dirty cutlery-ff57979fc12181e1", "new fork-fde20363341988a5", "dirty cutlery 2-e697e6673ba34267", "dirty cutlery 4-cda04cfaa2728606", "no clue-7b6aa74be938d105", "no clue2-63307e0b6e11978d", "bowl3-3d9a8057cfd77698", "frying pan-602ae1d27ad60cd2", "food bin-2fcc7254de1cdc3f", "sponge-dd06dd3546c214a5", "paring knife-3c4389caa8961a89", "washing up brush-5c929c5eb60a0d23", "chef's knife-2a4212a9e675e83a", "bread knife-ac6b5668dfb756b0", "chopping board-30ce9207f6d26b7b", "remaining frying pan-32da07cf6e756f93", "lid of food bin-53008f884eb070b3", "new thing-063bef0e819e7cf1", "new fork 2-bb1ca842e7021484", "new knife-0dac74fb3bfba0f7", "tupperware lid-31f8ee3b8aaca4e9", "olive oil bottle-816c865ac4cdd983", "pepper mill-f662f67bc5bc2bb5", "pan support-3e2c8a56a6352636", "both grill covers-4f2680dee04bc05e", "plastic microwave cover-553f1dcc3bbab873", "second pan support-295d5dbcb869109e", "mat-8d314d89c4fb8691", "roll of drawer liners-48cb1485799139be", "other glass countertop protector-4bfc29e7deb6a9ab", "drain protector-2640ae3f60f2a571", "new glass-15c25cecf7ae49ec", "bottle of squash-b1eb2675d7ff6e6e" ], "reasoning": "1. I watched the video to identify the red objects that the user touched with their hands.\n2. I saw the user pick up a red-handled knife (knife-78f51ce9495d0bc5) at 00:04.\n3. Then, I saw the user grab a red-handled dish brush (brush-74776df3b4f98c37) at 00:44.\n4. Next, the user grabs a red chip bag (red thing-de4868f3da0f5b24) at 01:27 for a total of 3 red objects.\n5. No other unique red objects are touched in the video.", "referenced_object_masks": [ "red thing-de4868f3da0f5b24", "knife-78f51ce9495d0bc5", "brush-74776df3b4f98c37" ] }, { "key": "P02-20240209-194141:b9f8598bd650b591cc3a907823c58309e9fd3fad", "video_id": "P02-20240209-194141", "question": "What is the fourth thing I touched with my right hand?", "answer_choice_0": "My sleeve", "answer_choice_1": "A plate", "answer_choice_2": "The trashcan", "answer_choice_3": "The faucet", "answer_choice_4": "A knife", "answer_id": 0, "answer": "My sleeve", "question_type": [ "Counting", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "Small thing to throw-ff936c7fde6cdd7b", "paper-cddf0363575a77bf", "knife-78f51ce9495d0bc5", "dirty plate-2fcaa07234d783ec", "bin lid-29369059ecdf8218", "another plate-214990ea27e29c82", "plate-c73b80b857dda267", "milk bottle-5cfb7518e14a1974", "glass-e1e5c13db45111a6", "scissors-5c52e5b121baf315", "brush-74776df3b4f98c37", "fork-6fa21391be43c692", "knife and fork-768aa635c0cb2387", "spoon-1c45c389db9047c3", "spatula-cf57787fc724fef1", "bowl-3c0f2e545387189d", "washing soup-acbb30a683107bfd", "plastic-5dae67113ef4ab31", "small bowl-e17f2059959db602", "grater-471f4305a01489f2", "box-115ed01fcd5adaa1", "red thing-de4868f3da0f5b24", "dirty cutlery-ff57979fc12181e1", "new fork-fde20363341988a5", "dirty cutlery 2-e697e6673ba34267", "dirty cutlery 4-cda04cfaa2728606", "no clue-7b6aa74be938d105", "no clue2-63307e0b6e11978d", "bowl3-3d9a8057cfd77698", "frying pan-602ae1d27ad60cd2", "food bin-2fcc7254de1cdc3f", "sponge-dd06dd3546c214a5", "paring knife-3c4389caa8961a89", "washing up brush-5c929c5eb60a0d23", "chef's knife-2a4212a9e675e83a", "bread knife-ac6b5668dfb756b0", "chopping board-30ce9207f6d26b7b", "remaining frying pan-32da07cf6e756f93", "lid of food bin-53008f884eb070b3", "new thing-063bef0e819e7cf1", "new fork 2-bb1ca842e7021484", "new knife-0dac74fb3bfba0f7", "tupperware lid-31f8ee3b8aaca4e9", "olive oil bottle-816c865ac4cdd983", "pepper mill-f662f67bc5bc2bb5", "pan support-3e2c8a56a6352636", "both grill covers-4f2680dee04bc05e", "plastic microwave cover-553f1dcc3bbab873", "second pan support-295d5dbcb869109e", "mat-8d314d89c4fb8691", "roll of drawer liners-48cb1485799139be", "other glass countertop protector-4bfc29e7deb6a9ab", "drain protector-2640ae3f60f2a571", "new glass-15c25cecf7ae49ec", "bottle of squash-b1eb2675d7ff6e6e" ], "reasoning": "1. I watched the video and counted the things the user touched with their right hand.\n2. I saw the user pick up some plastic (Small thing to throw-ff936c7fde6cdd7b) with their right hand at 00:03, then grab a red-handled knife (knife-78f51ce9495d0bc5) and a plate (dirty plate-2fcaa07234d783ec) with their right hand at 00:05, and then pull up their left sleeve at 00:18.\n3. Therefore, the fourth thing the user touched with their right hand is their sleeve.", "referenced_object_masks": [ "knife-78f51ce9495d0bc5", "Small thing to throw-ff936c7fde6cdd7b", "dirty plate-2fcaa07234d783ec" ] }, { "key": "P02-20240209-194141:dc716e4aeaa2ba44eed64f577ad8d13c74d6b125", "video_id": "P02-20240209-194141", "question": "Before I do the dishes, what is the ratio of the number of clean items I put away and the number of bowls to the left of the sink?", "answer_choice_0": "14:1", "answer_choice_1": "6:1", "answer_choice_2": "7:1", "answer_choice_3": "11:2", "answer_choice_4": "13:2", "answer_id": 3, "answer": "11:2", "question_type": [ "Action Recognition", "Counting", "Numerical Reasoning", "Object Recognition", "State Changes" ], "objects_of_interest": [ "Small thing to throw-ff936c7fde6cdd7b", "paper-cddf0363575a77bf", "knife-78f51ce9495d0bc5", "dirty plate-2fcaa07234d783ec", "bin lid-29369059ecdf8218", "another plate-214990ea27e29c82", "plate-c73b80b857dda267", "milk bottle-5cfb7518e14a1974", "glass-e1e5c13db45111a6", "scissors-5c52e5b121baf315", "brush-74776df3b4f98c37", "fork-6fa21391be43c692", "knife and fork-768aa635c0cb2387", "spoon-1c45c389db9047c3", "spatula-cf57787fc724fef1", "bowl-3c0f2e545387189d", "washing soup-acbb30a683107bfd", "plastic-5dae67113ef4ab31", "small bowl-e17f2059959db602", "grater-471f4305a01489f2", "box-115ed01fcd5adaa1", "red thing-de4868f3da0f5b24", "dirty cutlery-ff57979fc12181e1", "new fork-fde20363341988a5", "dirty cutlery 2-e697e6673ba34267", "dirty cutlery 4-cda04cfaa2728606", "no clue-7b6aa74be938d105", "no clue2-63307e0b6e11978d", "bowl3-3d9a8057cfd77698", "frying pan-602ae1d27ad60cd2", "food bin-2fcc7254de1cdc3f", "sponge-dd06dd3546c214a5", "paring knife-3c4389caa8961a89", "washing up brush-5c929c5eb60a0d23", "chef's knife-2a4212a9e675e83a", "bread knife-ac6b5668dfb756b0", "chopping board-30ce9207f6d26b7b", "remaining frying pan-32da07cf6e756f93", "lid of food bin-53008f884eb070b3", "new thing-063bef0e819e7cf1", "new fork 2-bb1ca842e7021484", "new knife-0dac74fb3bfba0f7", "tupperware lid-31f8ee3b8aaca4e9", "olive oil bottle-816c865ac4cdd983", "pepper mill-f662f67bc5bc2bb5", "pan support-3e2c8a56a6352636", "both grill covers-4f2680dee04bc05e", "plastic microwave cover-553f1dcc3bbab873", "second pan support-295d5dbcb869109e", "mat-8d314d89c4fb8691", "roll of drawer liners-48cb1485799139be", "other glass countertop protector-4bfc29e7deb6a9ab", "drain protector-2640ae3f60f2a571", "new glass-15c25cecf7ae49ec", "bottle of squash-b1eb2675d7ff6e6e" ], "reasoning": "1. I watched the video waiting for the user both to put clean items away and do the dishes. Everything before the dish washing would give me my answer.\n2. At 00:19, the user picks up plates that have been cleaned and left out to dry. I began to count the clean items that the user put away at this point: 2 plates (another plate-214990ea27e29c82, plate-c73b80b857dda267) at 00:21, 1 clean glass (glass-e1e5c13db45111a6) at 00:37, 1 clean spoon (spoon-1c45c389db9047c3) at 00:48, 1 clean fork (fork-6fa21391be43c692) at 00:50, 1 clean knife and 1 clean fork (knife and fork-768aa635c0cb2387) at 00:50, 1 clean knife at 00:51 and 1 cleaned pair of scissors (scissors-5c52e5b121baf315) at 00:51: 9 clean items in all.\n3. At 01:00 the user begins to get the sink ready to wash the dishes. There are 2 bowls (bowl3-3d9a80 and OBJ) visible to the left of the sink.\n4. Before doing the dishes, the user proceeds to put back a couple more cleaned items: 1 grater (grater-471f4305a01489f2) at 01:17 and 1 bowl (small bowl-e17f2059959db602) at 01:22: 2 more items.\n5. At 01:35 the user begins to do the dishes by loading the dirty items into the wash bin.\n6. I calculated the total number of cleaned items returned 9+2=11 to find the ratio of returned items to bowls to the left of the sink: 11:2.", "referenced_object_masks": [ "fork-6fa21391be43c692", "knife and fork-768aa635c0cb2387", "small bowl-e17f2059959db602", "scissors-5c52e5b121baf315", "another plate-214990ea27e29c82", "plate-c73b80b857dda267", "grater-471f4305a01489f2", "glass-e1e5c13db45111a6", "spoon-1c45c389db9047c3" ] }, { "key": "P02-20240209-194141:f280d952423950e0ef1a24911f3ca606e26f9d88", "video_id": "P02-20240209-194141", "question": "What's the difference between the number of times I pick up the food waste bin and the number of stove grates?", "answer_choice_0": "6", "answer_choice_1": "5", "answer_choice_2": "2", "answer_choice_3": "4", "answer_choice_4": "3", "answer_id": 1, "answer": "5", "question_type": [ "Counting", "Action Recognition", "Object Recognition", "Egocentric & Spatial Perception", "Numerical Reasoning" ], "objects_of_interest": [ "Small thing to throw-ff936c7fde6cdd7b", "paper-cddf0363575a77bf", "knife-78f51ce9495d0bc5", "dirty plate-2fcaa07234d783ec", "bin lid-29369059ecdf8218", "another plate-214990ea27e29c82", "plate-c73b80b857dda267", "milk bottle-5cfb7518e14a1974", "glass-e1e5c13db45111a6", "scissors-5c52e5b121baf315", "brush-74776df3b4f98c37", "fork-6fa21391be43c692", "knife and fork-768aa635c0cb2387", "spoon-1c45c389db9047c3", "spatula-cf57787fc724fef1", "bowl-3c0f2e545387189d", "washing soup-acbb30a683107bfd", "plastic-5dae67113ef4ab31", "small bowl-e17f2059959db602", "grater-471f4305a01489f2", "box-115ed01fcd5adaa1", "red thing-de4868f3da0f5b24", "dirty cutlery-ff57979fc12181e1", "new fork-fde20363341988a5", "dirty cutlery 2-e697e6673ba34267", "dirty cutlery 4-cda04cfaa2728606", "no clue-7b6aa74be938d105", "no clue2-63307e0b6e11978d", "bowl3-3d9a8057cfd77698", "frying pan-602ae1d27ad60cd2", "food bin-2fcc7254de1cdc3f", "sponge-dd06dd3546c214a5", "paring knife-3c4389caa8961a89", "washing up brush-5c929c5eb60a0d23", "chef's knife-2a4212a9e675e83a", "bread knife-ac6b5668dfb756b0", "chopping board-30ce9207f6d26b7b", "remaining frying pan-32da07cf6e756f93", "lid of food bin-53008f884eb070b3", "new thing-063bef0e819e7cf1", "new fork 2-bb1ca842e7021484", "new knife-0dac74fb3bfba0f7", "tupperware lid-31f8ee3b8aaca4e9", "olive oil bottle-816c865ac4cdd983", "pepper mill-f662f67bc5bc2bb5", "pan support-3e2c8a56a6352636", "both grill covers-4f2680dee04bc05e", "plastic microwave cover-553f1dcc3bbab873", "second pan support-295d5dbcb869109e", "mat-8d314d89c4fb8691", "roll of drawer liners-48cb1485799139be", "other glass countertop protector-4bfc29e7deb6a9ab", "drain protector-2640ae3f60f2a571", "new glass-15c25cecf7ae49ec", "bottle of squash-b1eb2675d7ff6e6e" ], "reasoning": "1. I watched the video and counted the number of times the user picked up the food waste bin (food bin-2fcc7254de1cdc3f). I saw them pick it up at 02:05, 05:46, 07:27, 07:54, 09:13, 12:27, and 14:40 for a total of 7 times.\n2. I watched the video until I saw the user lift the stove grates so I could be sure of how many there were. I saw them lift them up from the stove in 2 distinct pieces at 08:22.\n3. I subtracted 2 from 7 for a result of 5.", "referenced_object_masks": [ "food bin-2fcc7254de1cdc3f" ] }, { "key": "P02-20240209-194141:faa1e3c929263baaf4885ec1779977b3eb55b5f4", "video_id": "P02-20240209-194141", "question": "Where and in what order do I place the compost bin in relation to the stove throughout the video?", "answer_choice_0": "across from it, to the right of it, to the left of it", "answer_choice_1": "to the right of it, across from it, to the left of it", "answer_choice_2": "across from it, to the left of it, to the right of it", "answer_choice_3": "to the left of it, to the right of it, across from it", "answer_choice_4": "to the right of it, to the left of it, across from it", "answer_id": 2, "answer": "across from it, to the left of it, to the right of it", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Object Recognition", "State Changes", "Temporal Reasoning", "Event Occurence", "Situational Awareness" ], "objects_of_interest": [ "Small thing to throw-ff936c7fde6cdd7b", "paper-cddf0363575a77bf", "knife-78f51ce9495d0bc5", "dirty plate-2fcaa07234d783ec", "bin lid-29369059ecdf8218", "another plate-214990ea27e29c82", "plate-c73b80b857dda267", "milk bottle-5cfb7518e14a1974", "glass-e1e5c13db45111a6", "scissors-5c52e5b121baf315", "brush-74776df3b4f98c37", "fork-6fa21391be43c692", "knife and fork-768aa635c0cb2387", "spoon-1c45c389db9047c3", "spatula-cf57787fc724fef1", "bowl-3c0f2e545387189d", "washing soup-acbb30a683107bfd", "plastic-5dae67113ef4ab31", "small bowl-e17f2059959db602", "grater-471f4305a01489f2", "box-115ed01fcd5adaa1", "red thing-de4868f3da0f5b24", "dirty cutlery-ff57979fc12181e1", "new fork-fde20363341988a5", "dirty cutlery 2-e697e6673ba34267", "dirty cutlery 4-cda04cfaa2728606", "no clue-7b6aa74be938d105", "no clue2-63307e0b6e11978d", "bowl3-3d9a8057cfd77698", "frying pan-602ae1d27ad60cd2", "food bin-2fcc7254de1cdc3f", "sponge-dd06dd3546c214a5", "paring knife-3c4389caa8961a89", "washing up brush-5c929c5eb60a0d23", "chef's knife-2a4212a9e675e83a", "bread knife-ac6b5668dfb756b0", "chopping board-30ce9207f6d26b7b", "remaining frying pan-32da07cf6e756f93", "lid of food bin-53008f884eb070b3", "new thing-063bef0e819e7cf1", "new fork 2-bb1ca842e7021484", "new knife-0dac74fb3bfba0f7", "tupperware lid-31f8ee3b8aaca4e9", "olive oil bottle-816c865ac4cdd983", "pepper mill-f662f67bc5bc2bb5", "pan support-3e2c8a56a6352636", "both grill covers-4f2680dee04bc05e", "plastic microwave cover-553f1dcc3bbab873", "second pan support-295d5dbcb869109e", "mat-8d314d89c4fb8691", "roll of drawer liners-48cb1485799139be", "other glass countertop protector-4bfc29e7deb6a9ab", "drain protector-2640ae3f60f2a571", "new glass-15c25cecf7ae49ec", "bottle of squash-b1eb2675d7ff6e6e" ], "reasoning": "1. I began by looking for when the user handles the compost bin (food bin-2fcc7254de1cdc3f). The user picks it up for the first time at 02:07.\n2. From here, I took note of where the user placed the compost bin (food bin-2fcc7254de1cdc3f) down in relation to the stove: first across from it at 02:09; then to the left of it at 07:51, 08:19 and 11:57; finally to the right of it at 12:30. At 14:42 the user returns the compost bin (food bin-2fcc7254de1cdc3f) to its original location on the floor.", "referenced_object_masks": [ "food bin-2fcc7254de1cdc3f" ] }, { "key": "P02-20240209-231412:0ad4b5bb8742fb767bd65d0d4c2e77656c9063dd", "video_id": "P02-20240209-231412", "question": "How many times does the glass leave my right hand?", "answer_choice_0": "Two", "answer_choice_1": "Four", "answer_choice_2": "Five", "answer_choice_3": "Three", "answer_choice_4": "One", "answer_id": 3, "answer": "Three", "question_type": [ "Temporal Reasoning", "Counting", "Egocentric & Spatial Perception", "Action Recognition", "Object Recognition" ], "objects_of_interest": [ "empty glass-7e5b05423962f2f0", "bottle-0c4f03a25172ec15" ], "reasoning": "1. I locate where the glass (empty glass-7e5b05423962f2f0) appears in the user's right hand at 00:00.\n2. I identify when the user places the glass (empty glass-7e5b05423962f2f0) down on the counter with their right hand at 0:04.\n3. I identify when the user picks up the glass (empty glass-7e5b05423962f2f0) with their right hand at 00:17.\n4. I identify when the user transfers the glass (empty glass-7e5b05423962f2f0) from their right hand to their left hand at 00:18.\n5. I identify when the user transfers the glass (empty glass-7e5b05423962f2f0) from their left hand to their right hand at 00:27.\n6. I identify when the user places the glass (empty glass-7e5b05423962f2f0) down on the counter at 00:30.", "referenced_object_masks": [ "empty glass-7e5b05423962f2f0" ] }, { "key": "P02-20240209-231412:3dfd6d54250de327bb1c2fb8575ca7763daefa06", "video_id": "P02-20240209-231412", "question": "Where is the wooden door with glass panels located in relation to me when I walk into the kitchen area?", "answer_choice_0": "Ahead and to the right", "answer_choice_1": "Behind me", "answer_choice_2": "To the right off-screen", "answer_choice_3": "Ahead and to the left", "answer_choice_4": "To the left off-screen", "answer_id": 1, "answer": "Behind me", "question_type": [ "Temporal Reasoning", "Object Recognition", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "empty glass-7e5b05423962f2f0", "bottle-0c4f03a25172ec15" ], "reasoning": "1. I identify when the user first enters the kitchen area at 00:01.\n2. I locate the wooden door with glass panels at 00:05.\n3. I determine that the wooden door with glass panels is nearest to the far right corner of the counter.\n4. I retrace the user's steps to determine that they walked past the nearest corner en route to the kitchen area.", "referenced_object_masks": [] }, { "key": "P02-20240209-231412:620831cd458c68d841e2592a20e70d5569856f20", "video_id": "P02-20240209-231412", "question": "If I subtracted the number of times I splash juice into the glass from the number of vertical bars on the back of the wooden chair, what number would I have?", "answer_choice_0": "One", "answer_choice_1": "Four", "answer_choice_2": "Three", "answer_choice_3": "Five", "answer_choice_4": "Two", "answer_id": 2, "answer": "Three", "question_type": [ "Temporal Reasoning", "Object Recognition", "Numerical Reasoning", "Counting", "Action Recognition", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "empty glass-7e5b05423962f2f0", "bottle-0c4f03a25172ec15" ], "reasoning": "1. I locate when the back of the chair comes into clear view at 00:03.\n2. I count the five vertical bars on the back of the chair .\n3. I identify when the user splashes juice into the glass (empty glass-7e5b05423962f2f0) from 00:09 to 00:10.\n4. I count the two splashes of juice the user pours into the glass (empty glass-7e5b05423962f2f0).\n5. I subtracted the 2 splashes from the 5 bars to get 3.", "referenced_object_masks": [ "empty glass-7e5b05423962f2f0" ] }, { "key": "P02-20240209-231412:997292ef5b3c6add2bb7c08dfb43aa3adfb25d78", "video_id": "P02-20240209-231412", "question": "Where is the microwave in relation to the glass when I first place it on the counter?", "answer_choice_0": "Behind it", "answer_choice_1": "Above it", "answer_choice_2": "To the right of it", "answer_choice_3": "In front of it", "answer_choice_4": "To the left of it", "answer_id": 4, "answer": "To the left of it", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "empty glass-7e5b05423962f2f0", "bottle-0c4f03a25172ec15" ], "reasoning": "1. I locate the microwave , which first appears at 00:00 on the far side of the room.\n2. I locate where the glass (empty glass-7e5b05423962f2f0) appears at 00:00 in the user's right hand.\n3. I determine the timestamp for when the user places the glass (empty glass-7e5b05423962f2f0) down on the counter for the first time (0:04)\n4. I identify the microwave 's location in relation to the glass (empty glass-7e5b05423962f2f0).", "referenced_object_masks": [ "empty glass-7e5b05423962f2f0" ] }, { "key": "P02-20240209-231412:fdbca40b05b39c32f9fd50f32ca7aeecc20a68b9", "video_id": "P02-20240209-231412", "question": "What number would I have if I subtracted the number of rectangular boards sitting on the kitchen counter from the current time if I rounded to the nearest hour using a 12-hour clock?", "answer_choice_0": "7", "answer_choice_1": "8", "answer_choice_2": "10", "answer_choice_3": "9", "answer_choice_4": "11", "answer_id": 3, "answer": "9", "question_type": [ "Counting", "Object Recognition", "Temporal Reasoning", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "empty glass-7e5b05423962f2f0", "bottle-0c4f03a25172ec15" ], "reasoning": "1. I locate two rectangular boards placed on the kitchen counter at 00:00.\n2. I locate the wall clock at 00:27.\n3. I determine the wall clock shows a time of about 11:15.\n4. I determine that the nearest hour is 11:00.\n5. I subtracted the 2 rectangular boards from the 11 hours to get 9.", "referenced_object_masks": [] }, { "key": "P02-20240210-092616:458230143e9490608d56633779a35d22aebf9007", "video_id": "P02-20240210-092616", "question": "If I were facing the kitchen closet door, approximately where would the patio door be in relation to me?", "answer_choice_0": "90 degrees counterclockwise", "answer_choice_1": "90 degrees clockwise", "answer_choice_2": "180 degrees clockwise", "answer_choice_3": "225 degrees clockwise", "answer_choice_4": "225 degrees counterclockwise", "answer_id": 3, "answer": "225 degrees clockwise", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness", "State Changes" ], "objects_of_interest": [ "carry bag-b76f5233ba4b70af", "second carrier bag-2b10ccb50b76509b", "folded carrier bag-eb645a2a25cfe630" ], "reasoning": "1. When the user first walks into the kitchen at the start of the video, the patio door is visible until 00:01 on the user's right as they walk to the left toward the kitchen.\n2. At 00:03 the user opens the kitchen closet door at an angle.\n3. If the user were directly facing the closet door, they would have to turn either 135 degrees counterclockwise or 225 degrees clockwise to face it from where they were standing.\n4. Therefore, the answer is approximately 225 degrees clockwise.", "referenced_object_masks": [] }, { "key": "P02-20240210-092616:533a7c7aed3262faf8422418cb4381f79cfe19ab", "video_id": "P02-20240210-092616", "question": "Where is the chair in relation to me when I start folding the second bag?", "answer_choice_0": "On my right", "answer_choice_1": "I'm sitting on it", "answer_choice_2": "Directly in front of me", "answer_choice_3": "On my left", "answer_choice_4": "Directly behind me", "answer_id": 0, "answer": "On my right", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "carry bag-b76f5233ba4b70af", "second carrier bag-2b10ccb50b76509b", "folded carrier bag-eb645a2a25cfe630" ], "reasoning": "1. First, I determine when the user picks up the second bag (folded carrier bag-eb645a2a25cfe630). This occurs from 00:28-00:29. \n2. The user grabs the bag (folded carrier bag-eb645a2a25cfe630) and turns to the left before starting to fold the bag (folded carrier bag-eb645a2a25cfe630) at 00:36.\n3. At the time the user firts picks up the bag (folded carrier bag-eb645a2a25cfe630) the user is facing the counter with a view of the dining room. The chair is visible across the counter on the right. \n4. Therefore, the chair is positioned on the user's right after they turn to the left and start folding the bag (folded carrier bag-eb645a2a25cfe630).", "referenced_object_masks": [ "folded carrier bag-eb645a2a25cfe630" ] }, { "key": "P02-20240210-092616:6164db6b2a43325c2287be95bc24def6d28e5087", "video_id": "P02-20240210-092616", "question": "Is there an appliance that is to the right of me when I fold my second bag compared to when I fold my first bag?", "answer_choice_0": "Yes, the microwave", "answer_choice_1": "Yes, the toaster", "answer_choice_2": "Yes, the stove", "answer_choice_3": "No, the positions are the same both times", "answer_choice_4": "Yes, the refrigerator", "answer_id": 0, "answer": "Yes, the microwave", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness", "State Changes" ], "objects_of_interest": [ "carry bag-b76f5233ba4b70af", "second carrier bag-2b10ccb50b76509b", "folded carrier bag-eb645a2a25cfe630" ], "reasoning": "1. The user folds the first bag (second carrier bag-2b10ccb50b76509b) from 00:13-00:27 with a refrigerator on their right, visible just under the user's right hand. \n2. When the user is folding the second bag, (folded carrier bag-eb645a2a25cfe630), starting from 00:31-00:42, they turn and face the washing machine. Now the microwave and the refrigerator are on their right side. \n3. Therefore the answer is yes, there is another appliance on the user's right when they fold the second bag compared to the first -- namely, the microwave .", "referenced_object_masks": [ "folded carrier bag-eb645a2a25cfe630", "second carrier bag-2b10ccb50b76509b" ] }, { "key": "P02-20240210-092616:9dc90f4777a59d767a0a44ceaf194c77fdba26b2", "video_id": "P02-20240210-092616", "question": "How many bags would I have to fold in total if I pulled 4 more from the main storage bag?", "answer_choice_0": "7", "answer_choice_1": "5", "answer_choice_2": "3", "answer_choice_3": "2", "answer_choice_4": "6", "answer_id": 4, "answer": "6", "question_type": [ "Counterfactual", "Counting", "Object Recognition", "Numerical Reasoning" ], "objects_of_interest": [ "carry bag-b76f5233ba4b70af", "second carrier bag-2b10ccb50b76509b", "folded carrier bag-eb645a2a25cfe630" ], "reasoning": "1. First, I determine how many bags the user pulls from the main storage bag (carry bag-b76f5233ba4b70af). I determine at 00:04, the user pulls 2 bags .\n2. Next, I determine the total amount of bags the user would have pulled if 4 more were added. By adding 4+2, the total amount would be 6. \n3. Therefore, the user would have had to fold 6 bags in total.", "referenced_object_masks": [ "carry bag-b76f5233ba4b70af" ] }, { "key": "P02-20240210-092616:b429cdaffc2c64f7ff6cb4dd3c090fa38a05b7dc", "video_id": "P02-20240210-092616", "question": "How many times do I look away from the chair after folding the first bag?", "answer_choice_0": "2 times", "answer_choice_1": "3 times", "answer_choice_2": "4 times", "answer_choice_3": "I never look away", "answer_choice_4": "1 time", "answer_id": 4, "answer": "1 time", "question_type": [ "Counting", "Event Occurence", "Situational Awareness", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "carry bag-b76f5233ba4b70af", "second carrier bag-2b10ccb50b76509b", "folded carrier bag-eb645a2a25cfe630" ], "reasoning": "1. First, I determine the time span when the user folds the first bag (second carrier bag-2b10ccb50b76509b). This occurs from 00:13-00:28.\n2. The user looks away from the chair at 00:30 and remains looking away for the remainder of the video. \n3. Therefore, the number of times the user looks away from the chair after folding the first bag (second carrier bag-2b10ccb50b76509b) is once.", "referenced_object_masks": [ "second carrier bag-2b10ccb50b76509b" ] }, { "key": "P02-20240210-100308:0496c8ca56a10462f859e9f20a7b8f2cf94cb076", "video_id": "P02-20240210-100308", "question": "How long from when I set the bags down on the counter does it take me to place an item into the fridge?", "answer_choice_0": "1 minute and 50 seconds", "answer_choice_1": "2 minutes and 35 seconds", "answer_choice_2": "3 minutes and 1 second", "answer_choice_3": "2 minutes and 53 seconds", "answer_choice_4": "2 minutes and 11 seconds", "answer_id": 4, "answer": "2 minutes and 11 seconds", "question_type": [ "Action Recognition", "Object Recognition", "Numerical Reasoning", "Temporal Reasoning", "Situational Awareness" ], "objects_of_interest": [ "bag1-80c78fea043044b8", "bag2-29acd80a7ccef27a", "bag3-f3047b10fb20748b", "raspberries-82a409ae73bef8ee", "eggs-9936026394f31f50", "apple turnover-ec11c098f3ff4ba6", "mushrooms-c59a38624beeb91c", "kebab wraps-64bed98c84d186ea", "raspberries2-5fa60bc46cc39158", "soap-83c308e9d8a1af07", "bag of tortilla chips-d6e22cf0fa353f6b", "bag of lemons-e7208f76d8b47300", "butter-3a6f7c2e808de8ce", "tomatoes-dd8c184cfd5bff98", "roll of pastry-05693451adcb0726", "pasta sauce-c9aa28d612d29561", "baguette1-eeaa69cc9978167c", "baguette2-3ea922d8e0c31d99", "bag4-0795261cf6dfe828", "red onions-5326f8c248026991", "apples-adb7547b50854259", "new bag-f6d7e131622e4b8f", "food bin-1b84b0b4be429fd3", "tea-1e7d0923f2429369", "bag of grated cheese-a08827826c65edc3", "roll of pastry2-edbec4dc47961d0a", "milk-61bd734b8fdc0c67", "bag5-ddcf85728b77cb92", "right hand vegetable drawer-cd322e857a6d8bea", "bag of food-64eed06757c837d1", "left hand vegetable drawer-f7e84070c89ff0a6", "bag of food2-f5f098ad977ee22e", "tin of soup-1dbddf59c8ffc413" ], "reasoning": "1. I watched the video and looked for the moment when the user placed bags down on the counter. I found this occurred simultaneously at 00:07, when the user placed the first bag (bag1-80c78fea043044b8), the second bag (bag2-29acd80a7ccef27a), the third bag (bag3-f3047b10fb20748b), and the fourth bag (bag4-0795261cf6dfe828) down all at once.\n2. Then, I looked for the time when the user opened a fridge. This occurred at 02:14, when the user opened a fridge door .\n3. Then, I looked for when the user placed something inside of it. I found this just a few seconds later at 02:18, when the user placed the butter (butter-3a6f7c2e808de8ce) on a fridge shelf . \n4. Knowing the two target times are 02:18 and 00:07, I subtracted the two and got 02:11, making 2 minutes and 11 seconds the final answer.", "referenced_object_masks": [ "bag2-29acd80a7ccef27a", "bag1-80c78fea043044b8", "butter-3a6f7c2e808de8ce", "bag4-0795261cf6dfe828", "bag3-f3047b10fb20748b" ] }, { "key": "P02-20240210-100308:539d6f3b06590cfc29230ff31f6378ffa647acf7", "video_id": "P02-20240210-100308", "question": "If there had been 5 additional items inside the first 2 grocery bags I unpacked, how many grocery items would there be on the counter after unpacking these 2 bags?", "answer_choice_0": "14", "answer_choice_1": "20", "answer_choice_2": "12", "answer_choice_3": "9", "answer_choice_4": "18", "answer_id": 4, "answer": "18", "question_type": [ "Action Recognition", "Object Recognition", "Numerical Reasoning", "Counterfactual" ], "objects_of_interest": [ "bag1-80c78fea043044b8", "bag2-29acd80a7ccef27a", "bag3-f3047b10fb20748b", "raspberries-82a409ae73bef8ee", "eggs-9936026394f31f50", "apple turnover-ec11c098f3ff4ba6", "mushrooms-c59a38624beeb91c", "kebab wraps-64bed98c84d186ea", "raspberries2-5fa60bc46cc39158", "soap-83c308e9d8a1af07", "bag of tortilla chips-d6e22cf0fa353f6b", "bag of lemons-e7208f76d8b47300", "butter-3a6f7c2e808de8ce", "tomatoes-dd8c184cfd5bff98", "roll of pastry-05693451adcb0726", "pasta sauce-c9aa28d612d29561", "baguette1-eeaa69cc9978167c", "baguette2-3ea922d8e0c31d99", "bag4-0795261cf6dfe828", "red onions-5326f8c248026991", "apples-adb7547b50854259", "new bag-f6d7e131622e4b8f", "food bin-1b84b0b4be429fd3", "tea-1e7d0923f2429369", "bag of grated cheese-a08827826c65edc3", "roll of pastry2-edbec4dc47961d0a", "milk-61bd734b8fdc0c67", "bag5-ddcf85728b77cb92", "right hand vegetable drawer-cd322e857a6d8bea", "bag of food-64eed06757c837d1", "left hand vegetable drawer-f7e84070c89ff0a6", "bag of food2-f5f098ad977ee22e", "tin of soup-1dbddf59c8ffc413" ], "reasoning": "1. I watched the video and looked for when the user first started to remove items from the first 2 grocery bags. I found this occurred at 00:08, when the user reached their hand into one bag (bag3-f3047b10fb20748b).\n2. At 00:08, the user pulled a package of raspberries (raspberries-82a409ae73bef8ee) out of the bag. At 00:10, the user pulled out eggs (eggs-9936026394f31f50). At 00:13, the user pulled out an apple turnover (apple turnover-ec11c098f3ff4ba6). At 00:16, the user pulled out mushrooms (mushrooms-c59a38624beeb91c). There was nothing else in this bag (bag3-f3047b10fb20748b), so that totals 4 items so far.\n3. At 00:31, the user begins to pull items out of the 2nd bag (bag2-29acd80a7ccef27a). First, they take out a package of kebab wraps (kebab wraps-64bed98c84d186ea). Following this, they pull out another package of raspberries (raspberries2-5fa60bc46cc39158) at 00:33, soap (soap-83c308e9d8a1af07) at 00:35, a bag of tortilla chips (bag of tortilla chips-d6e22cf0fa353f6b) at 00:37, a bag of lemons (bag of lemons-e7208f76d8b47300) at 00:39, butter (butter-3a6f7c2e808de8ce) at 00:41, tomatoes (tomatoes-dd8c184cfd5bff98) at 00:43, a roll of pastry (roll of pastry-05693451adcb0726) at 00:45, and finally pasta sauce (pasta sauce-c9aa28d612d29561) at 00:48. This totals 9 items from this bag (bag2-29acd80a7ccef27a).\n4. Adding all of these up, I counted 13 items. If there were 5 additional items, that would result in 18, making that the answer.", "referenced_object_masks": [ "apple turnover-ec11c098f3ff4ba6", "raspberries-82a409ae73bef8ee", "raspberries2-5fa60bc46cc39158", "pasta sauce-c9aa28d612d29561", "bag2-29acd80a7ccef27a", "bag of tortilla chips-d6e22cf0fa353f6b", "eggs-9936026394f31f50", "mushrooms-c59a38624beeb91c", "kebab wraps-64bed98c84d186ea", "bag of lemons-e7208f76d8b47300", "soap-83c308e9d8a1af07", "bag3-f3047b10fb20748b", "tomatoes-dd8c184cfd5bff98", "butter-3a6f7c2e808de8ce", "roll of pastry-05693451adcb0726" ] }, { "key": "P02-20240210-100308:7081c304c4593037fdbc7c5388f4d9dc1aa0801a", "video_id": "P02-20240210-100308", "question": "Where is my food bin located in relation to my fridge?", "answer_choice_0": "To the right of it", "answer_choice_1": "Behind it", "answer_choice_2": "On top of it", "answer_choice_3": "To the left of it", "answer_choice_4": "In front of it", "answer_id": 0, "answer": "To the right of it", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "bag1-80c78fea043044b8", "bag2-29acd80a7ccef27a", "bag3-f3047b10fb20748b", "raspberries-82a409ae73bef8ee", "eggs-9936026394f31f50", "apple turnover-ec11c098f3ff4ba6", "mushrooms-c59a38624beeb91c", "kebab wraps-64bed98c84d186ea", "raspberries2-5fa60bc46cc39158", "soap-83c308e9d8a1af07", "bag of tortilla chips-d6e22cf0fa353f6b", "bag of lemons-e7208f76d8b47300", "butter-3a6f7c2e808de8ce", "tomatoes-dd8c184cfd5bff98", "roll of pastry-05693451adcb0726", "pasta sauce-c9aa28d612d29561", "baguette1-eeaa69cc9978167c", "baguette2-3ea922d8e0c31d99", "bag4-0795261cf6dfe828", "red onions-5326f8c248026991", "apples-adb7547b50854259", "new bag-f6d7e131622e4b8f", "food bin-1b84b0b4be429fd3", "tea-1e7d0923f2429369", "bag of grated cheese-a08827826c65edc3", "roll of pastry2-edbec4dc47961d0a", "milk-61bd734b8fdc0c67", "bag5-ddcf85728b77cb92", "right hand vegetable drawer-cd322e857a6d8bea", "bag of food-64eed06757c837d1", "left hand vegetable drawer-f7e84070c89ff0a6", "bag of food2-f5f098ad977ee22e", "tin of soup-1dbddf59c8ffc413" ], "reasoning": "1. I watched the video in order to find out where the food bin (food bin-1b84b0b4be429fd3) was located, as well as the fridge . Watching the video, I found the food bin (food bin-1b84b0b4be429fd3) at 01:19, when the user emptied out a bag (bag2-29acd80a7ccef27a) into it.\n2. Then, I looked to locate the fridge. I found it at 02:14, when the user opened it. It was located directly below where the user was emptying out the groceries.\n3. In order to understand the spatial relation between the food bin (food bin-1b84b0b4be429fd3) and the fridge I went back to when the user moved from one to the other. Before reaching the food bin (food bin-1b84b0b4be429fd3) at 01:19, the user turned to the right from the counter at 01:18, where the fridge is located directly below it, carrying the bag (bag2-29acd80a7ccef27a) with them to empty it. That led me to the final answer of to the right of the fridge.", "referenced_object_masks": [ "bag2-29acd80a7ccef27a", "food bin-1b84b0b4be429fd3" ] }, { "key": "P02-20240210-100308:c7e868fde9c20562653a5acc798e39a348ec9a27", "video_id": "P02-20240210-100308", "question": "If I wanted to place the pasta sauce in a drawer with bread, which drawer would that be if they were numbered in order from top to bottom?", "answer_choice_0": "The 6th drawer", "answer_choice_1": "The 3rd drawer", "answer_choice_2": "The 4th drawer", "answer_choice_3": "The 7th drawer", "answer_choice_4": "The 5th drawer", "answer_id": 2, "answer": "The 4th drawer", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Counting" ], "objects_of_interest": [ "bag1-80c78fea043044b8", "bag2-29acd80a7ccef27a", "bag3-f3047b10fb20748b", "raspberries-82a409ae73bef8ee", "eggs-9936026394f31f50", "apple turnover-ec11c098f3ff4ba6", "mushrooms-c59a38624beeb91c", "kebab wraps-64bed98c84d186ea", "raspberries2-5fa60bc46cc39158", "soap-83c308e9d8a1af07", "bag of tortilla chips-d6e22cf0fa353f6b", "bag of lemons-e7208f76d8b47300", "butter-3a6f7c2e808de8ce", "tomatoes-dd8c184cfd5bff98", "roll of pastry-05693451adcb0726", "pasta sauce-c9aa28d612d29561", "baguette1-eeaa69cc9978167c", "baguette2-3ea922d8e0c31d99", "bag4-0795261cf6dfe828", "red onions-5326f8c248026991", "apples-adb7547b50854259", "new bag-f6d7e131622e4b8f", "food bin-1b84b0b4be429fd3", "tea-1e7d0923f2429369", "bag of grated cheese-a08827826c65edc3", "roll of pastry2-edbec4dc47961d0a", "milk-61bd734b8fdc0c67", "bag5-ddcf85728b77cb92", "right hand vegetable drawer-cd322e857a6d8bea", "bag of food-64eed06757c837d1", "left hand vegetable drawer-f7e84070c89ff0a6", "bag of food2-f5f098ad977ee22e", "tin of soup-1dbddf59c8ffc413" ], "reasoning": "1. I watched the video and looked for when the user is placing the pasta sauce (pasta sauce-c9aa28d612d29561) in a drawer. First, I identified the pasta sauce at 00:48, when the user pulls it out of a bag (bag2-29acd80a7ccef27a).\n2. I then watched the video to continue to look for when the pasta sauce (pasta sauce-c9aa28d612d29561) is put away. I found this at 03:21, when the user picked up the pasta sauce (pasta sauce-c9aa28d612d29561) and walked to a closet. The user opened the closet at 03:15, revealing many items inside, including drawers .\n3. At 031:18, the user leaned down to begin opening various drawers . At this point, I started looking for bread . From 03:20-03:21, I saw bread briefly visible in the yellow drawer .\n4. Counting from the top, I determined that the yellow drawer was the 4th drawer, making that the answer.", "referenced_object_masks": [ "bag2-29acd80a7ccef27a", "pasta sauce-c9aa28d612d29561" ] }, { "key": "P02-20240210-100308:de164c6ca8257d4c8dcce0103e85e5f1229bd4cb", "video_id": "P02-20240210-100308", "question": "Where did I place the 10th item in the fridge?", "answer_choice_0": "The bottom shelf", "answer_choice_1": "The door shelf", "answer_choice_2": "The top shelf", "answer_choice_3": "The left vegetable drawer", "answer_choice_4": "The right vegetable drawer", "answer_id": 0, "answer": "The bottom shelf", "question_type": [ "Counting", "Action Recognition", "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "bag1-80c78fea043044b8", "bag2-29acd80a7ccef27a", "bag3-f3047b10fb20748b", "raspberries-82a409ae73bef8ee", "eggs-9936026394f31f50", "apple turnover-ec11c098f3ff4ba6", "mushrooms-c59a38624beeb91c", "kebab wraps-64bed98c84d186ea", "raspberries2-5fa60bc46cc39158", "soap-83c308e9d8a1af07", "bag of tortilla chips-d6e22cf0fa353f6b", "bag of lemons-e7208f76d8b47300", "butter-3a6f7c2e808de8ce", "tomatoes-dd8c184cfd5bff98", "roll of pastry-05693451adcb0726", "pasta sauce-c9aa28d612d29561", "baguette1-eeaa69cc9978167c", "baguette2-3ea922d8e0c31d99", "bag4-0795261cf6dfe828", "red onions-5326f8c248026991", "apples-adb7547b50854259", "new bag-f6d7e131622e4b8f", "food bin-1b84b0b4be429fd3", "tea-1e7d0923f2429369", "bag of grated cheese-a08827826c65edc3", "roll of pastry2-edbec4dc47961d0a", "milk-61bd734b8fdc0c67", "bag5-ddcf85728b77cb92", "right hand vegetable drawer-cd322e857a6d8bea", "bag of food-64eed06757c837d1", "left hand vegetable drawer-f7e84070c89ff0a6", "bag of food2-f5f098ad977ee22e", "tin of soup-1dbddf59c8ffc413" ], "reasoning": "1. I watched the video, looking for when the user started adding items to the fridge . I found this moment at 02:14.\n2. I then started counting the items as the user started placing them inside the fridge . The user places the first item, the butter (butter-3a6f7c2e808de8ce) at 02:18, followed by the milk (milk-61bd734b8fdc0c67) which is placed at 02:20. Then, the mushrooms (mushrooms-c59a38624beeb91c) are placed at 02:25, followed by the red onions (red onions-5326f8c248026991) at 02:30. 2 packages of raspberries (raspberries-82a409ae73bef8ee, raspberries2-5fa60bc46cc39158) are added at 02:35, followed by more apples (apples-adb7547b50854259) at 02:40 and lemons (bag of lemons-e7208f76d8b47300) at 02:42.\n3. At this point, I watched to see that the user picked up 3 items at once. Knowing this would soon bring the total to 10, I watched carefully to see where the next 2 would be placed. I saw the user add the eggs (eggs-9936026394f31f50) at 02:49 and the tomatoes (tomatoes-dd8c184cfd5bff98) at 02:51.\n4. At 02:51, I noticed the user added the tomatoes (tomatoes-dd8c184cfd5bff98) to the fridge in the bottom shelf, making that the answer.", "referenced_object_masks": [ "apples-adb7547b50854259", "red onions-5326f8c248026991", "eggs-9936026394f31f50", "milk-61bd734b8fdc0c67", "mushrooms-c59a38624beeb91c", "bag of lemons-e7208f76d8b47300", "raspberries-82a409ae73bef8ee", "raspberries2-5fa60bc46cc39158", "butter-3a6f7c2e808de8ce", "tomatoes-dd8c184cfd5bff98" ] }, { "key": "P02-20240210-100930:06c4ab8fc1ecf2ab5cf4e3b47e0e4505b3434cf7", "video_id": "P02-20240210-100930", "question": "What is the color of the cap of the bottle sitting in the front right of the cabinet?", "answer_choice_0": "Blue", "answer_choice_1": ", Yellow", "answer_choice_2": "Red", "answer_choice_3": "Black", "answer_choice_4": "Purple", "answer_id": 0, "answer": "Blue", "question_type": [ "Object Recognition", "Spatial Perception", "Situational Awareness" ], "objects_of_interest": [ "bottle of soap-bb3264e72dd5c093" ], "reasoning": "1. I watched the video and observed the user walk towards and open the cabinet door at 0:04.\n2. I continued watching to look at what is present in the cabinet after it was opened, identifying many different bottles .\n3. At 0:05, I identified the far right bottle as the one with the pale colored liquid in it, and the blue cap.", "referenced_object_masks": [] }, { "key": "P02-20240210-100930:2f0ac9f0e495a354caaa66df893503d74710587a", "video_id": "P02-20240210-100930", "question": "What time would it be if it was 15 minutes before the time shown on the clock hanging on the wall?", "answer_choice_0": "09:55", "answer_choice_1": "11:15", "answer_choice_2": "12:00", "answer_choice_3": "10:45", "answer_choice_4": "09:45", "answer_id": 0, "answer": "09:55", "question_type": [ "Goal Reasoning", "Numerical Reasoning", "Temporal Reasoning" ], "objects_of_interest": [ "bottle of soap-bb3264e72dd5c093" ], "reasoning": "1. I watched the video and identified the clock hanging on the wall , which first appears at 0:08.\n2. I then looked at the clock's hands and reasoned that it shows an approximate time of 10:10.\n3. Then, I did the math reasoning to discover that 15 minutes before this would be 09:55.", "referenced_object_masks": [] }, { "key": "P02-20240210-100930:324a9e7ee8de42bf58d71a02443aa130ec198c78", "video_id": "P02-20240210-100930", "question": "If I was sitting at the dining room table, how many open chairs would there be?", "answer_choice_0": "4", "answer_choice_1": "1", "answer_choice_2": "3", "answer_choice_3": "0", "answer_choice_4": "2", "answer_id": 4, "answer": "2", "question_type": [ "Counting", "Object Recognition", "Counterfactual", "Numerical Reasoning" ], "objects_of_interest": [ "bottle of soap-bb3264e72dd5c093" ], "reasoning": "1. I watched the video until 00:07, when I saw the table in the dining room.\n2. I looked around the table and counted 2 empty chairs . There appears to be a third chair facing the user, left of the open cardboard box, but it is draped over with a light green object, therefore making it not \"open\" but blocked by the covering. Also, it is not clear the draped over object is really a chair at all. \n3. Then, I did the math that if the user was sitting in one of the 2 \"open\" chairs, there would be 1 empty one.", "referenced_object_masks": [] }, { "key": "P02-20240210-100930:37574da312347160d83d7edfe6197676f671d691", "video_id": "P02-20240210-100930", "question": "What is the product of the number of hearts on the floormat and the number of bowls sitting to the right of the sink?", "answer_choice_0": "30", "answer_choice_1": "40", "answer_choice_2": "36", "answer_choice_3": "20", "answer_choice_4": "24", "answer_id": 0, "answer": "30", "question_type": [ "Numerical Reasoning", "Object Recognition", "Counting", "Situational Awareness" ], "objects_of_interest": [ "bottle of soap-bb3264e72dd5c093" ], "reasoning": "1. I watched the video to locate the floormat , which is in frame at 0:00.\n2. Then, I counted the hearts on the floormat , counting a total of 15.\n3. I then continued to watch the video until 0:06, when I counted two bowls sitting to the right of the sink.\n4. Lastly, I calculated 15 x 2, which is 30.", "referenced_object_masks": [] }, { "key": "P02-20240210-100930:585077ca67424225499095fcb0001ef6db2798db", "video_id": "P02-20240210-100930", "question": "If I would have opened the door to the right of the cabinet door I opened in the video, what appliance would I have put the bottle of soap in?", "answer_choice_0": "The oven", "answer_choice_1": "The refrigerator", "answer_choice_2": "The microwave", "answer_choice_3": "The washer/dryer", "answer_choice_4": "The dishwasher", "answer_id": 3, "answer": "The washer/dryer", "question_type": [ "Counterfactual", "Spatial Perception", "Object Recognition" ], "objects_of_interest": [ "bottle of soap-bb3264e72dd5c093" ], "reasoning": "1. I watched the video to identify the cabinet door that is opened in order to put the bottle of soap (bottle of soap-bb3264e72dd5c093) away. This occurs at 0:04.\n2. I then looked immediately to the right and identified the appliance next to it as the washer/dryer .", "referenced_object_masks": [ "bottle of soap-bb3264e72dd5c093" ] }, { "key": "P02-20240210-113925:5670476072b835fd34ed0627cebf7aa56b7c92cf", "video_id": "P02-20240210-113925", "question": "How much time passes between when I crack the first egg into the measuring jug and when I crack the last egg into it?", "answer_choice_0": "40 seconds", "answer_choice_1": "30 seconds", "answer_choice_2": "23 seconds", "answer_choice_3": "44 seconds", "answer_choice_4": "37 seconds", "answer_id": 4, "answer": "37 seconds", "question_type": [ "Event Occurence", "Temporal Reasoning", "Object Recognition", "Numerical Reasoning" ], "objects_of_interest": [ "plate-8ec7008bb5c6e06b", "bread knife-faca30420b41e6cb", "baguette-73f51daf8ea0ec91", "bacon medallions-575f77c91da5af8f", "carton of eggs-e0a197e792966f3f", "larger carton of eggs-83c8a647b81ecabe", "block of cheese-02f59a5088f969d4", "carton of milk-ac16f1b915683ef6", "frying pan-23a7a6a7e66459e4", "olive oil bottle-d3cf5610c23a17d2", "white frying pan-708293d1e0a27d03", "large white frying pan-6d8d5bbe6f507c1b", "bacon medallion polybag-68a100b3d2839940", "unsmoked bacon medallion packet-176fb61068c79f74", "roll of kitchen towel-f07229a2080c4df5", "one sheet of kitchen towel-863f526570172169", "unsmoked baking medallion-9890f8df6b8b0042", "unsmoked baking medallion first-65ee48a0d73a0a88", "smoked bacon medallion first-442dec2933331978", "smoked bacon medallion second-918aef6513e3e25a", "washing up liquid bottle-c21b43a42d71a47d", "smoked bacon medallion packet-58171de0c6f4ce8a", "measuring jug-823b256a0a840084", "first egg-edbc0b70d2816b8d", "second egg-1c4df3e32a5b1f3f", "spatula black-62bb424d9c6ad8d6", "third egg-0361a7231e7ea90e", "fourth egg-d0e39f7790815449", "bunch of utensils-e5dcd030924cf800", "fork first-ef474b2696aba54e", "pepper mill-115789cdaa8d9119", "top of pepper mill-a98c11571b400506", "kettle-0b92187cdc1dc7c5", "half of baguette-a00739b48d06890a", "right half of baguette-6f02e769ce4698ff", "one fourth left of baguette-3d0234f98420cd96", "one fourth right of baguette-6fecd115ef1737f6", "food bin-e4383e8805a162af", "butter knife-4dfe3449281ca441", "grater-0b027602a6ca451c", "tub of butter-912757b9811a2dfe", "lid of butter tub-513ef491d3c825c3", "salt bottle-d0b6b7c5f2d83d7d", "spices-4b0bfacfe0c54818", "jar with green lid-930b22dd95f4b081", "jar of paprika-e98594e8c58e4a12", "jar of garam masala-eefcc4a33a22238a", "spice bags-e261c425f072629c", "plate second-9a8ff9648a75f56e" ], "reasoning": "1. I watched the video to find when the user began cracking eggs \n2. I first noted that the user grabbed the measuring jug (measuring jug-823b256a0a840084) from the cabinet from 02:58-03:03.\n3. Then, I continued to watch and saw the user crack the first egg (first egg-edbc0b70d2816b8d) at 03:30. \n4. The user continued and cracked the second egg (second egg-1c4df3e32a5b1f3f) at 03:37.\n5. The user continued to cook, cracking the third egg (third egg-0361a7231e7ea90e) at 03:59.\n6. Next, the user cracked a fourth egg (fourth egg-d0e39f7790815449) into the measuring jug (measuring jug-823b256a0a840084) at 04:07.\n7. The user moved to other things after this, and I watched to the end and did not observe them crack any other eggs, so the last egg was cracked at 04:07.\n8. Lastly, I calculated that 37 seconds lapsed between the first egg and last egg being cracked.", "referenced_object_masks": [ "third egg-0361a7231e7ea90e", "fourth egg-d0e39f7790815449", "measuring jug-823b256a0a840084", "first egg-edbc0b70d2816b8d", "second egg-1c4df3e32a5b1f3f" ] }, { "key": "P02-20240210-113925:5c744c6b3831e3cec26dd89336305ae65e728d80", "video_id": "P02-20240210-113925", "question": "After rinsing the eggs, what is the first item, not including the eggs themselves, that I touch with my left hand?", "answer_choice_0": "The pan on the stove", "answer_choice_1": "The white plate", "answer_choice_2": "The loaf of bread", "answer_choice_3": "The smaller egg carton", "answer_choice_4": "The refrigerator door", "answer_id": 3, "answer": "The smaller egg carton", "question_type": [ "Temporal Reasoning", "Event Occurence", "Situational Awareness", "Object Recognition" ], "objects_of_interest": [ "plate-8ec7008bb5c6e06b", "bread knife-faca30420b41e6cb", "baguette-73f51daf8ea0ec91", "bacon medallions-575f77c91da5af8f", "carton of eggs-e0a197e792966f3f", "larger carton of eggs-83c8a647b81ecabe", "block of cheese-02f59a5088f969d4", "carton of milk-ac16f1b915683ef6", "frying pan-23a7a6a7e66459e4", "olive oil bottle-d3cf5610c23a17d2", "white frying pan-708293d1e0a27d03", "large white frying pan-6d8d5bbe6f507c1b", "bacon medallion polybag-68a100b3d2839940", "unsmoked bacon medallion packet-176fb61068c79f74", "roll of kitchen towel-f07229a2080c4df5", "one sheet of kitchen towel-863f526570172169", "unsmoked baking medallion-9890f8df6b8b0042", "unsmoked baking medallion first-65ee48a0d73a0a88", "smoked bacon medallion first-442dec2933331978", "smoked bacon medallion second-918aef6513e3e25a", "washing up liquid bottle-c21b43a42d71a47d", "smoked bacon medallion packet-58171de0c6f4ce8a", "measuring jug-823b256a0a840084", "first egg-edbc0b70d2816b8d", "second egg-1c4df3e32a5b1f3f", "spatula black-62bb424d9c6ad8d6", "third egg-0361a7231e7ea90e", "fourth egg-d0e39f7790815449", "bunch of utensils-e5dcd030924cf800", "fork first-ef474b2696aba54e", "pepper mill-115789cdaa8d9119", "top of pepper mill-a98c11571b400506", "kettle-0b92187cdc1dc7c5", "half of baguette-a00739b48d06890a", "right half of baguette-6f02e769ce4698ff", "one fourth left of baguette-3d0234f98420cd96", "one fourth right of baguette-6fecd115ef1737f6", "food bin-e4383e8805a162af", "butter knife-4dfe3449281ca441", "grater-0b027602a6ca451c", "tub of butter-912757b9811a2dfe", "lid of butter tub-513ef491d3c825c3", "salt bottle-d0b6b7c5f2d83d7d", "spices-4b0bfacfe0c54818", "jar with green lid-930b22dd95f4b081", "jar of paprika-e98594e8c58e4a12", "jar of garam masala-eefcc4a33a22238a", "spice bags-e261c425f072629c", "plate second-9a8ff9648a75f56e" ], "reasoning": "1. I watched the video to identify when the user rinses the eggs (first egg-edbc0b70d2816b8d, second egg-1c4df3e32a5b1f3f), which occurs from 03:11 to 03:16.\n2. At this point in the video, I noted that the user has both eggs (first egg-edbc0b70d2816b8d, second egg-1c4df3e32a5b1f3f) held in both hands.\n3. I continued to watch the video to identify what the user touches first with their left hand.\n4. They grab the spatula (spatula black-62bb424d9c6ad8d6) at 03:18 with the right hand.\n5. Then, at 03:22, they touch the middle stovetop nob with the right hand.\n6. I continued to watch the user crack both eggs (first egg-edbc0b70d2816b8d, second egg-1c4df3e32a5b1f3f) from 03:18-03:28, still having not touched anything with the left hand.\n7. I continued to watch and saw that at 03:40, the user touched the egg carton (carton of eggs-e0a197e792966f3f) with the left hand.", "referenced_object_masks": [ "carton of eggs-e0a197e792966f3f", "spatula black-62bb424d9c6ad8d6", "first egg-edbc0b70d2816b8d", "second egg-1c4df3e32a5b1f3f" ] }, { "key": "P02-20240210-113925:631ff9889ff7d964bfbe8063bfc4f826f1d1eccf", "video_id": "P02-20240210-113925", "question": "In what order do I add scrambled eggs to the two pieces of bread?", "answer_choice_0": "L-R-L-L-R", "answer_choice_1": "R-R-R-L-L-L", "answer_choice_2": "R-L-R-L-R-L", "answer_choice_3": "L-R-L-R", "answer_choice_4": "R-R-L-L", "answer_id": 2, "answer": "R-L-R-L-R-L", "question_type": [ "Event Occurence", "Situational Awareness", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "plate-8ec7008bb5c6e06b", "bread knife-faca30420b41e6cb", "baguette-73f51daf8ea0ec91", "bacon medallions-575f77c91da5af8f", "carton of eggs-e0a197e792966f3f", "larger carton of eggs-83c8a647b81ecabe", "block of cheese-02f59a5088f969d4", "carton of milk-ac16f1b915683ef6", "frying pan-23a7a6a7e66459e4", "olive oil bottle-d3cf5610c23a17d2", "white frying pan-708293d1e0a27d03", "large white frying pan-6d8d5bbe6f507c1b", "bacon medallion polybag-68a100b3d2839940", "unsmoked bacon medallion packet-176fb61068c79f74", "roll of kitchen towel-f07229a2080c4df5", "one sheet of kitchen towel-863f526570172169", "unsmoked baking medallion-9890f8df6b8b0042", "unsmoked baking medallion first-65ee48a0d73a0a88", "smoked bacon medallion first-442dec2933331978", "smoked bacon medallion second-918aef6513e3e25a", "washing up liquid bottle-c21b43a42d71a47d", "smoked bacon medallion packet-58171de0c6f4ce8a", "measuring jug-823b256a0a840084", "first egg-edbc0b70d2816b8d", "second egg-1c4df3e32a5b1f3f", "spatula black-62bb424d9c6ad8d6", "third egg-0361a7231e7ea90e", "fourth egg-d0e39f7790815449", "bunch of utensils-e5dcd030924cf800", "fork first-ef474b2696aba54e", "pepper mill-115789cdaa8d9119", "top of pepper mill-a98c11571b400506", "kettle-0b92187cdc1dc7c5", "half of baguette-a00739b48d06890a", "right half of baguette-6f02e769ce4698ff", "one fourth left of baguette-3d0234f98420cd96", "one fourth right of baguette-6fecd115ef1737f6", "food bin-e4383e8805a162af", "butter knife-4dfe3449281ca441", "grater-0b027602a6ca451c", "tub of butter-912757b9811a2dfe", "lid of butter tub-513ef491d3c825c3", "salt bottle-d0b6b7c5f2d83d7d", "spices-4b0bfacfe0c54818", "jar with green lid-930b22dd95f4b081", "jar of paprika-e98594e8c58e4a12", "jar of garam masala-eefcc4a33a22238a", "spice bags-e261c425f072629c", "plate second-9a8ff9648a75f56e" ], "reasoning": "1. I identify the pan (frying pan-23a7a6a7e66459e4) filled with scrambled eggs in the user's left hand at 11:23.\n2. From 11:26 to 12:01, I observe the user add scrambled eggs to the bread (right half of baguette-6f02e769ce4698ff, half of baguette-a00739b48d06890a) with the spatula (spatula black-62bb424d9c6ad8d6).\n3. I observe the user adding scrambled eggs to the right piece of bread (right half of baguette-6f02e769ce4698ff) at 11:27, 11:43, and 11:53.\n4. I observe the user adding scrambled eggs to the left piece of bread (half of baguette-a00739b48d06890a) at 11:32, 11:48, and 11:56.\n5. I determine the order in which the user adds scrambled eggs to the bread (right half of baguette-6f02e769ce4698ff, half of baguette-a00739b48d06890a) is right, left, right, left, right, left (R-L-R-L-R-L).", "referenced_object_masks": [ "right half of baguette-6f02e769ce4698ff", "half of baguette-a00739b48d06890a", "frying pan-23a7a6a7e66459e4", "spatula black-62bb424d9c6ad8d6" ] }, { "key": "P02-20240210-113925:81ee1850ccefc21e2124d1e5527e1d47f3e44aca", "video_id": "P02-20240210-113925", "question": "If I would have grabbed the leftmost item on the top shelf of the pantry, what is the predominant color of that item?", "answer_choice_0": "Red", "answer_choice_1": "Black", "answer_choice_2": "Blue", "answer_choice_3": "White", "answer_choice_4": "Yellow", "answer_id": 2, "answer": "Blue", "question_type": [ "Counting", "Object Recognition", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "plate-8ec7008bb5c6e06b", "bread knife-faca30420b41e6cb", "baguette-73f51daf8ea0ec91", "bacon medallions-575f77c91da5af8f", "carton of eggs-e0a197e792966f3f", "larger carton of eggs-83c8a647b81ecabe", "block of cheese-02f59a5088f969d4", "carton of milk-ac16f1b915683ef6", "frying pan-23a7a6a7e66459e4", "olive oil bottle-d3cf5610c23a17d2", "white frying pan-708293d1e0a27d03", "large white frying pan-6d8d5bbe6f507c1b", "bacon medallion polybag-68a100b3d2839940", "unsmoked bacon medallion packet-176fb61068c79f74", "roll of kitchen towel-f07229a2080c4df5", "one sheet of kitchen towel-863f526570172169", "unsmoked baking medallion-9890f8df6b8b0042", "unsmoked baking medallion first-65ee48a0d73a0a88", "smoked bacon medallion first-442dec2933331978", "smoked bacon medallion second-918aef6513e3e25a", "washing up liquid bottle-c21b43a42d71a47d", "smoked bacon medallion packet-58171de0c6f4ce8a", "measuring jug-823b256a0a840084", "first egg-edbc0b70d2816b8d", "second egg-1c4df3e32a5b1f3f", "spatula black-62bb424d9c6ad8d6", "third egg-0361a7231e7ea90e", "fourth egg-d0e39f7790815449", "bunch of utensils-e5dcd030924cf800", "fork first-ef474b2696aba54e", "pepper mill-115789cdaa8d9119", "top of pepper mill-a98c11571b400506", "kettle-0b92187cdc1dc7c5", "half of baguette-a00739b48d06890a", "right half of baguette-6f02e769ce4698ff", "one fourth left of baguette-3d0234f98420cd96", "one fourth right of baguette-6fecd115ef1737f6", "food bin-e4383e8805a162af", "butter knife-4dfe3449281ca441", "grater-0b027602a6ca451c", "tub of butter-912757b9811a2dfe", "lid of butter tub-513ef491d3c825c3", "salt bottle-d0b6b7c5f2d83d7d", "spices-4b0bfacfe0c54818", "jar with green lid-930b22dd95f4b081", "jar of paprika-e98594e8c58e4a12", "jar of garam masala-eefcc4a33a22238a", "spice bags-e261c425f072629c", "plate second-9a8ff9648a75f56e" ], "reasoning": "1. I watched the video to find when the user opens the pantry door , which occurs at 09:47-09:48.\n2. I then watched the user approach the top shelf, where there are spices (spice bags-e261c425f072629c)\n3. I looked to the far left of the shelf and identified a box of \"Extra Large Tissues\" , which is predominantly blue.", "referenced_object_masks": [ "spice bags-e261c425f072629c" ] }, { "key": "P02-20240210-113925:8b3ae5edb60a10693bf1de13bac61214a53ae1d6", "video_id": "P02-20240210-113925", "question": "What is the quotient of the maximum quantity of eggs that could be stored in the cartons I take out of the fridge divided by the number of items located on the middle shelf of the refrigerator door?", "answer_choice_0": "6", "answer_choice_1": "5", "answer_choice_2": "4", "answer_choice_3": "3", "answer_choice_4": "10", "answer_id": 3, "answer": "3", "question_type": [ "Egocentric & Spatial Perception", "Counting", "Numerical Reasoning", "Object Recognition" ], "objects_of_interest": [ "plate-8ec7008bb5c6e06b", "bread knife-faca30420b41e6cb", "baguette-73f51daf8ea0ec91", "bacon medallions-575f77c91da5af8f", "carton of eggs-e0a197e792966f3f", "larger carton of eggs-83c8a647b81ecabe", "block of cheese-02f59a5088f969d4", "carton of milk-ac16f1b915683ef6", "frying pan-23a7a6a7e66459e4", "olive oil bottle-d3cf5610c23a17d2", "white frying pan-708293d1e0a27d03", "large white frying pan-6d8d5bbe6f507c1b", "bacon medallion polybag-68a100b3d2839940", "unsmoked bacon medallion packet-176fb61068c79f74", "roll of kitchen towel-f07229a2080c4df5", "one sheet of kitchen towel-863f526570172169", "unsmoked baking medallion-9890f8df6b8b0042", "unsmoked baking medallion first-65ee48a0d73a0a88", "smoked bacon medallion first-442dec2933331978", "smoked bacon medallion second-918aef6513e3e25a", "washing up liquid bottle-c21b43a42d71a47d", "smoked bacon medallion packet-58171de0c6f4ce8a", "measuring jug-823b256a0a840084", "first egg-edbc0b70d2816b8d", "second egg-1c4df3e32a5b1f3f", "spatula black-62bb424d9c6ad8d6", "third egg-0361a7231e7ea90e", "fourth egg-d0e39f7790815449", "bunch of utensils-e5dcd030924cf800", "fork first-ef474b2696aba54e", "pepper mill-115789cdaa8d9119", "top of pepper mill-a98c11571b400506", "kettle-0b92187cdc1dc7c5", "half of baguette-a00739b48d06890a", "right half of baguette-6f02e769ce4698ff", "one fourth left of baguette-3d0234f98420cd96", "one fourth right of baguette-6fecd115ef1737f6", "food bin-e4383e8805a162af", "butter knife-4dfe3449281ca441", "grater-0b027602a6ca451c", "tub of butter-912757b9811a2dfe", "lid of butter tub-513ef491d3c825c3", "salt bottle-d0b6b7c5f2d83d7d", "spices-4b0bfacfe0c54818", "jar with green lid-930b22dd95f4b081", "jar of paprika-e98594e8c58e4a12", "jar of garam masala-eefcc4a33a22238a", "spice bags-e261c425f072629c", "plate second-9a8ff9648a75f56e" ], "reasoning": "1. I watched the video to see when the user opens the refrigerator door , which occurs from 00:18-00:20.\n2. Once the user opened the fridge door , I looked at the middle shelf of the door and counted six items on the middle shelf .\n3. I continued to watch the video to see the user take out the carton of eggs (carton of eggs-e0a197e792966f3f) and then the larger carton of eggs (larger carton of eggs-83c8a647b81ecabe) from 00:24-00:29.\n4. I then looked at the egg cartons to identify that the larger one (larger carton of eggs-83c8a647b81ecabe) would hold a max of 12 eggs, while the other carton (carton of eggs-e0a197e792966f3f) would hold 6 eggs.\n5. I calculated the total maximum egg total by doing 12+6, which equals 18.\n6. Then I did the final math, which is 18/6=3.", "referenced_object_masks": [ "carton of eggs-e0a197e792966f3f", "larger carton of eggs-83c8a647b81ecabe" ] }, { "key": "P02-20240210-122334:1292549ad6c8b393201390902d85e23cbf9c98f0", "video_id": "P02-20240210-122334", "question": "What number would I get if I subtract the number of plates I remove from the table from the number of times I pick out dirt from the stove burners right after putting the sponge down next to the stove?", "answer_choice_0": "5", "answer_choice_1": "6", "answer_choice_2": "3", "answer_choice_3": "2", "answer_choice_4": "4", "answer_id": 2, "answer": "3", "question_type": [ "Temporal Reasoning", "Action Recognition", "Counting", "Object Recognition", "Numerical Reasoning" ], "objects_of_interest": [ "plate-64932742e661fa8d", "plate2-ebb4519e77479a84", "empty packets-688cf092ab1da6c7", "empty transparent packet-1f72fdcdb9854618", "empty transparent packet2-7d3c992d9a0a70d6", "empty packet 4-89e119b446942033", "empty packet 5-785086a34dabf922", "fork-fb888ab5b17daedb", "measuring jug-8de253aa703bd253", "spatula-f409325dba434135", "washing up bowl-e5acdbb5d8e400e5", "washing up liquid-08083f440e3a1e1e", "knife-192755582037f312", "frying pan-66d1fede8722f487", "cutting board-f6820a9b6bb2197d", "tupperware box-78ecadb9df769a90", "tupperware lid-2620440fdb6553cc", "smaller plate-1e25a8ba2a47a847", "larger plate-4cd348e03aa9b6f7", "paring knife-f587f6a18ed9b816", "chef's knife-184f253b825b5443", "small serrated knife-439866a448f31235", "normal knife-cbda343e0c7f16a4", "butter knife-4e1044d9ff5fc820", "table spoon-85a973ab94c5b31a", "large spoon-d13bc1235c3a0133", "bowl-6151fe36af40abd7", "bowl2-b86172c7ce33eb2a", "bag of sponges-d85f8c4d8004f8b8", "baguette-41bdad65b0509598", "bread knife-67c8c6584330b3b9", "quarter of baguette-d872288c54495ad3", "one quarter of baguette-4a3d9ec1c48afa87", "sponge-870455e237e8d974", "cheese grater-30dad965b3119841", "butter knife2-1c1e3a17e4b1b400", "smaller frying pan-e41d2ec1d9183a72", "fork2-55cd49b529a35f7b", "food bin-9ce62ee24b34513b", "drain protector-06cdcdcbcfb5a167" ], "reasoning": "1. I determine that the user puts the sponge (sponge-870455e237e8d974) down next to the stove at 06:55.\n2. I determine that the user picks dirt from the stove burners between 06:55 and 07:04.\n3. I count the number of times the user picks dirt from the stove burners between 06:55 and 07:04: first at 06:55 (1), then at 06:57 (2), then again at 06:59 (3), again at 07:01 (4) and once more at 07:02 (5), or 5 times in all.\n4. I identify the 2 plates (plate-64932742e661fa8d) and (plate2-ebb4519e77479a84), which first appear at 00:02 on the table.\n5. I determine that the user removes the plates (plate-64932742e661fa8d) and (plate2-ebb4519e77479a84) from the table at 00:02.\n6. I subtract the 2 plates (plate-64932742e661fa8d) and (plate2-ebb4519e77479a84) from the 5 times the user picks dirt from the stove burners: 5-2=3.", "referenced_object_masks": [ "plate-64932742e661fa8d", "sponge-870455e237e8d974", "plate2-ebb4519e77479a84" ] }, { "key": "P02-20240210-122334:2c599f0dc33152aa2e3c8939335d131121523987", "video_id": "P02-20240210-122334", "question": "Where is the bird in relation to me when I place the second object into the drying rack?", "answer_choice_0": "To the right of me", "answer_choice_1": "Behind me", "answer_choice_2": "To the left of me", "answer_choice_3": "Above me", "answer_choice_4": "In front of me", "answer_id": 4, "answer": "In front of me", "question_type": [ "Temporal Reasoning", "Egocentric & Spatial Perception", "Action Recognition", "Counting", "Object Recognition" ], "objects_of_interest": [ "plate-64932742e661fa8d", "plate2-ebb4519e77479a84", "empty packets-688cf092ab1da6c7", "empty transparent packet-1f72fdcdb9854618", "empty transparent packet2-7d3c992d9a0a70d6", "empty packet 4-89e119b446942033", "empty packet 5-785086a34dabf922", "fork-fb888ab5b17daedb", "measuring jug-8de253aa703bd253", "spatula-f409325dba434135", "washing up bowl-e5acdbb5d8e400e5", "washing up liquid-08083f440e3a1e1e", "knife-192755582037f312", "frying pan-66d1fede8722f487", "cutting board-f6820a9b6bb2197d", "tupperware box-78ecadb9df769a90", "tupperware lid-2620440fdb6553cc", "smaller plate-1e25a8ba2a47a847", "larger plate-4cd348e03aa9b6f7", "paring knife-f587f6a18ed9b816", "chef's knife-184f253b825b5443", "small serrated knife-439866a448f31235", "normal knife-cbda343e0c7f16a4", "butter knife-4e1044d9ff5fc820", "table spoon-85a973ab94c5b31a", "large spoon-d13bc1235c3a0133", "bowl-6151fe36af40abd7", "bowl2-b86172c7ce33eb2a", "bag of sponges-d85f8c4d8004f8b8", "baguette-41bdad65b0509598", "bread knife-67c8c6584330b3b9", "quarter of baguette-d872288c54495ad3", "one quarter of baguette-4a3d9ec1c48afa87", "sponge-870455e237e8d974", "cheese grater-30dad965b3119841", "butter knife2-1c1e3a17e4b1b400", "smaller frying pan-e41d2ec1d9183a72", "fork2-55cd49b529a35f7b", "food bin-9ce62ee24b34513b", "drain protector-06cdcdcbcfb5a167" ], "reasoning": "1. I locate the bird , which first appears at 03:27 in the backyard. \n2. I determine that the user places the second object into the drying rack at 03:33.\n3. I identify the second object as the knife (bread knife-67c8c6584330b3b9), which first appears at 00:05 on the kitchen counter. \n4. I identify the bird 's location in relation to the user as they place the knife (bread knife-67c8c6584330b3b9) into the drying rack at 03:33.", "referenced_object_masks": [ "bread knife-67c8c6584330b3b9" ] }, { "key": "P02-20240210-122334:646683894f9a938a7809a1ffdaec071579dd1ba2", "video_id": "P02-20240210-122334", "question": "What number would I get if I subtract the number of times I bang the drain protector against the garbage bin from the number of hearts on the brown doormat?", "answer_choice_0": "Nine", "answer_choice_1": "Seven", "answer_choice_2": "Eight", "answer_choice_3": "Six", "answer_choice_4": "Five", "answer_id": 1, "answer": "Seven", "question_type": [ "Temporal Reasoning", "Egocentric & Spatial Perception", "Action Recognition", "Counting", "Object Recognition", "Numerical Reasoning" ], "objects_of_interest": [ "plate-64932742e661fa8d", "plate2-ebb4519e77479a84", "empty packets-688cf092ab1da6c7", "empty transparent packet-1f72fdcdb9854618", "empty transparent packet2-7d3c992d9a0a70d6", "empty packet 4-89e119b446942033", "empty packet 5-785086a34dabf922", "fork-fb888ab5b17daedb", "measuring jug-8de253aa703bd253", "spatula-f409325dba434135", "washing up bowl-e5acdbb5d8e400e5", "washing up liquid-08083f440e3a1e1e", "knife-192755582037f312", "frying pan-66d1fede8722f487", "cutting board-f6820a9b6bb2197d", "tupperware box-78ecadb9df769a90", "tupperware lid-2620440fdb6553cc", "smaller plate-1e25a8ba2a47a847", "larger plate-4cd348e03aa9b6f7", "paring knife-f587f6a18ed9b816", "chef's knife-184f253b825b5443", "small serrated knife-439866a448f31235", "normal knife-cbda343e0c7f16a4", "butter knife-4e1044d9ff5fc820", "table spoon-85a973ab94c5b31a", "large spoon-d13bc1235c3a0133", "bowl-6151fe36af40abd7", "bowl2-b86172c7ce33eb2a", "bag of sponges-d85f8c4d8004f8b8", "baguette-41bdad65b0509598", "bread knife-67c8c6584330b3b9", "quarter of baguette-d872288c54495ad3", "one quarter of baguette-4a3d9ec1c48afa87", "sponge-870455e237e8d974", "cheese grater-30dad965b3119841", "butter knife2-1c1e3a17e4b1b400", "smaller frying pan-e41d2ec1d9183a72", "fork2-55cd49b529a35f7b", "food bin-9ce62ee24b34513b", "drain protector-06cdcdcbcfb5a167" ], "reasoning": "1. I identify the brown doormat , which first appears at 00:05 directly ahead of the user.\n2. I determine that the brown doormat has 15 hearts on it.\n3. I determine that the user bangs the drain protector (drain protector-06cdcdcbcfb5a167) against the garbage bin 8 times between 08:27 and 08:32.\n4. I subtract 8 from 15.", "referenced_object_masks": [ "drain protector-06cdcdcbcfb5a167" ] }, { "key": "P02-20240210-122334:9161f6e611bb780b6c153f832740929a9be78eb2", "video_id": "P02-20240210-122334", "question": "Where is the clock in relation to me when I dry my hands?", "answer_choice_0": "Behind me", "answer_choice_1": "Below me", "answer_choice_2": "Above me", "answer_choice_3": "To the left of me", "answer_choice_4": "To the right of me", "answer_id": 2, "answer": "Above me", "question_type": [ "Temporal Reasoning", "Egocentric & Spatial Perception", "Action Recognition", "Object Recognition" ], "objects_of_interest": [ "plate-64932742e661fa8d", "plate2-ebb4519e77479a84", "empty packets-688cf092ab1da6c7", "empty transparent packet-1f72fdcdb9854618", "empty transparent packet2-7d3c992d9a0a70d6", "empty packet 4-89e119b446942033", "empty packet 5-785086a34dabf922", "fork-fb888ab5b17daedb", "measuring jug-8de253aa703bd253", "spatula-f409325dba434135", "washing up bowl-e5acdbb5d8e400e5", "washing up liquid-08083f440e3a1e1e", "knife-192755582037f312", "frying pan-66d1fede8722f487", "cutting board-f6820a9b6bb2197d", "tupperware box-78ecadb9df769a90", "tupperware lid-2620440fdb6553cc", "smaller plate-1e25a8ba2a47a847", "larger plate-4cd348e03aa9b6f7", "paring knife-f587f6a18ed9b816", "chef's knife-184f253b825b5443", "small serrated knife-439866a448f31235", "normal knife-cbda343e0c7f16a4", "butter knife-4e1044d9ff5fc820", "table spoon-85a973ab94c5b31a", "large spoon-d13bc1235c3a0133", "bowl-6151fe36af40abd7", "bowl2-b86172c7ce33eb2a", "bag of sponges-d85f8c4d8004f8b8", "baguette-41bdad65b0509598", "bread knife-67c8c6584330b3b9", "quarter of baguette-d872288c54495ad3", "one quarter of baguette-4a3d9ec1c48afa87", "sponge-870455e237e8d974", "cheese grater-30dad965b3119841", "butter knife2-1c1e3a17e4b1b400", "smaller frying pan-e41d2ec1d9183a72", "fork2-55cd49b529a35f7b", "food bin-9ce62ee24b34513b", "drain protector-06cdcdcbcfb5a167" ], "reasoning": "1. I identify the clock , which first appears at 08:49 above the radiator .\n2. I determine that the user dries their hands between 08:52 to 08:56 with the towel on the radiator .\n3. I identify the location of the clock in relation to the user between 08:52 and 08:56.", "referenced_object_masks": [] }, { "key": "P02-20240210-122334:e8058715589171e57e30925009efee9818f4e3f8", "video_id": "P02-20240210-122334", "question": "What number would I get if I subtracted the number of sponges left after putting the bag away from the number of tiles with fruit on them?", "answer_choice_0": "Two", "answer_choice_1": "Three", "answer_choice_2": "One", "answer_choice_3": "Four", "answer_choice_4": "Five", "answer_id": 2, "answer": "One", "question_type": [ "Temporal Reasoning", "Action Recognition", "Counting", "Object Recognition", "Numerical Reasoning" ], "objects_of_interest": [ "plate-64932742e661fa8d", "plate2-ebb4519e77479a84", "empty packets-688cf092ab1da6c7", "empty transparent packet-1f72fdcdb9854618", "empty transparent packet2-7d3c992d9a0a70d6", "empty packet 4-89e119b446942033", "empty packet 5-785086a34dabf922", "fork-fb888ab5b17daedb", "measuring jug-8de253aa703bd253", "spatula-f409325dba434135", "washing up bowl-e5acdbb5d8e400e5", "washing up liquid-08083f440e3a1e1e", "knife-192755582037f312", "frying pan-66d1fede8722f487", "cutting board-f6820a9b6bb2197d", "tupperware box-78ecadb9df769a90", "tupperware lid-2620440fdb6553cc", "smaller plate-1e25a8ba2a47a847", "larger plate-4cd348e03aa9b6f7", "paring knife-f587f6a18ed9b816", "chef's knife-184f253b825b5443", "small serrated knife-439866a448f31235", "normal knife-cbda343e0c7f16a4", "butter knife-4e1044d9ff5fc820", "table spoon-85a973ab94c5b31a", "large spoon-d13bc1235c3a0133", "bowl-6151fe36af40abd7", "bowl2-b86172c7ce33eb2a", "bag of sponges-d85f8c4d8004f8b8", "baguette-41bdad65b0509598", "bread knife-67c8c6584330b3b9", "quarter of baguette-d872288c54495ad3", "one quarter of baguette-4a3d9ec1c48afa87", "sponge-870455e237e8d974", "cheese grater-30dad965b3119841", "butter knife2-1c1e3a17e4b1b400", "smaller frying pan-e41d2ec1d9183a72", "fork2-55cd49b529a35f7b", "food bin-9ce62ee24b34513b", "drain protector-06cdcdcbcfb5a167" ], "reasoning": "1. I identify two tiles with fruit on them , both of which appear at 00:08 on the backsplash to the left of the sink.\n2. I locate the bag of sponges (bag of sponges-d85f8c4d8004f8b8), which first appears in the drawer at 02:23.\n3. I identify the timestamp for when the user puts the sponges (bag of sponges-d85f8c4d8004f8b8) back into the drawer at 02:29.\n4. I determine that there is one sponge in the bag of sponges (bag of sponges-d85f8c4d8004f8b8) when the user puts it back into the drawer at 02:29.\n5. I subtract one sponge from the two tiles.", "referenced_object_masks": [ "bag of sponges-d85f8c4d8004f8b8" ] }, { "key": "P02-20240210-195833:064942be9500a8b7c97f43c094e09754efbe2872", "video_id": "P02-20240210-195833", "question": "What is the ratio of times I scrub the front of the plate in a counterclockwise circle to the times I tap the washing up bowl on the bottom?", "answer_choice_0": "2:1", "answer_choice_1": "4:3", "answer_choice_2": "2:3", "answer_choice_3": "3:4", "answer_choice_4": "1:1", "answer_id": 0, "answer": "2:1", "question_type": [ "Action Recognition", "Object Recognition", "Counting", "Numerical Reasoning" ], "objects_of_interest": [ "empty glass-69fa070d9f31cfcc", "washing up bowl-f43ec943ffafa0c2", "squash bottle-1fe5a797ae62882a", "washing up liquid bottle-bbf6011dd9a068bc", "butter knife-9c054243c8e98c3b", "sponge-89dfc6bc986e3906", "plate-fcc349e1aec002c7", "tupperware box lid-bb2eac1a91c92d40", "tupperware box-694331787a4a346c", "washing up brush-7af4bb31bfc915ce", "drain protector-bdf450097dc89e86", "food bin-d9f627b6c2651dce" ], "reasoning": "1. I watched the user scrub the front of the plate (plate-fcc349e1aec002c7) with the sponge (sponge-89dfc6bc986e3906) beginning at 00:59.\n2. The user scrubs the front of the plate (plate-fcc349e1aec002c7) in four counter-clockwise circles.\n3. The user finishes with the washing up bowl (washing up bowl-f43ec943ffafa0c2) and turns it over at 02:41.\n4: The user then pats the bottom of the washing up bowl (washing up bowl-f43ec943ffafa0c2) twice.\n5. The ratio is 2:1.", "referenced_object_masks": [ "washing up bowl-f43ec943ffafa0c2", "plate-fcc349e1aec002c7", "sponge-89dfc6bc986e3906" ] }, { "key": "P02-20240210-195833:254bd02fa0cdcb9930477a7b43ba025d14bae6c7", "video_id": "P02-20240210-195833", "question": "Which cleaning tool and dirty dish are not used together?", "answer_choice_0": "Washing up brush plus plate", "answer_choice_1": "Sponge plus plate", "answer_choice_2": "Sponge plus tupperware lid", "answer_choice_3": "Washing up brush plus tupperware container", "answer_choice_4": "Sponge plus butter knife", "answer_id": 0, "answer": "Washing up brush plus plate", "question_type": [ "Counterfactual", "Action Recognition", "Object Recognition" ], "objects_of_interest": [ "empty glass-69fa070d9f31cfcc", "washing up bowl-f43ec943ffafa0c2", "squash bottle-1fe5a797ae62882a", "washing up liquid bottle-bbf6011dd9a068bc", "butter knife-9c054243c8e98c3b", "sponge-89dfc6bc986e3906", "plate-fcc349e1aec002c7", "tupperware box lid-bb2eac1a91c92d40", "tupperware box-694331787a4a346c", "washing up brush-7af4bb31bfc915ce", "drain protector-bdf450097dc89e86", "food bin-d9f627b6c2651dce" ], "reasoning": "1. I watched to see what items the user washes with either the sponge (sponge-89dfc6bc986e3906) or the washing up brush (washing up brush-7af4bb31bfc915ce).\n2. The user uses the sponge (sponge-89dfc6bc986e3906) to wash the plate (plate-fcc349e1aec002c7) at 00:56, the tupperware lid (tupperware box lid-bb2eac1a91c92d40) at 01:11, and the butter knife (butter knife-9c054243c8e98c3b) at 01:35.\n3. The user uses the washing up brush (washing up brush-7af4bb31bfc915ce) to wash the tupperware container (tupperware box-694331787a4a346c) at 01:45.\n4. The washing up brush (washing up brush-7af4bb31bfc915ce) plus the plate (plate-fcc349e1aec002c7) is the one combination that does not occur.", "referenced_object_masks": [ "tupperware box-694331787a4a346c", "butter knife-9c054243c8e98c3b", "tupperware box lid-bb2eac1a91c92d40", "plate-fcc349e1aec002c7", "sponge-89dfc6bc986e3906", "washing up brush-7af4bb31bfc915ce" ] }, { "key": "P02-20240210-195833:63b20cd2b51e322dba7287c7dbf52f40fe2d9af8", "video_id": "P02-20240210-195833", "question": "How many times do I walk over to the radiator while not holding the drain protector?", "answer_choice_0": "1", "answer_choice_1": "2", "answer_choice_2": "0", "answer_choice_3": "4", "answer_choice_4": "3", "answer_id": 1, "answer": "2", "question_type": [ "Action Recognition", "Counterfactual", "Counting", "Numerical Reasoning", "Object Recognition" ], "objects_of_interest": [ "empty glass-69fa070d9f31cfcc", "washing up bowl-f43ec943ffafa0c2", "squash bottle-1fe5a797ae62882a", "washing up liquid bottle-bbf6011dd9a068bc", "butter knife-9c054243c8e98c3b", "sponge-89dfc6bc986e3906", "plate-fcc349e1aec002c7", "tupperware box lid-bb2eac1a91c92d40", "tupperware box-694331787a4a346c", "washing up brush-7af4bb31bfc915ce", "drain protector-bdf450097dc89e86", "food bin-d9f627b6c2651dce" ], "reasoning": "1. I watched to see how many times the user walked over to the radiator .\n2. The user walks to the radiator 3 times, at 03:10, 03:20, and 03:35.\n3. The user is only holding the drain protector (drain protector-bdf450097dc89e86) the second time.", "referenced_object_masks": [ "drain protector-bdf450097dc89e86" ] }, { "key": "P02-20240210-195833:6599dacba0ca94d151864ec6548017743e76550d", "video_id": "P02-20240210-195833", "question": "How many items do I wash while the butter knife is soaking in the washing bowl?", "answer_choice_0": "2", "answer_choice_1": "4", "answer_choice_2": "1", "answer_choice_3": "3", "answer_choice_4": "5", "answer_id": 3, "answer": "3", "question_type": [ "Counting", "Object Recognition", "Temporal Reasoning", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "empty glass-69fa070d9f31cfcc", "washing up bowl-f43ec943ffafa0c2", "squash bottle-1fe5a797ae62882a", "washing up liquid bottle-bbf6011dd9a068bc", "butter knife-9c054243c8e98c3b", "sponge-89dfc6bc986e3906", "plate-fcc349e1aec002c7", "tupperware box lid-bb2eac1a91c92d40", "tupperware box-694331787a4a346c", "washing up brush-7af4bb31bfc915ce", "drain protector-bdf450097dc89e86", "food bin-d9f627b6c2651dce" ], "reasoning": "1. I watched the user place the butter knife (butter knife-9c054243c8e98c3b) into the washing up bowl (washing up bowl-f43ec943ffafa0c2) at 00:41.\n2. At 00:59, the user begins to wash the plate (plate-fcc349e1aec002c7).\n3. At 01:11, the user begins to wash the tupperware lid (tupperware box lid-bb2eac1a91c92d40).\n4. At 01:28, the user begins to wash the tupperware container (tupperware box-694331787a4a346c).\n5. At 01:35, the user removes the butter knife (butter knife-9c054243c8e98c3b) from the washing bowl.\n6. Therefore there are three items that get washed between 00:41 and 01:35.", "referenced_object_masks": [ "butter knife-9c054243c8e98c3b", "washing up bowl-f43ec943ffafa0c2", "tupperware box lid-bb2eac1a91c92d40", "plate-fcc349e1aec002c7", "tupperware box-694331787a4a346c" ] }, { "key": "P02-20240210-195833:c7cb548143f01af7093eefd6feaf97360be0945f", "video_id": "P02-20240210-195833", "question": "Where is the toaster in relation to me when I'm pouring the juice?", "answer_choice_0": "On top of the cutting board to my right", "answer_choice_1": "On top of the washing machine to my left", "answer_choice_2": "On the counter in front of me", "answer_choice_3": "On the counter behind me", "answer_choice_4": "On top of the washing machine to my right", "answer_id": 3, "answer": "On the counter behind me", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Object Recognition" ], "objects_of_interest": [ "empty glass-69fa070d9f31cfcc", "washing up bowl-f43ec943ffafa0c2", "squash bottle-1fe5a797ae62882a", "washing up liquid bottle-bbf6011dd9a068bc", "butter knife-9c054243c8e98c3b", "sponge-89dfc6bc986e3906", "plate-fcc349e1aec002c7", "tupperware box lid-bb2eac1a91c92d40", "tupperware box-694331787a4a346c", "washing up brush-7af4bb31bfc915ce", "drain protector-bdf450097dc89e86", "food bin-d9f627b6c2651dce" ], "reasoning": "1. I watched looking for the user to see the toaster which happens first at 00:02.\n2. The toaster is located on a counter to the left of the stove.\n3. I watched for the user to pour the juice, which occurs at 00:18.\n4. When the user pours the juice, they are standing at the counter opposite the toaster , making its location on the counter behind the user.", "referenced_object_masks": [] }, { "key": "P02-20240210-220650:5699cd38fced564345965f9ce9f5b573d89aa980", "video_id": "P02-20240210-220650", "question": "How many times did I place an item on the counter before I drank water?", "answer_choice_0": "3", "answer_choice_1": "2", "answer_choice_2": "5", "answer_choice_3": "1", "answer_choice_4": "4", "answer_id": 0, "answer": "3", "question_type": [ "Action Recognition", "Object Recognition", "Counting", "Temporal Reasoning" ], "objects_of_interest": [ "glass-87c1220d900e4246", "water bottle-aea964291277e5a8", "bottle-f145486ee093aa49", "lid of bottle-0b7d5e1fd4613a32" ], "reasoning": "1. I found the moment that the user drank water from the glass (glass-87c1220d900e4246) at 00:40.\n2. I then watched from the beginning of the video to 00:40 to determine how many items the user set on the kitchen counter within this timeframe. The user set down the glass (glass-87c1220d900e4246) and the water bottle (water bottle-aea964291277e5a8) at 00:07. The user set down the water bottle (water bottle-aea964291277e5a8) again at 00:33. This is the third and final instance before the user drank at 00:40.", "referenced_object_masks": [ "water bottle-aea964291277e5a8", "glass-87c1220d900e4246" ] }, { "key": "P02-20240210-220650:82bc0f55730f72ddcf7f0610112295ac5b40bf25", "video_id": "P02-20240210-220650", "question": "What is the difference in duration between how long the dining table was to my right and how long the dining table was to my left?", "answer_choice_0": "14 seconds", "answer_choice_1": "18 seconds", "answer_choice_2": "10 seconds", "answer_choice_3": "16 seconds", "answer_choice_4": "12 seconds", "answer_id": 3, "answer": "16 seconds", "question_type": [ "Egocentric & Spatial Perception", "Numerical Reasoning", "Object Recognition", "Situational Awareness", "Temporal Reasoning" ], "objects_of_interest": [ "glass-87c1220d900e4246", "water bottle-aea964291277e5a8", "bottle-f145486ee093aa49", "lid of bottle-0b7d5e1fd4613a32" ], "reasoning": "1. I identified the dining table at 00:07 against the wall facing the kitchen counter. \n2. I marked each duration when the dining table was to the right of the user: 00:17-00:28, and 00:33-00:39 when the user faced the sink, and 00:03 to 00:05, when the user entered the kitchen. This totals 19 seconds.\n3. I marked each duration when the dining table was to the left of the user. This happens from 00:42 to 00:45 while the user dries their hands on a towel . This totals 3 seconds.\n4. The difference between 19 seconds and 3 seconds is 16 seconds.", "referenced_object_masks": [] }, { "key": "P02-20240210-220650:c9a91909174cce63adb24193ef1089f958967d70", "video_id": "P02-20240210-220650", "question": "When I'm filling up the glass with water from the sink, which direction would I have to look to find a clock?", "answer_choice_0": "Right, above the microwave", "answer_choice_1": "Left, next to the oven", "answer_choice_2": "Up, above the sink", "answer_choice_3": "Right, above the dining table", "answer_choice_4": "Behind, on the wall", "answer_id": 4, "answer": "Behind, on the wall", "question_type": [ "Counterfactual", "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness", "Action Recognition" ], "objects_of_interest": [ "glass-87c1220d900e4246", "water bottle-aea964291277e5a8", "bottle-f145486ee093aa49", "lid of bottle-0b7d5e1fd4613a32" ], "reasoning": "1. I searched the surroundings for a clock and found one visible from 00:42 to 00:43, hanging on the wall.\n2. I then found the moment when the user filled up the glass (glass-87c1220d900e4246) from the sink from 00:35 to 00:39. When the user turned completely around, the clock became visible. This means that when the user was at the sink, the clock was behind them.", "referenced_object_masks": [ "glass-87c1220d900e4246" ] }, { "key": "P02-20240210-220650:ccaf505d7e94100db223822278dcd755d84e6ada", "video_id": "P02-20240210-220650", "question": "How many more instances of pouring a liquid into a container would I need to equal the number of instances when the sink is to my left?", "answer_choice_0": "1", "answer_choice_1": "3", "answer_choice_2": "2", "answer_choice_3": "0", "answer_choice_4": "4", "answer_id": 3, "answer": "0", "question_type": [ "Egocentric & Spatial Perception", "Action Recognition", "Numerical Reasoning", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "glass-87c1220d900e4246", "water bottle-aea964291277e5a8", "bottle-f145486ee093aa49", "lid of bottle-0b7d5e1fd4613a32" ], "reasoning": "1. I identified the sink at 00:05 and marked each instance that the sink was to the left of the user. This happens from 00:07 to 00:17, 00:29 to 00:32, and 00:40 to 00:42. This totals 3 separate instances.\n2. I then observed the user each time that they poured liquid into a container. At 00:12, the user pours an orange liquid from a plastic bottle (bottle-f145486ee093aa49) into the glass (glass-87c1220d900e4246). At 00:24, the user fills the clear bottle (water bottle-aea964291277e5a8) with water from the sink. At 00:35, the user fills the glass (glass-87c1220d900e4246) with water from the sink. This totals 3 instances of pouring a liquid into a container.\n3. The number of instances for both occurrences are the same. No more additional instances are needed for them to be equal.", "referenced_object_masks": [ "water bottle-aea964291277e5a8", "bottle-f145486ee093aa49", "glass-87c1220d900e4246" ] }, { "key": "P02-20240210-220650:de4d349bc9f71e2e6439c1e18e17f6c75906b37c", "video_id": "P02-20240210-220650", "question": "What item am I handling when my left hand is closest to the stove?", "answer_choice_0": "A clear bottle", "answer_choice_1": "The lid for the clear bottle", "answer_choice_2": "The lid for the plastic bottle", "answer_choice_3": "A plastic bottle", "answer_choice_4": "A glass", "answer_id": 1, "answer": "The lid for the clear bottle", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness", "Action Recognition" ], "objects_of_interest": [ "glass-87c1220d900e4246", "water bottle-aea964291277e5a8", "bottle-f145486ee093aa49", "lid of bottle-0b7d5e1fd4613a32" ], "reasoning": "1. I identified the location of the stove at 00:05. The front edge of the stove aligns with the left edge of the sink.\n2. I watched the video carefully and focused on the left hand of the user. The user's left hand is closest to the stove at 00:19, when the user sets down the lid of the clear water bottle (lid of bottle-0b7d5e1fd4613a32). The user also picks up the lid of the clear water bottle (lid of bottle-0b7d5e1fd4613a32) from the same location at 00:28. The user's left hand is closest to the stove at these two instances.", "referenced_object_masks": [ "lid of bottle-0b7d5e1fd4613a32" ] }, { "key": "P02-20240211-100250:0e2d2a07a5f58a6bde4b7dd3a96ecc14fbd834b3", "video_id": "P02-20240211-100250", "question": "Why did the green and white mat move?", "answer_choice_0": "The user kicked it away", "answer_choice_1": "The mat did not move", "answer_choice_2": "The vacuum pulled it", "answer_choice_3": "The user picked it up", "answer_choice_4": "There was no mat", "answer_id": 3, "answer": "The user picked it up", "question_type": [ "Object Recognition", "Event Occurence", "Temporal Reasoning", "Action Recognition", "Cause and Effect" ], "objects_of_interest": [ "vacuum cleaner-773697708cc2197f", "mat-eba912ec072326a3", "chair-0c1d08c5e067ceb1", "chair2-09438845ba578928", "shoe-71425b879953f210", "shoe2-9ca2b8cfe24266fc", "food bin-07d4563952e24001", "recycling bin-9e4a3b7b4aee317c", "bin-c8450ffc88146963", "mat2-2eb5710c5ed53870", "brush-64b7511bd7b45307", "head of brush-760d4a6ca1b86f6e", "wand attachment-20270c64deedc8f5", "carrier bag-60e2f676a2eeaa7c" ], "reasoning": "1. I watched the video and found a green and white mat (mat-eba912ec072326a3) at 00:01.\n2. I saw the green and white mat (mat-eba912ec072326a3) move at 00:48.\n3. Just before the green and white mat (mat-eba912ec072326a3) moved at 00:48, I saw the user touch and pick it up.\n4. Therefore, the green and white mat (mat-eba912ec072326a3) moved at 00:48, because the user picked it up.", "referenced_object_masks": [ "mat-eba912ec072326a3" ] }, { "key": "P02-20240211-100250:3a273f896359bd87d3396d35fa44643c54371f8c", "video_id": "P02-20240211-100250", "question": "How many more bins did I move to the right compared to any other direction while I was using the roller attachment of the vacuum cleaner?", "answer_choice_0": "4", "answer_choice_1": "1", "answer_choice_2": "2", "answer_choice_3": "3", "answer_choice_4": "0", "answer_id": 1, "answer": "1", "question_type": [ "Object Recognition", "Temporal Reasoning", "Action Recognition", "Egocentric & Spatial Perception", "Counting", "Numerical Reasoning", "Situational Awareness" ], "objects_of_interest": [ "vacuum cleaner-773697708cc2197f", "mat-eba912ec072326a3", "chair-0c1d08c5e067ceb1", "chair2-09438845ba578928", "shoe-71425b879953f210", "shoe2-9ca2b8cfe24266fc", "food bin-07d4563952e24001", "recycling bin-9e4a3b7b4aee317c", "bin-c8450ffc88146963", "mat2-2eb5710c5ed53870", "brush-64b7511bd7b45307", "head of brush-760d4a6ca1b86f6e", "wand attachment-20270c64deedc8f5", "carrier bag-60e2f676a2eeaa7c" ], "reasoning": "1. I watched the video and found the vacuum cleaner (vacuum cleaner-773697708cc2197f) at 00:12.\n2. I found three bins (food bin-07d4563952e24001, recycling bin-9e4a3b7b4aee317c, bin-c8450ffc88146963) at 00:13.\n3. I watched the user grab and start using the vacuum cleaner (vacuum cleaner-773697708cc2197f) with the roller attachment at 00:25.\n4. I saw the user move one bin (food bin-07d4563952e24001) to the right at 01:58, while using the vacuum cleaner (vacuum cleaner-773697708cc2197f) with the roller attachment.\n5. I saw the user move another bin (recycling bin-9e4a3b7b4aee317c) to the right at 01:59, while using the vacuum cleaner (vacuum cleaner-773697708cc2197f) with the roller attachment.\n6. I saw the user move the third bin (bin-c8450ffc88146963) toward themselves at 02:01, while using the vacuum cleaner (vacuum cleaner-773697708cc2197f) with the roller attachment.\n7. I counted the number of bins (food bin-07d4563952e24001, recycling bin-9e4a3b7b4aee317c) that the user moved to the right while using the vacuum cleaner (vacuum cleaner-773697708cc2197f) with the roller attachment, calculating a total of 2.\n8. I saw the user move the bins (food bin-07d4563952e24001, recycling bin-9e4a3b7b4aee317c, bin-c8450ffc88146963) again from 04:56-05:07, but while they used a different attachment for the vacuum (vacuum cleaner-773697708cc2197f), so I did not include them in my count.\n9. I counted the number of bins that the user moved in any other direction while using the vacuum cleaner (vacuum cleaner-773697708cc2197f) with the roller attachment, calculating a total of 1.\n10. I calculated 2-1=1.\n11. Therefore, the user moved 1 more bin to the right compared to any other direction while they were using the vacuum cleaner (vacuum cleaner-773697708cc2197f).", "referenced_object_masks": [ "recycling bin-9e4a3b7b4aee317c", "food bin-07d4563952e24001", "bin-c8450ffc88146963", "vacuum cleaner-773697708cc2197f" ] }, { "key": "P02-20240211-100250:4d97a17e7b239b99c8b9ed84145d83b55d8b25a8", "video_id": "P02-20240211-100250", "question": "When did I pull a vacuum cleaner for the second time?", "answer_choice_0": "00:28", "answer_choice_1": "00:31", "answer_choice_2": "00:30", "answer_choice_3": "00:32", "answer_choice_4": "00:29", "answer_id": 1, "answer": "00:31", "question_type": [ "Object Recognition", "Action Recognition", "Counting", "Temporal Reasoning" ], "objects_of_interest": [ "vacuum cleaner-773697708cc2197f", "mat-eba912ec072326a3", "chair-0c1d08c5e067ceb1", "chair2-09438845ba578928", "shoe-71425b879953f210", "shoe2-9ca2b8cfe24266fc", "food bin-07d4563952e24001", "recycling bin-9e4a3b7b4aee317c", "bin-c8450ffc88146963", "mat2-2eb5710c5ed53870", "brush-64b7511bd7b45307", "head of brush-760d4a6ca1b86f6e", "wand attachment-20270c64deedc8f5", "carrier bag-60e2f676a2eeaa7c" ], "reasoning": "1. I watched the video and found the vacuum cleaner (vacuum cleaner-773697708cc2197f) at 00:12.\n2. I watched the user grab the vacuum cleaner (vacuum cleaner-773697708cc2197f) at 00:25.\n3. I watched and counted each time the user pulled the vacuum cleaner (vacuum cleaner-773697708cc2197f), with the first being at 00:30 and the second at 00:31.\n4. Therefore, the user pulled the vacuum cleaner (vacuum cleaner-773697708cc2197f) for the second time at 00:31.", "referenced_object_masks": [ "vacuum cleaner-773697708cc2197f" ] }, { "key": "P02-20240211-100250:c0f2e60f90026751fb20d1ffcd9ea563bdbfe993", "video_id": "P02-20240211-100250", "question": "What is on the wall above the tallest lidded bin?", "answer_choice_0": "A calendar", "answer_choice_1": "A shelf", "answer_choice_2": "A checklist", "answer_choice_3": "A portrait", "answer_choice_4": "A window", "answer_id": 0, "answer": "A calendar", "question_type": [ "Object Recognition", "Situational Awareness", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "vacuum cleaner-773697708cc2197f", "mat-eba912ec072326a3", "chair-0c1d08c5e067ceb1", "chair2-09438845ba578928", "shoe-71425b879953f210", "shoe2-9ca2b8cfe24266fc", "food bin-07d4563952e24001", "recycling bin-9e4a3b7b4aee317c", "bin-c8450ffc88146963", "mat2-2eb5710c5ed53870", "brush-64b7511bd7b45307", "head of brush-760d4a6ca1b86f6e", "wand attachment-20270c64deedc8f5", "carrier bag-60e2f676a2eeaa7c" ], "reasoning": "1. I watched the video and found the tallest lidded bin (bin-c8450ffc88146963) at 02:00.\n2. At 02:00, I found the wall behind the bin (bin-c8450ffc88146963).\n3. I looked above the bin (bin-c8450ffc88146963) on the wall behind it at 02:00.\n4. At 02:00, I found a calendar hanging above the bin (bin-c8450ffc88146963) on the wall behind it.\n5. Therefore, a calendar is on the wall above the tallest lidded bin (bin-c8450ffc88146963).", "referenced_object_masks": [ "bin-c8450ffc88146963" ] }, { "key": "P02-20240211-100250:f1242f5c90a73cc7ad50e7059bd4849f2b18e091", "video_id": "P02-20240211-100250", "question": "How many times did I touch one white chair before I touched a second white chair?", "answer_choice_0": "2", "answer_choice_1": "3", "answer_choice_2": "5", "answer_choice_3": "4", "answer_choice_4": "1", "answer_id": 1, "answer": "3", "question_type": [ "Object Recognition", "Action Recognition", "Counting" ], "objects_of_interest": [ "vacuum cleaner-773697708cc2197f", "mat-eba912ec072326a3", "chair-0c1d08c5e067ceb1", "chair2-09438845ba578928", "shoe-71425b879953f210", "shoe2-9ca2b8cfe24266fc", "food bin-07d4563952e24001", "recycling bin-9e4a3b7b4aee317c", "bin-c8450ffc88146963", "mat2-2eb5710c5ed53870", "brush-64b7511bd7b45307", "head of brush-760d4a6ca1b86f6e", "wand attachment-20270c64deedc8f5", "carrier bag-60e2f676a2eeaa7c" ], "reasoning": "1. I watched the video and found two white chairs (chair-0c1d08c5e067ceb1, chair2-09438845ba578928) at 00:01.\n2. I watched the user touch a white chair (chair-0c1d08c5e067ceb1) for the first time at 00:57.\n3. I counted a total of 3 times that the user touched the white chair (chair-0c1d08c5e067ceb1) at 00:57, 01:11, and 01:12.\n4. I watched the user touch another white chair (chair2-09438845ba578928) at 01:16.\n5. Therefore, the user touched one white chair (chair-0c1d08c5e067ceb1) 3 times before they touched a second white chair (chair2-09438845ba578928).", "referenced_object_masks": [ "chair-0c1d08c5e067ceb1", "chair2-09438845ba578928" ] }, { "key": "P02-20240211-104948:00150269f467aaf06b9d4210133d10ff40b360c1", "video_id": "P02-20240211-104948", "question": "What would I be looking at while I opened the box of Wheat Bixies if I looked to my left?", "answer_choice_0": "Out the window", "answer_choice_1": "The pantry", "answer_choice_2": "The door to the garage", "answer_choice_3": "The dining room table", "answer_choice_4": "The radiator", "answer_id": 0, "answer": "Out the window", "question_type": [ "Action Recognition", "Object Recognition", "Egocentric & Spatial Perception", "Counterfactual" ], "objects_of_interest": [ "all carrier bags-d54376ea92cf1af7", "rightmost carrier bag-682cb18b5ed5f5c4", "frangipane-8b43365d1b09c985", "frangipane2-5580c512cda07492", "can of pringles-d3635d3dc1b5531b", "can of pringles2-f67789528aff7ce5", "swiss roll cake-2230fbc143c96017", "packet of apple danish pastries-99bdab3a838a194b", "quorn packets-cf7f23bceaafa8eb", "quorn packet2-5eadb26b12a4f457", "leftmost carrier bag-fa25752cfec8ad6f", "pizza slice-a8a05eb362ca3ec2", "roll of puff pastry-0cda4e2edf3105b3", "can of pineapples-780fa17dc5400ad6", "can of pineapples2-ca9fe51af34643e4", "can of pineapples3-3f2b3a9321b99236", "can of pineapples4-6a9383c4c87b3fe0", "carton of yogurt-505589367156c32a", "fourth bag-92c8e381f75e6e0a", "can of pineapples5-70c725e63a38d8eb", "can of pineapples6-d78d809c291423ad", "scales-35da3bed55898c36", "crisps bag-7e4dcb8a5694e8d3", "third bag-64b6237fac95cdc6", "fifth bag-1239c880d162fd85", "spoon-6f80a71a3bb48afe", "garage door key-4cd3842da5e8fe96", "bowl-4032c0d45fabcd05", "bowl2-0a2d843cb6b4ab1e", "towel-eab548cdee23cd03", "box of wheat bixies-854ff77cb87576bc", "rightmost stack of wheat bixies-efda54b5a290c087", "serrated knife-c615c55478a0cd65", "wheat bixie-bbb2d07b2328c798", "milk bottle-d1cedd5f79885778", "multivitamin bottle-deeb23d95841fad6", "multivitamin pill-3d89ede390d4b237" ], "reasoning": "1. I watched for the user to open the box of Wheat Bixies (box of wheat bixies-854ff77cb87576bc), which the user starts to do at 03:57.\n2. The user is standing facing the counter that looks out across the room at that time stamp.\n3. Just previously to this, the user turned right from the opposite counter, turning past the sink and the window over it at 03:55.\n4. At that position, if the user looked to their left, they would see out the window over the sink.", "referenced_object_masks": [ "box of wheat bixies-854ff77cb87576bc" ] }, { "key": "P02-20240211-104948:267e2ed28b1bc4e1b913036d9648aecc89ed7ee0", "video_id": "P02-20240211-104948", "question": "What is the ratio of times I turn the flattened rightmost carrier bag to the right and to the left?", "answer_choice_0": "2:1", "answer_choice_1": "1:2", "answer_choice_2": "1:3", "answer_choice_3": "3:1", "answer_choice_4": "3:4", "answer_id": 1, "answer": "1:2", "question_type": [ "Action Recognition", "Object Recognition", "Numerical Reasoning", "Counting" ], "objects_of_interest": [ "all carrier bags-d54376ea92cf1af7", "rightmost carrier bag-682cb18b5ed5f5c4", "frangipane-8b43365d1b09c985", "frangipane2-5580c512cda07492", "can of pringles-d3635d3dc1b5531b", "can of pringles2-f67789528aff7ce5", "swiss roll cake-2230fbc143c96017", "packet of apple danish pastries-99bdab3a838a194b", "quorn packets-cf7f23bceaafa8eb", "quorn packet2-5eadb26b12a4f457", "leftmost carrier bag-fa25752cfec8ad6f", "pizza slice-a8a05eb362ca3ec2", "roll of puff pastry-0cda4e2edf3105b3", "can of pineapples-780fa17dc5400ad6", "can of pineapples2-ca9fe51af34643e4", "can of pineapples3-3f2b3a9321b99236", "can of pineapples4-6a9383c4c87b3fe0", "carton of yogurt-505589367156c32a", "fourth bag-92c8e381f75e6e0a", "can of pineapples5-70c725e63a38d8eb", "can of pineapples6-d78d809c291423ad", "scales-35da3bed55898c36", "crisps bag-7e4dcb8a5694e8d3", "third bag-64b6237fac95cdc6", "fifth bag-1239c880d162fd85", "spoon-6f80a71a3bb48afe", "garage door key-4cd3842da5e8fe96", "bowl-4032c0d45fabcd05", "bowl2-0a2d843cb6b4ab1e", "towel-eab548cdee23cd03", "box of wheat bixies-854ff77cb87576bc", "rightmost stack of wheat bixies-efda54b5a290c087", "serrated knife-c615c55478a0cd65", "wheat bixie-bbb2d07b2328c798", "milk bottle-d1cedd5f79885778", "multivitamin bottle-deeb23d95841fad6", "multivitamin pill-3d89ede390d4b237" ], "reasoning": "1. I watched for the user to pick up the rightmost carrier bag (rightmost carrier bag-682cb18b5ed5f5c4) at 00:49, and flatten it at 00:51.\n2. Once flattened, the user turns the rightmost carrier bag (rightmost carrier bag-682cb18b5ed5f5c4) once to the right from 00:51 to 00:52. \n3. The user then turns the rightmost carrier bag (rightmost carrier bag-682cb18b5ed5f5c4) twice to the left from 00:52 to 00:53, before moving on to fold it.\n4. This makes for a ratio of 1:2.", "referenced_object_masks": [ "rightmost carrier bag-682cb18b5ed5f5c4" ] }, { "key": "P02-20240211-104948:352c27b987dbe0d4b4076898cd74f9172c991da3", "video_id": "P02-20240211-104948", "question": "Why can't I put the 5th can of pineapple in the green drawer in the pantry?", "answer_choice_0": "It's too crowded", "answer_choice_1": "I can't reach it", "answer_choice_2": "The drawer is stuck", "answer_choice_3": "The can is too big", "answer_choice_4": "There is no green drawer", "answer_id": 3, "answer": "The can is too big", "question_type": [ "Action Recognition", "Object Recognition", "Egocentric & Spatial Perception", "Situational Awareness" ], "objects_of_interest": [ "all carrier bags-d54376ea92cf1af7", "rightmost carrier bag-682cb18b5ed5f5c4", "frangipane-8b43365d1b09c985", "frangipane2-5580c512cda07492", "can of pringles-d3635d3dc1b5531b", "can of pringles2-f67789528aff7ce5", "swiss roll cake-2230fbc143c96017", "packet of apple danish pastries-99bdab3a838a194b", "quorn packets-cf7f23bceaafa8eb", "quorn packet2-5eadb26b12a4f457", "leftmost carrier bag-fa25752cfec8ad6f", "pizza slice-a8a05eb362ca3ec2", "roll of puff pastry-0cda4e2edf3105b3", "can of pineapples-780fa17dc5400ad6", "can of pineapples2-ca9fe51af34643e4", "can of pineapples3-3f2b3a9321b99236", "can of pineapples4-6a9383c4c87b3fe0", "carton of yogurt-505589367156c32a", "fourth bag-92c8e381f75e6e0a", "can of pineapples5-70c725e63a38d8eb", "can of pineapples6-d78d809c291423ad", "scales-35da3bed55898c36", "crisps bag-7e4dcb8a5694e8d3", "third bag-64b6237fac95cdc6", "fifth bag-1239c880d162fd85", "spoon-6f80a71a3bb48afe", "garage door key-4cd3842da5e8fe96", "bowl-4032c0d45fabcd05", "bowl2-0a2d843cb6b4ab1e", "towel-eab548cdee23cd03", "box of wheat bixies-854ff77cb87576bc", "rightmost stack of wheat bixies-efda54b5a290c087", "serrated knife-c615c55478a0cd65", "wheat bixie-bbb2d07b2328c798", "milk bottle-d1cedd5f79885778", "multivitamin bottle-deeb23d95841fad6", "multivitamin pill-3d89ede390d4b237" ], "reasoning": "1. I watched for the user to attempt to put the 5th can of pineapple (can of pineapples5-70c725e63a38d8eb) away. \n2. The user picks up the 5th can of pineapple (can of pineapples5-70c725e63a38d8eb) at 01:16 and walks to the pantry.\n3. After opening the pantry door, the user crouches down and pulls the green door open at 01:26.\n4. The user puts the 5th can of pineapple (can of pineapples5-70c725e63a38d8eb) in the green drawer at 01:26, but the green drawer will not completely close because the 5th can of pineapple (can of pineapples5-70c725e63a38d8eb) is too big.", "referenced_object_masks": [ "can of pineapples5-70c725e63a38d8eb" ] }, { "key": "P02-20240211-104948:35f03fc6789ac0c99824ec349bf94e5641ced68f", "video_id": "P02-20240211-104948", "question": "What two places do I decide not to put the Pingles cans?", "answer_choice_0": "The cabinet above the toaster and the drawer to the left of the stove", "answer_choice_1": "The pantry and the garage", "answer_choice_2": "The pantry and the cabinet above the toaster", "answer_choice_3": "The cabinet above the toaster and the garage.\"", "answer_choice_4": "The drawer to the left of the stove and the garage", "answer_id": 2, "answer": "The pantry and the cabinet above the toaster", "question_type": [ "Counterfactual", "Action Recognition", "Object Recognition" ], "objects_of_interest": [ "all carrier bags-d54376ea92cf1af7", "rightmost carrier bag-682cb18b5ed5f5c4", "frangipane-8b43365d1b09c985", "frangipane2-5580c512cda07492", "can of pringles-d3635d3dc1b5531b", "can of pringles2-f67789528aff7ce5", "swiss roll cake-2230fbc143c96017", "packet of apple danish pastries-99bdab3a838a194b", "quorn packets-cf7f23bceaafa8eb", "quorn packet2-5eadb26b12a4f457", "leftmost carrier bag-fa25752cfec8ad6f", "pizza slice-a8a05eb362ca3ec2", "roll of puff pastry-0cda4e2edf3105b3", "can of pineapples-780fa17dc5400ad6", "can of pineapples2-ca9fe51af34643e4", "can of pineapples3-3f2b3a9321b99236", "can of pineapples4-6a9383c4c87b3fe0", "carton of yogurt-505589367156c32a", "fourth bag-92c8e381f75e6e0a", "can of pineapples5-70c725e63a38d8eb", "can of pineapples6-d78d809c291423ad", "scales-35da3bed55898c36", "crisps bag-7e4dcb8a5694e8d3", "third bag-64b6237fac95cdc6", "fifth bag-1239c880d162fd85", "spoon-6f80a71a3bb48afe", "garage door key-4cd3842da5e8fe96", "bowl-4032c0d45fabcd05", "bowl2-0a2d843cb6b4ab1e", "towel-eab548cdee23cd03", "box of wheat bixies-854ff77cb87576bc", "rightmost stack of wheat bixies-efda54b5a290c087", "serrated knife-c615c55478a0cd65", "wheat bixie-bbb2d07b2328c798", "milk bottle-d1cedd5f79885778", "multivitamin bottle-deeb23d95841fad6", "multivitamin pill-3d89ede390d4b237" ], "reasoning": "1. I watched for the user to put away the 1st can of Pringles (can of pringles-d3635d3dc1b5531b) and the 2nd can of Pringles (can of pringles2-f67789528aff7ce5).\n2. The user takes the 1st can of Pringles (can of pringles-d3635d3dc1b5531b) and the 2nd can of Pringles (can of pringles2-f67789528aff7ce5) out of the rightmost carrier bag (rightmost carrier bag-682cb18b5ed5f5c4) and places them on the counter from 01:12-01:15.\n3. The user picks the cans of Pringles (can of pringles-d3635d3dc1b5531b, can of pringles2-f67789528aff7ce5) again at 02:37.\n4. At 2:40, the user opens the pantry door as if to put the Pringles cans (can of pringles-d3635d3dc1b5531b, can of pringles2-f67789528aff7ce5)] inside, but shuts the door at 02:45.\n5. At 02:47, the user opens the cabinet above the toaster , again as if to put the Pringles cans (can of pringles-d3635d3dc1b5531b, can of pringles2-f67789528aff7ce5) inside, but instead puts the Pringles cans (can of pringles-d3635d3dc1b5531b, can of pringles2-f67789528aff7ce5) on top of the cabinet at 02:49-02:50.", "referenced_object_masks": [ "can of pringles-d3635d3dc1b5531b", "can of pringles2-f67789528aff7ce5", "rightmost carrier bag-682cb18b5ed5f5c4" ] }, { "key": "P02-20240211-104948:7daccc1763ba42f42d159c0b4d4e36f00085ca9f", "video_id": "P02-20240211-104948", "question": "Where would the Quorn packets go if I put them in the utility to the right of the utility I put them in?", "answer_choice_0": "The refrigerator", "answer_choice_1": "The dryer", "answer_choice_2": "The stove", "answer_choice_3": "The washing machine", "answer_choice_4": "The toaster", "answer_id": 1, "answer": "The dryer", "question_type": [ "Egocentric & Spatial Perception", "Action Recognition", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "all carrier bags-d54376ea92cf1af7", "rightmost carrier bag-682cb18b5ed5f5c4", "frangipane-8b43365d1b09c985", "frangipane2-5580c512cda07492", "can of pringles-d3635d3dc1b5531b", "can of pringles2-f67789528aff7ce5", "swiss roll cake-2230fbc143c96017", "packet of apple danish pastries-99bdab3a838a194b", "quorn packets-cf7f23bceaafa8eb", "quorn packet2-5eadb26b12a4f457", "leftmost carrier bag-fa25752cfec8ad6f", "pizza slice-a8a05eb362ca3ec2", "roll of puff pastry-0cda4e2edf3105b3", "can of pineapples-780fa17dc5400ad6", "can of pineapples2-ca9fe51af34643e4", "can of pineapples3-3f2b3a9321b99236", "can of pineapples4-6a9383c4c87b3fe0", "carton of yogurt-505589367156c32a", "fourth bag-92c8e381f75e6e0a", "can of pineapples5-70c725e63a38d8eb", "can of pineapples6-d78d809c291423ad", "scales-35da3bed55898c36", "crisps bag-7e4dcb8a5694e8d3", "third bag-64b6237fac95cdc6", "fifth bag-1239c880d162fd85", "spoon-6f80a71a3bb48afe", "garage door key-4cd3842da5e8fe96", "bowl-4032c0d45fabcd05", "bowl2-0a2d843cb6b4ab1e", "towel-eab548cdee23cd03", "box of wheat bixies-854ff77cb87576bc", "rightmost stack of wheat bixies-efda54b5a290c087", "serrated knife-c615c55478a0cd65", "wheat bixie-bbb2d07b2328c798", "milk bottle-d1cedd5f79885778", "multivitamin bottle-deeb23d95841fad6", "multivitamin pill-3d89ede390d4b237" ], "reasoning": "1. I watched for the user to pick up the Quorn packets (quorn packets-cf7f23bceaafa8eb, quorn packet2-5eadb26b12a4f457), which they retrieve from the rightmost carrier bag (rightmost carrier bag-682cb18b5ed5f5c4) at 00:19.\n2. The user then turns to the right and puts the Quorn packets (quorn packets-cf7f23bceaafa8eb, quorn packet2-5eadb26b12a4f457) on the counter to the left of the stove.\n3. At 03:07, the user picks up the Quorn packets (quorn packets-cf7f23bceaafa8eb, quorn packet2-5eadb26b12a4f457) again and goes into the garage with them.\n4. At 03:13, the user opens a small freezer and puts the Quorn packets (quorn packets-cf7f23bceaafa8eb, quorn packet2-5eadb26b12a4f457) inside.\n5. To the right of the freezer that the Quorn packets (quorn packets-cf7f23bceaafa8eb, quorn packet2-5eadb26b12a4f457) are put into is a dryer.\n6. I could see a washing machine in the kitchen at 00:03, so I inferred that the utility next to the freezer was a dryer and not a washing machine.", "referenced_object_masks": [ "rightmost carrier bag-682cb18b5ed5f5c4", "quorn packets-cf7f23bceaafa8eb", "quorn packet2-5eadb26b12a4f457" ] }, { "key": "P02-20240211-111219:2035be0cce36cdd4023791f0efefdb52cd23d3fe", "video_id": "P02-20240211-111219", "question": "According to the video, what three items are the same color?", "answer_choice_0": "Stovetop, countertop, and microwave door", "answer_choice_1": "Grass, electric kettle, and paper towels", "answer_choice_2": "Knife handle, spoon, and kitchen walls", "answer_choice_3": "The cookbook, bowl, and cabinet doors", "answer_choice_4": "The cap of the dish soap, bottom of the unused sponge, and brush handle", "answer_id": 4, "answer": "The cap of the dish soap, bottom of the unused sponge, and brush handle", "question_type": [ "Object Recognition", "State Changes", "Goal Reasoning", "Situational Awareness" ], "objects_of_interest": [ "bowl-764f222304103f9e", "cookbook-6312b3693a939863", "phone-7be8b1e7d88f0f61", "glass-d78ab19a0902ff01", "spoon-fe40c4605e58c78e", "citrus juicer-6b8f54a251687d81", "mixing bowl-f01376f10fc4ea3c", "stack of colanders-a48a7cb02f99245e", "kitchen roll-937750c49c1fbf9b", "tissue-511eb9e8b3ca8758", "washing up basin-ae78a70ef71ee52d", "washing up liquid bottle-77bb5e7f234a887e", "sponge-2a4a7f58f048ad14", "tea towel-70debc3f33ea6f0f", "container of sugar-95f982ea59a0247e", "measuring jug-ba5445105fa5cfa3", "bowl2-0ff6a1316ca66591" ], "reasoning": "1. I watched the video to note the cookbook (cookbook-6312b3693a939863), visible at 00:06, is white and red.\n2. I continued watching and found that the bowl at 0:44 is clear, so the first option doesn't work.\n3. I watched until 1:11 to see the dish soap bottle (washing up liquid bottle-77bb5e7f234a887e) had a red cap, along with the sponge and brush handle , both of which are in frame at this time.\n4. I watched the video to confirm that none of the other options are the same color.", "referenced_object_masks": [ "washing up liquid bottle-77bb5e7f234a887e", "cookbook-6312b3693a939863" ] }, { "key": "P02-20240211-111219:2d05a94e37d06c111a47f3414c05726d721e572f", "video_id": "P02-20240211-111219", "question": "For how many total seconds did I dry the clear bowl in the video?", "answer_choice_0": "13 seconds", "answer_choice_1": "20 seconds", "answer_choice_2": "14 seconds", "answer_choice_3": "19 seconds", "answer_choice_4": "16 seconds", "answer_id": 4, "answer": "16 seconds", "question_type": [ "Counterfactual", "Spatial Perception", "Numerical Reasoning", "Object Recognition" ], "objects_of_interest": [ "bowl-764f222304103f9e", "cookbook-6312b3693a939863", "phone-7be8b1e7d88f0f61", "glass-d78ab19a0902ff01", "spoon-fe40c4605e58c78e", "citrus juicer-6b8f54a251687d81", "mixing bowl-f01376f10fc4ea3c", "stack of colanders-a48a7cb02f99245e", "kitchen roll-937750c49c1fbf9b", "tissue-511eb9e8b3ca8758", "washing up basin-ae78a70ef71ee52d", "washing up liquid bottle-77bb5e7f234a887e", "sponge-2a4a7f58f048ad14", "tea towel-70debc3f33ea6f0f", "container of sugar-95f982ea59a0247e", "measuring jug-ba5445105fa5cfa3", "bowl2-0ff6a1316ca66591" ], "reasoning": "1. I watched the video and noted that the user extracted the clear bowl from the stack of colanders (stack of colanders-a48a7cb02f99245e) from 00:44-00:45.\n2. I then continued to watch the video and noted that the user washes the bowl from 1:35-1:52.\n3. Continuing, I noted that the user grabbed a hand towel (tissue-511eb9e8b3ca8758) and dried the bowl from 2:00-2:08, for a total of 8 seconds.\n4. I let the video progress, and noted that the user returns to dry the bowl with the towel again from 2:19-2:27, another 8 seconds.\n5. Total time drying the bowl is 16 seconds (8+8).", "referenced_object_masks": [ "stack of colanders-a48a7cb02f99245e", "tissue-511eb9e8b3ca8758" ] }, { "key": "P02-20240211-111219:337b8ac6b3b87025020ff43965a9b79e013bdf52", "video_id": "P02-20240211-111219", "question": "How many mugs can I see in the cabinet from which I grab the large, clear bowl?", "answer_choice_0": "7", "answer_choice_1": "10", "answer_choice_2": "6", "answer_choice_3": "8", "answer_choice_4": "4", "answer_id": 0, "answer": "7", "question_type": [ "Counting", "Object Recognition", "Event Occurence" ], "objects_of_interest": [ "bowl-764f222304103f9e", "cookbook-6312b3693a939863", "phone-7be8b1e7d88f0f61", "glass-d78ab19a0902ff01", "spoon-fe40c4605e58c78e", "citrus juicer-6b8f54a251687d81", "mixing bowl-f01376f10fc4ea3c", "stack of colanders-a48a7cb02f99245e", "kitchen roll-937750c49c1fbf9b", "tissue-511eb9e8b3ca8758", "washing up basin-ae78a70ef71ee52d", "washing up liquid bottle-77bb5e7f234a887e", "sponge-2a4a7f58f048ad14", "tea towel-70debc3f33ea6f0f", "container of sugar-95f982ea59a0247e", "measuring jug-ba5445105fa5cfa3", "bowl2-0ff6a1316ca66591" ], "reasoning": "1. I watched the video to identify when the user opens the cabinet with the stack of colanders (stack of colanders-a48a7cb02f99245e), which occurs from 00:39-00:45.\n2. I continued watching the video to see that at 00:44-00:45, the user puts the stack of colanders (stack of colanders-a48a7cb02f99245e) back in the cabinet after grabbing a clear bowl from the bottom of the stack.\n3. I rewound back to 00:39 to watch the user open the cabinet.\n4. At this point, I counted 7 mugs sitting on the bottom shelf of the cabinet.", "referenced_object_masks": [ "stack of colanders-a48a7cb02f99245e" ] }, { "key": "P02-20240211-111219:7050ca682e3e4c9028bc02ca031e307fa7ea3d17", "video_id": "P02-20240211-111219", "question": "At what point of the video do I start to wash the item I set by the sink?", "answer_choice_0": "1:46", "answer_choice_1": "4:12", "answer_choice_2": "3:31", "answer_choice_3": "2:24", "answer_choice_4": "3:13", "answer_id": 4, "answer": "3:13", "question_type": [ "Temporal Reasoning", "Event Occurence", "Object Recognition" ], "objects_of_interest": [ "bowl-764f222304103f9e", "cookbook-6312b3693a939863", "phone-7be8b1e7d88f0f61", "glass-d78ab19a0902ff01", "spoon-fe40c4605e58c78e", "citrus juicer-6b8f54a251687d81", "mixing bowl-f01376f10fc4ea3c", "stack of colanders-a48a7cb02f99245e", "kitchen roll-937750c49c1fbf9b", "tissue-511eb9e8b3ca8758", "washing up basin-ae78a70ef71ee52d", "washing up liquid bottle-77bb5e7f234a887e", "sponge-2a4a7f58f048ad14", "tea towel-70debc3f33ea6f0f", "container of sugar-95f982ea59a0247e", "measuring jug-ba5445105fa5cfa3", "bowl2-0ff6a1316ca66591" ], "reasoning": "1. I started the video and noted that the first item the user placed by the sink was a white bowl (bowl-764f222304103f9e) with a spoon in it (spoon-fe40c4605e58c78e) from 0:01-0:04.\n2. I continued to watch the video to note that the user begins washing dishes at 1:15.\n3. Further along in the video, I noted that the user began washing the bowl (bowl-764f222304103f9e) with the spoon (spoon-fe40c4605e58c78e) at 3:13.", "referenced_object_masks": [ "bowl-764f222304103f9e", "spoon-fe40c4605e58c78e" ] }, { "key": "P02-20240211-111219:9d7fece7fefda4a3f02759f288302cc7b6066742", "video_id": "P02-20240211-111219", "question": "What is the fourth to last picture shown on the cookbook when I am flipping through its pages from 00:29-00:34?", "answer_choice_0": "Pancakes", "answer_choice_1": "Red preserves", "answer_choice_2": "Pies", "answer_choice_3": "Biscuits", "answer_choice_4": "Juice", "answer_id": 1, "answer": "Red preserves", "question_type": [ "Counterfactual", "Spatial Perception", "Reading", "Object Recognition" ], "objects_of_interest": [ "bowl-764f222304103f9e", "cookbook-6312b3693a939863", "phone-7be8b1e7d88f0f61", "glass-d78ab19a0902ff01", "spoon-fe40c4605e58c78e", "citrus juicer-6b8f54a251687d81", "mixing bowl-f01376f10fc4ea3c", "stack of colanders-a48a7cb02f99245e", "kitchen roll-937750c49c1fbf9b", "tissue-511eb9e8b3ca8758", "washing up basin-ae78a70ef71ee52d", "washing up liquid bottle-77bb5e7f234a887e", "sponge-2a4a7f58f048ad14", "tea towel-70debc3f33ea6f0f", "container of sugar-95f982ea59a0247e", "measuring jug-ba5445105fa5cfa3", "bowl2-0ff6a1316ca66591" ], "reasoning": "1. I watched the video from 0:29-0:34 to find when the user flips through the cookbook (cookbook-6312b3693a939863).\n2. I noticed that the last picture shown on the cookbook (cookbook-6312b3693a939863), shown from 00:33-00:34, shows some pastries.\n3. I looked for the second to last picture shown in the cookbook (cookbook-6312b3693a939863), and find it at 00:32.\n4. At this timeframe, I see a spoon in a glass bowl filled with a light-colored food.\n5. I looked for the third to last picture shown in the cookbook (cookbook-6312b3693a939863), and found it at 00:31.\n6. I found it to be a set of small pies.\n7. I looked for the fourth to last picture shown in the cookbook (cookbook-6312b3693a939863), and I found it at 00:31, just before the picture of the pies.\n8. It is a picture of a glass jar of red preserves with a white lacy covering.\n9. Therefore, the answer is red preserves.", "referenced_object_masks": [ "cookbook-6312b3693a939863" ] }, { "key": "P02-20240211-111822:017bd7ac5ef7e2903b8b99ffe1bbc024951e2b16", "video_id": "P02-20240211-111822", "question": "How many lemons did the user shave with the grater?", "answer_choice_0": "0", "answer_choice_1": "2", "answer_choice_2": "3", "answer_choice_3": "1", "answer_choice_4": "4", "answer_id": 1, "answer": "2", "question_type": [ "Action Recognition", "Object Recognition", "Counting" ], "objects_of_interest": [ "phone-6744b55e629fbace", "hand blender attachment-2167c6bed0eb93dd", "hand blender-ca90da5b47137ab7", "hand blender cup-22f9b7158abd04a1", "washing up liquid bottle-825d5fd852e54610", "lid of hand blender cup-80b474eb460f7385", "sponge-ed4b1649de908d0a", "tea towel-1d373cdc4aaac937", "grater-8d788e8f028fc2bc", "plate-61db953b4a54c394", "bag of lemons-1c956539b3df3967", "punnets of raspberries-32aabb197004447a", "serrated knife-df2254f61cacc2ff", "lemon-b00bb617003dacac", "lemon2-56e4b71d5bfca0e3", "lemon3-ca1ff11afec1db47", "cutting board-dbfc7c6ebc5d3d47", "box of scales-4f8e8e0d3c43b823", "scales instructions-dfcf034f2d367837", "scales-d96dcf1060d55db3", "recipe book-728a08ba785943b9", "punnets of raspberries2-19b69198c95c5c56", "citrus juicer-44122046a14f6348", "food bin-a6d6362547eb1e8d", "lemon half-74e253ad3d7423cc", "lemon half2-1ab63128991488b9", "lemon half3-b9bae9722227adce", "container of sugar-2ebbc2ee15abf30d", "butter packet-ccf210777c7099ac", "butter packet2-6e451728af67fbb2", "stick of butter-fc1ce29145811450", "portion of butter-7b695c68d52254ec", "butter portion3-0a1aba42119aa8ea", "large mixing bowl-6a273b47425454a7", "saucepan-8e0305e33e07dc6b", "bigger saucepan-31b73dac0d238414", "lid of saucepan-fb99ded0fbc1d80a", "foam bottom of raspberry punnet-59e8f1b83bbab430", "plastic film lid-c869d8a729e2648a", "large plastic spoon-9dae0b46a533a7e2", "lid of sugar container-bf09b0c200c5d9a5", "bowl-43da25bf8c60818c", "teaspoon-1e46e1937149d3df", "kitchen roll holder-b7cf8787fcd4ac94", "kitchen towel-2f75ecd03f504e76", "egg carton-b74057bab01ab1c7", "egg-1e0a9c3f0f18b7bc", "egg2-3604159c0bf0f094", "egg3-58b911b052079207", "egg4-f1e0e13f8ebb09df", "fork-7eea868053353450", "cardboard container-e6e9d58ce4242517", "black washing up bowl-fb98537fa9fc5097", "frying pan-5fbfc013bc7c4637", "frying pan2-3cee48bd4ce2ea88", "oven mitts1-6faa4e8f0eee3903", "oven mitt2-b8b670aba479ef69", "smaller frying pan-541b208539c17909", "plate2-de20f7abcee1dca1", "plate3-403f22906468f4eb", "bowl2-d2817ac082cc7f9c", "bowl3-fe4ac808869fbce4", "lid of tupperware-4cad87cfbdda3072", "tupperware box-e7540763f2ce9433", "spatula-d292bed0be82a3cc", "bread knife-42811ef2c0816983", "cutlery-d0b68955c61b0b81", "butter knife-b63d9196d4bb5dc3", "other butter knife-0baf56dfd3c4bcd0", "knife-e0bd6d5a320bd2ad", "fork2-77e220ac85c0948b", "washing up brush-95ed7efe8b675933", "small mixing bowl-87743c3988bbbdcc", "citrus juicer guard-de5cb9b367d6534b", "glass countertop protector-969306577d8aba0b" ], "reasoning": "1. I watch the video and first notice the user begin shaving a lemon (lemon-b00bb617003dacac) with the grater at 02:57. This is the first lemon.\n2. I continue watching the video and notice the user begins to shave a second lemon (lemon2-56e4b71d5bfca0e3) at 04:17.\n3. Continuing to watch the video I then notice the user put the grater away at 06:06 as well as the rest of the lemons at 06:40 and conclude the total number of lemons grated is two.", "referenced_object_masks": [ "lemon2-56e4b71d5bfca0e3", "lemon-b00bb617003dacac" ] }, { "key": "P02-20240211-111822:1cad15c05dd94accea6d26f084a3b0f0ef1bf494", "video_id": "P02-20240211-111822", "question": "Between 10:30 and 11:00 in the video, how many times does my right arm partially obscure the handle of the immersion blender?", "answer_choice_0": "37", "answer_choice_1": "21", "answer_choice_2": "39", "answer_choice_3": "32", "answer_choice_4": ", 28", "answer_id": 3, "answer": "32", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Event Occurence", "Counting" ], "objects_of_interest": [ "phone-6744b55e629fbace", "hand blender attachment-2167c6bed0eb93dd", "hand blender-ca90da5b47137ab7", "hand blender cup-22f9b7158abd04a1", "washing up liquid bottle-825d5fd852e54610", "lid of hand blender cup-80b474eb460f7385", "sponge-ed4b1649de908d0a", "tea towel-1d373cdc4aaac937", "grater-8d788e8f028fc2bc", "plate-61db953b4a54c394", "bag of lemons-1c956539b3df3967", "punnets of raspberries-32aabb197004447a", "serrated knife-df2254f61cacc2ff", "lemon-b00bb617003dacac", "lemon2-56e4b71d5bfca0e3", "lemon3-ca1ff11afec1db47", "cutting board-dbfc7c6ebc5d3d47", "box of scales-4f8e8e0d3c43b823", "scales instructions-dfcf034f2d367837", "scales-d96dcf1060d55db3", "recipe book-728a08ba785943b9", "punnets of raspberries2-19b69198c95c5c56", "citrus juicer-44122046a14f6348", "food bin-a6d6362547eb1e8d", "lemon half-74e253ad3d7423cc", "lemon half2-1ab63128991488b9", "lemon half3-b9bae9722227adce", "container of sugar-2ebbc2ee15abf30d", "butter packet-ccf210777c7099ac", "butter packet2-6e451728af67fbb2", "stick of butter-fc1ce29145811450", "portion of butter-7b695c68d52254ec", "butter portion3-0a1aba42119aa8ea", "large mixing bowl-6a273b47425454a7", "saucepan-8e0305e33e07dc6b", "bigger saucepan-31b73dac0d238414", "lid of saucepan-fb99ded0fbc1d80a", "foam bottom of raspberry punnet-59e8f1b83bbab430", "plastic film lid-c869d8a729e2648a", "large plastic spoon-9dae0b46a533a7e2", "lid of sugar container-bf09b0c200c5d9a5", "bowl-43da25bf8c60818c", "teaspoon-1e46e1937149d3df", "kitchen roll holder-b7cf8787fcd4ac94", "kitchen towel-2f75ecd03f504e76", "egg carton-b74057bab01ab1c7", "egg-1e0a9c3f0f18b7bc", "egg2-3604159c0bf0f094", "egg3-58b911b052079207", "egg4-f1e0e13f8ebb09df", "fork-7eea868053353450", "cardboard container-e6e9d58ce4242517", "black washing up bowl-fb98537fa9fc5097", "frying pan-5fbfc013bc7c4637", "frying pan2-3cee48bd4ce2ea88", "oven mitts1-6faa4e8f0eee3903", "oven mitt2-b8b670aba479ef69", "smaller frying pan-541b208539c17909", "plate2-de20f7abcee1dca1", "plate3-403f22906468f4eb", "bowl2-d2817ac082cc7f9c", "bowl3-fe4ac808869fbce4", "lid of tupperware-4cad87cfbdda3072", "tupperware box-e7540763f2ce9433", "spatula-d292bed0be82a3cc", "bread knife-42811ef2c0816983", "cutlery-d0b68955c61b0b81", "butter knife-b63d9196d4bb5dc3", "other butter knife-0baf56dfd3c4bcd0", "knife-e0bd6d5a320bd2ad", "fork2-77e220ac85c0948b", "washing up brush-95ed7efe8b675933", "small mixing bowl-87743c3988bbbdcc", "citrus juicer guard-de5cb9b367d6534b", "glass countertop protector-969306577d8aba0b" ], "reasoning": "1. I got the video to 10:30, noting that at this time, the user's right arm is partially obstructing the handle of the immersion blender (hand blender-ca90da5b47137ab7), meaning this starting instance doesn't count.\n2. I then started to watch the video, observing the user move their right arm back and forth to juice the lemon halves (lemon half-74e253ad3d7423cc). Each time the arm crosses in front of the immersion blender (hand blender-ca90da5b47137ab7) it counts as one time for the question, and from 10:30-10:38, I counted 12 times the blender (hand blender-ca90da5b47137ab7) was obstructed. \n3. From 10:38-10:41, the user throws the first lemon half (lemon half-74e253ad3d7423cc) into the food bin (food bin-a6d6362547eb1e8d) before grabbing the second lemon half (lemon half2-1ab63128991488b9).\n4. The user places the second lemon half (lemon half2-1ab63128991488b9) on the juicer (citrus juicer-44122046a14f6348) and juices from 10:42-10:57, partially obscuring the immersion blender (hand blender-ca90da5b47137ab7) 20 more times.\n5. As I finished watching the span, I saw the user pick up the lemon half (emon half2-1ab63128991488b9) and throw it in the food bin (food bin-a6d6362547eb1e8d) from 10:57-11:00.\n6. 12 + 20 = 32 times.", "referenced_object_masks": [ "hand blender-ca90da5b47137ab7", "food bin-a6d6362547eb1e8d", "citrus juicer-44122046a14f6348", "lemon half2-1ab63128991488b9", "emon half2-1ab63128991488b9", "lemon half-74e253ad3d7423cc" ] }, { "key": "P02-20240211-111822:2bf77dbe8e7ba7af3d3dd2fe5ede1218076006e5", "video_id": "P02-20240211-111822", "question": "When I move the large black pot from the bottom left section of the stove to the top left section of the stove, what is the next ingredient that I add to the large black pot?", "answer_choice_0": "Pepper", "answer_choice_1": "Butter", "answer_choice_2": "Sugar", "answer_choice_3": "Eggs", "answer_choice_4": "Water", "answer_id": 2, "answer": "Sugar", "question_type": [ "Action Recognition", "Object Recognition", "Temporal Reasoning", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "phone-6744b55e629fbace", "hand blender attachment-2167c6bed0eb93dd", "hand blender-ca90da5b47137ab7", "hand blender cup-22f9b7158abd04a1", "washing up liquid bottle-825d5fd852e54610", "lid of hand blender cup-80b474eb460f7385", "sponge-ed4b1649de908d0a", "tea towel-1d373cdc4aaac937", "grater-8d788e8f028fc2bc", "plate-61db953b4a54c394", "bag of lemons-1c956539b3df3967", "punnets of raspberries-32aabb197004447a", "serrated knife-df2254f61cacc2ff", "lemon-b00bb617003dacac", "lemon2-56e4b71d5bfca0e3", "lemon3-ca1ff11afec1db47", "cutting board-dbfc7c6ebc5d3d47", "box of scales-4f8e8e0d3c43b823", "scales instructions-dfcf034f2d367837", "scales-d96dcf1060d55db3", "recipe book-728a08ba785943b9", "punnets of raspberries2-19b69198c95c5c56", "citrus juicer-44122046a14f6348", "food bin-a6d6362547eb1e8d", "lemon half-74e253ad3d7423cc", "lemon half2-1ab63128991488b9", "lemon half3-b9bae9722227adce", "container of sugar-2ebbc2ee15abf30d", "butter packet-ccf210777c7099ac", "butter packet2-6e451728af67fbb2", "stick of butter-fc1ce29145811450", "portion of butter-7b695c68d52254ec", "butter portion3-0a1aba42119aa8ea", "large mixing bowl-6a273b47425454a7", "saucepan-8e0305e33e07dc6b", "bigger saucepan-31b73dac0d238414", "lid of saucepan-fb99ded0fbc1d80a", "foam bottom of raspberry punnet-59e8f1b83bbab430", "plastic film lid-c869d8a729e2648a", "large plastic spoon-9dae0b46a533a7e2", "lid of sugar container-bf09b0c200c5d9a5", "bowl-43da25bf8c60818c", "teaspoon-1e46e1937149d3df", "kitchen roll holder-b7cf8787fcd4ac94", "kitchen towel-2f75ecd03f504e76", "egg carton-b74057bab01ab1c7", "egg-1e0a9c3f0f18b7bc", "egg2-3604159c0bf0f094", "egg3-58b911b052079207", "egg4-f1e0e13f8ebb09df", "fork-7eea868053353450", "cardboard container-e6e9d58ce4242517", "black washing up bowl-fb98537fa9fc5097", "frying pan-5fbfc013bc7c4637", "frying pan2-3cee48bd4ce2ea88", "oven mitts1-6faa4e8f0eee3903", "oven mitt2-b8b670aba479ef69", "smaller frying pan-541b208539c17909", "plate2-de20f7abcee1dca1", "plate3-403f22906468f4eb", "bowl2-d2817ac082cc7f9c", "bowl3-fe4ac808869fbce4", "lid of tupperware-4cad87cfbdda3072", "tupperware box-e7540763f2ce9433", "spatula-d292bed0be82a3cc", "bread knife-42811ef2c0816983", "cutlery-d0b68955c61b0b81", "butter knife-b63d9196d4bb5dc3", "other butter knife-0baf56dfd3c4bcd0", "knife-e0bd6d5a320bd2ad", "fork2-77e220ac85c0948b", "washing up brush-95ed7efe8b675933", "small mixing bowl-87743c3988bbbdcc", "citrus juicer guard-de5cb9b367d6534b", "glass countertop protector-969306577d8aba0b" ], "reasoning": "1. I watch the video until I notice the large black pot (bigger saucepan-31b73dac0d238414) on the bottom left section of the stove which happens at 15:09.\n2. I continue to watch until the user moves the pot (bigger saucepan-31b73dac0d238414) from the bottom left section to the top left section which happens at 23:33.\n3. I continue to watch the video from this point on to find the moment when the user adds another ingredient to the large black pot (bigger saucepan-31b73dac0d238414).\n4. At 29:06 I notice the user picking up a bowl (bowl-43da25bf8c60818c) containing sugar that originated from the container of sugar (container of sugar-2ebbc2ee15abf30d) from earlier in the video.\n5. At 29:06 the user pours the sugar into the large black pot (bigger saucepan-31b73dac0d238414), making this the first ingredient added since moving the pot (bigger saucepan-31b73dac0d238414) from the bottom left section of the stove to the top left section of the stove.", "referenced_object_masks": [ "container of sugar-2ebbc2ee15abf30d", "bigger saucepan-31b73dac0d238414", "bowl-43da25bf8c60818c" ] }, { "key": "P02-20240211-111822:5ab28853a501ab26ada18277e78d7c863f173370", "video_id": "P02-20240211-111822", "question": "If I only juiced lemons for the full duration of the video, how many lemon halves would I be able to juice, based on the amount of time I spend juicing the first lemon half and rounded to one decimal point?", "answer_choice_0": "103.1", "answer_choice_1": "107.9", "answer_choice_2": "110.3", "answer_choice_3": "104.9", "answer_choice_4": "95.3", "answer_id": 0, "answer": "103.1", "question_type": [ "Action Recognition", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "phone-6744b55e629fbace", "hand blender attachment-2167c6bed0eb93dd", "hand blender-ca90da5b47137ab7", "hand blender cup-22f9b7158abd04a1", "washing up liquid bottle-825d5fd852e54610", "lid of hand blender cup-80b474eb460f7385", "sponge-ed4b1649de908d0a", "tea towel-1d373cdc4aaac937", "grater-8d788e8f028fc2bc", "plate-61db953b4a54c394", "bag of lemons-1c956539b3df3967", "punnets of raspberries-32aabb197004447a", "serrated knife-df2254f61cacc2ff", "lemon-b00bb617003dacac", "lemon2-56e4b71d5bfca0e3", "lemon3-ca1ff11afec1db47", "cutting board-dbfc7c6ebc5d3d47", "box of scales-4f8e8e0d3c43b823", "scales instructions-dfcf034f2d367837", "scales-d96dcf1060d55db3", "recipe book-728a08ba785943b9", "punnets of raspberries2-19b69198c95c5c56", "citrus juicer-44122046a14f6348", "food bin-a6d6362547eb1e8d", "lemon half-74e253ad3d7423cc", "lemon half2-1ab63128991488b9", "lemon half3-b9bae9722227adce", "container of sugar-2ebbc2ee15abf30d", "butter packet-ccf210777c7099ac", "butter packet2-6e451728af67fbb2", "stick of butter-fc1ce29145811450", "portion of butter-7b695c68d52254ec", "butter portion3-0a1aba42119aa8ea", "large mixing bowl-6a273b47425454a7", "saucepan-8e0305e33e07dc6b", "bigger saucepan-31b73dac0d238414", "lid of saucepan-fb99ded0fbc1d80a", "foam bottom of raspberry punnet-59e8f1b83bbab430", "plastic film lid-c869d8a729e2648a", "large plastic spoon-9dae0b46a533a7e2", "lid of sugar container-bf09b0c200c5d9a5", "bowl-43da25bf8c60818c", "teaspoon-1e46e1937149d3df", "kitchen roll holder-b7cf8787fcd4ac94", "kitchen towel-2f75ecd03f504e76", "egg carton-b74057bab01ab1c7", "egg-1e0a9c3f0f18b7bc", "egg2-3604159c0bf0f094", "egg3-58b911b052079207", "egg4-f1e0e13f8ebb09df", "fork-7eea868053353450", "cardboard container-e6e9d58ce4242517", "black washing up bowl-fb98537fa9fc5097", "frying pan-5fbfc013bc7c4637", "frying pan2-3cee48bd4ce2ea88", "oven mitts1-6faa4e8f0eee3903", "oven mitt2-b8b670aba479ef69", "smaller frying pan-541b208539c17909", "plate2-de20f7abcee1dca1", "plate3-403f22906468f4eb", "bowl2-d2817ac082cc7f9c", "bowl3-fe4ac808869fbce4", "lid of tupperware-4cad87cfbdda3072", "tupperware box-e7540763f2ce9433", "spatula-d292bed0be82a3cc", "bread knife-42811ef2c0816983", "cutlery-d0b68955c61b0b81", "butter knife-b63d9196d4bb5dc3", "other butter knife-0baf56dfd3c4bcd0", "knife-e0bd6d5a320bd2ad", "fork2-77e220ac85c0948b", "washing up brush-95ed7efe8b675933", "small mixing bowl-87743c3988bbbdcc", "citrus juicer guard-de5cb9b367d6534b", "glass countertop protector-969306577d8aba0b" ], "reasoning": "1. I watched the video to see when the user began juicing the lemons (lemon-b00bb617003dacac, emon half-74e253ad3d7423cc) at 09:23. \n2. From this point, I watched and observed that the user continuously juices the lemon (lemon half-74e253ad3d7423cc) with the citrus juicer (citrus juicer-44122046a14f6348) from 09:23-09:39, which equals 16 seconds.\n3. From 09:29-09:45, I watched the user move to bring the trashcan (food bin-a6d6362547eb1e8d) to the counter.\n5. Then, from 09:45-09:51, I watched the user use the citrus juicer (citrus juicer-44122046a14f6348) to juice the lemon half (lemon half-74e253ad3d7423cc) from 09:45-09:51, or a total of 6 more seconds.\n6. I calculated that the lemon half (lemon half-74e253ad3d7423cc) is juiced for a total of 22 seconds.\n7. I then saw that the video is 37:48 in length, which equals 2,268 seconds. \n8. I then did the final math which is 2268 / 22 - 103.1.", "referenced_object_masks": [ "food bin-a6d6362547eb1e8d", "citrus juicer-44122046a14f6348", "lemon-b00bb617003dacac", "emon half-74e253ad3d7423cc", "lemon half-74e253ad3d7423cc" ] }, { "key": "P02-20240211-111822:812ed0d7ee127502ab1372f9a31bf814729c61a8", "video_id": "P02-20240211-111822", "question": "After stirring in all the ingredients into the bowl on the stove, how many times do I tap the spoon against the bowl?", "answer_choice_0": "28", "answer_choice_1": "40", "answer_choice_2": "24", "answer_choice_3": "38", "answer_choice_4": "33", "answer_id": 4, "answer": "33", "question_type": [ "Action Recognition", "Counting", "Event Occurence", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "phone-6744b55e629fbace", "hand blender attachment-2167c6bed0eb93dd", "hand blender-ca90da5b47137ab7", "hand blender cup-22f9b7158abd04a1", "washing up liquid bottle-825d5fd852e54610", "lid of hand blender cup-80b474eb460f7385", "sponge-ed4b1649de908d0a", "tea towel-1d373cdc4aaac937", "grater-8d788e8f028fc2bc", "plate-61db953b4a54c394", "bag of lemons-1c956539b3df3967", "punnets of raspberries-32aabb197004447a", "serrated knife-df2254f61cacc2ff", "lemon-b00bb617003dacac", "lemon2-56e4b71d5bfca0e3", "lemon3-ca1ff11afec1db47", "cutting board-dbfc7c6ebc5d3d47", "box of scales-4f8e8e0d3c43b823", "scales instructions-dfcf034f2d367837", "scales-d96dcf1060d55db3", "recipe book-728a08ba785943b9", "punnets of raspberries2-19b69198c95c5c56", "citrus juicer-44122046a14f6348", "food bin-a6d6362547eb1e8d", "lemon half-74e253ad3d7423cc", "lemon half2-1ab63128991488b9", "lemon half3-b9bae9722227adce", "container of sugar-2ebbc2ee15abf30d", "butter packet-ccf210777c7099ac", "butter packet2-6e451728af67fbb2", "stick of butter-fc1ce29145811450", "portion of butter-7b695c68d52254ec", "butter portion3-0a1aba42119aa8ea", "large mixing bowl-6a273b47425454a7", "saucepan-8e0305e33e07dc6b", "bigger saucepan-31b73dac0d238414", "lid of saucepan-fb99ded0fbc1d80a", "foam bottom of raspberry punnet-59e8f1b83bbab430", "plastic film lid-c869d8a729e2648a", "large plastic spoon-9dae0b46a533a7e2", "lid of sugar container-bf09b0c200c5d9a5", "bowl-43da25bf8c60818c", "teaspoon-1e46e1937149d3df", "kitchen roll holder-b7cf8787fcd4ac94", "kitchen towel-2f75ecd03f504e76", "egg carton-b74057bab01ab1c7", "egg-1e0a9c3f0f18b7bc", "egg2-3604159c0bf0f094", "egg3-58b911b052079207", "egg4-f1e0e13f8ebb09df", "fork-7eea868053353450", "cardboard container-e6e9d58ce4242517", "black washing up bowl-fb98537fa9fc5097", "frying pan-5fbfc013bc7c4637", "frying pan2-3cee48bd4ce2ea88", "oven mitts1-6faa4e8f0eee3903", "oven mitt2-b8b670aba479ef69", "smaller frying pan-541b208539c17909", "plate2-de20f7abcee1dca1", "plate3-403f22906468f4eb", "bowl2-d2817ac082cc7f9c", "bowl3-fe4ac808869fbce4", "lid of tupperware-4cad87cfbdda3072", "tupperware box-e7540763f2ce9433", "spatula-d292bed0be82a3cc", "bread knife-42811ef2c0816983", "cutlery-d0b68955c61b0b81", "butter knife-b63d9196d4bb5dc3", "other butter knife-0baf56dfd3c4bcd0", "knife-e0bd6d5a320bd2ad", "fork2-77e220ac85c0948b", "washing up brush-95ed7efe8b675933", "small mixing bowl-87743c3988bbbdcc", "citrus juicer guard-de5cb9b367d6534b", "glass countertop protector-969306577d8aba0b" ], "reasoning": "1. I watched the video from 29:06 until 32:50 when the user mixes the various ingredients into the bowl (large mixing bowl-6a273b47425454a7) on the stove.\n2. After stirring the items in the bowl (large mixing bowl-6a273b47425454a7), the user takes the spoon (large plastic spoon-9dae0b46a533a7e2) out.\n3. From 32:54-32:56, the user taps the spoon (large plastic spoon-9dae0b46a533a7e2) on the rim of the bowl (large mixing bowl-6a273b47425454a7) 15 times.\n4. Then, from 32:56-33:00, the user looks to tap the spoon (large plastic spoon-9dae0b46a533a7e2) on the bowl (large mixing bowl-6a273b47425454a7), but no sound is made so the spoon (large plastic spoon-9dae0b46a533a7e2) was never tapped in this span.\n7. I next observed the user tap the spoon (large plastic spoon-9dae0b46a533a7e2) on the bowl's rim (large mixing bowl-6a273b47425454a7) 18 more times from 33:00-33:03.\n8. The user then put the spoon (large plastic spoon-9dae0b46a533a7e2) in the sink (OBJ, ending the action.\n9. I then calculated 18 + 15 = 33.", "referenced_object_masks": [ "large plastic spoon-9dae0b46a533a7e2", "large mixing bowl-6a273b47425454a7" ] }, { "key": "P02-20240211-111822:88cf94915c0038c9253ab43eee2f84302d744f5e", "video_id": "P02-20240211-111822", "question": "What change occurs to the tea towel from the beginning to the end of the video?", "answer_choice_0": "It is hung on a chair", "answer_choice_1": "It gets hung on the stove", "answer_choice_2": "It gets put in a laundry hamper", "answer_choice_3": "It gets hung on the radiator", "answer_choice_4": "It is sitting in a bowl", "answer_id": 3, "answer": "It gets hung on the radiator", "question_type": [ "State Changes", "Object Recognition", "Event Occurence" ], "objects_of_interest": [ "phone-6744b55e629fbace", "hand blender attachment-2167c6bed0eb93dd", "hand blender-ca90da5b47137ab7", "hand blender cup-22f9b7158abd04a1", "washing up liquid bottle-825d5fd852e54610", "lid of hand blender cup-80b474eb460f7385", "sponge-ed4b1649de908d0a", "tea towel-1d373cdc4aaac937", "grater-8d788e8f028fc2bc", "plate-61db953b4a54c394", "bag of lemons-1c956539b3df3967", "punnets of raspberries-32aabb197004447a", "serrated knife-df2254f61cacc2ff", "lemon-b00bb617003dacac", "lemon2-56e4b71d5bfca0e3", "lemon3-ca1ff11afec1db47", "cutting board-dbfc7c6ebc5d3d47", "box of scales-4f8e8e0d3c43b823", "scales instructions-dfcf034f2d367837", "scales-d96dcf1060d55db3", "recipe book-728a08ba785943b9", "punnets of raspberries2-19b69198c95c5c56", "citrus juicer-44122046a14f6348", "food bin-a6d6362547eb1e8d", "lemon half-74e253ad3d7423cc", "lemon half2-1ab63128991488b9", "lemon half3-b9bae9722227adce", "container of sugar-2ebbc2ee15abf30d", "butter packet-ccf210777c7099ac", "butter packet2-6e451728af67fbb2", "stick of butter-fc1ce29145811450", "portion of butter-7b695c68d52254ec", "butter portion3-0a1aba42119aa8ea", "large mixing bowl-6a273b47425454a7", "saucepan-8e0305e33e07dc6b", "bigger saucepan-31b73dac0d238414", "lid of saucepan-fb99ded0fbc1d80a", "foam bottom of raspberry punnet-59e8f1b83bbab430", "plastic film lid-c869d8a729e2648a", "large plastic spoon-9dae0b46a533a7e2", "lid of sugar container-bf09b0c200c5d9a5", "bowl-43da25bf8c60818c", "teaspoon-1e46e1937149d3df", "kitchen roll holder-b7cf8787fcd4ac94", "kitchen towel-2f75ecd03f504e76", "egg carton-b74057bab01ab1c7", "egg-1e0a9c3f0f18b7bc", "egg2-3604159c0bf0f094", "egg3-58b911b052079207", "egg4-f1e0e13f8ebb09df", "fork-7eea868053353450", "cardboard container-e6e9d58ce4242517", "black washing up bowl-fb98537fa9fc5097", "frying pan-5fbfc013bc7c4637", "frying pan2-3cee48bd4ce2ea88", "oven mitts1-6faa4e8f0eee3903", "oven mitt2-b8b670aba479ef69", "smaller frying pan-541b208539c17909", "plate2-de20f7abcee1dca1", "plate3-403f22906468f4eb", "bowl2-d2817ac082cc7f9c", "bowl3-fe4ac808869fbce4", "lid of tupperware-4cad87cfbdda3072", "tupperware box-e7540763f2ce9433", "spatula-d292bed0be82a3cc", "bread knife-42811ef2c0816983", "cutlery-d0b68955c61b0b81", "butter knife-b63d9196d4bb5dc3", "other butter knife-0baf56dfd3c4bcd0", "knife-e0bd6d5a320bd2ad", "fork2-77e220ac85c0948b", "washing up brush-95ed7efe8b675933", "small mixing bowl-87743c3988bbbdcc", "citrus juicer guard-de5cb9b367d6534b", "glass countertop protector-969306577d8aba0b" ], "reasoning": "1. I began watching the video and noted that the tea towel (tea towel-1d373cdc4aaac937) begins piled up on the counter.\n2. I continued watching and noted that the user moves the towel (tea towel-1d373cdc4aaac937) to the radiator to the right from 04:24-04:27. The user sets the towel (tea towel-1d373cdc4aaac937) next to another towel already hanging there.\n3. I watched the rest of the video and observed that the towel (tea towel-1d373cdc4aaac937) does not move. \n4. I confirmed it by finding the last instance of the video in which the towel (tea towel-1d373cdc4aaac937) is visible, which occurs at 36:56. Here, the towel (tea towel-1d373cdc4aaac937) is still on the radiator.", "referenced_object_masks": [ "tea towel-1d373cdc4aaac937" ] }, { "key": "P02-20240211-111822:99a21fcf927be6a43aacc7e0f58fda10cbadfc0f", "video_id": "P02-20240211-111822", "question": "What time is displayed on my phone when I check it for the third time?", "answer_choice_0": "2:19", "answer_choice_1": "11:18", "answer_choice_2": "09:34", "answer_choice_3": "11:54", "answer_choice_4": "12:35", "answer_id": 3, "answer": "11:54", "question_type": [ "Object Recognition", "Temporal Reasoning", "Egocentric & Spatial Perception", "Reading", "Counting" ], "objects_of_interest": [ "phone-6744b55e629fbace", "hand blender attachment-2167c6bed0eb93dd", "hand blender-ca90da5b47137ab7", "hand blender cup-22f9b7158abd04a1", "washing up liquid bottle-825d5fd852e54610", "lid of hand blender cup-80b474eb460f7385", "sponge-ed4b1649de908d0a", "tea towel-1d373cdc4aaac937", "grater-8d788e8f028fc2bc", "plate-61db953b4a54c394", "bag of lemons-1c956539b3df3967", "punnets of raspberries-32aabb197004447a", "serrated knife-df2254f61cacc2ff", "lemon-b00bb617003dacac", "lemon2-56e4b71d5bfca0e3", "lemon3-ca1ff11afec1db47", "cutting board-dbfc7c6ebc5d3d47", "box of scales-4f8e8e0d3c43b823", "scales instructions-dfcf034f2d367837", "scales-d96dcf1060d55db3", "recipe book-728a08ba785943b9", "punnets of raspberries2-19b69198c95c5c56", "citrus juicer-44122046a14f6348", "food bin-a6d6362547eb1e8d", "lemon half-74e253ad3d7423cc", "lemon half2-1ab63128991488b9", "lemon half3-b9bae9722227adce", "container of sugar-2ebbc2ee15abf30d", "butter packet-ccf210777c7099ac", "butter packet2-6e451728af67fbb2", "stick of butter-fc1ce29145811450", "portion of butter-7b695c68d52254ec", "butter portion3-0a1aba42119aa8ea", "large mixing bowl-6a273b47425454a7", "saucepan-8e0305e33e07dc6b", "bigger saucepan-31b73dac0d238414", "lid of saucepan-fb99ded0fbc1d80a", "foam bottom of raspberry punnet-59e8f1b83bbab430", "plastic film lid-c869d8a729e2648a", "large plastic spoon-9dae0b46a533a7e2", "lid of sugar container-bf09b0c200c5d9a5", "bowl-43da25bf8c60818c", "teaspoon-1e46e1937149d3df", "kitchen roll holder-b7cf8787fcd4ac94", "kitchen towel-2f75ecd03f504e76", "egg carton-b74057bab01ab1c7", "egg-1e0a9c3f0f18b7bc", "egg2-3604159c0bf0f094", "egg3-58b911b052079207", "egg4-f1e0e13f8ebb09df", "fork-7eea868053353450", "cardboard container-e6e9d58ce4242517", "black washing up bowl-fb98537fa9fc5097", "frying pan-5fbfc013bc7c4637", "frying pan2-3cee48bd4ce2ea88", "oven mitts1-6faa4e8f0eee3903", "oven mitt2-b8b670aba479ef69", "smaller frying pan-541b208539c17909", "plate2-de20f7abcee1dca1", "plate3-403f22906468f4eb", "bowl2-d2817ac082cc7f9c", "bowl3-fe4ac808869fbce4", "lid of tupperware-4cad87cfbdda3072", "tupperware box-e7540763f2ce9433", "spatula-d292bed0be82a3cc", "bread knife-42811ef2c0816983", "cutlery-d0b68955c61b0b81", "butter knife-b63d9196d4bb5dc3", "other butter knife-0baf56dfd3c4bcd0", "knife-e0bd6d5a320bd2ad", "fork2-77e220ac85c0948b", "washing up brush-95ed7efe8b675933", "small mixing bowl-87743c3988bbbdcc", "citrus juicer guard-de5cb9b367d6534b", "glass countertop protector-969306577d8aba0b" ], "reasoning": "1. Watching the video I first notice the cell phone (phone-6744b55e629fbace) at 00:02.\n2. I continued to watch the video and noticed the user checked their phone (phone-6744b55e629fbace) again at 18:36. I note this as the second time the user checks their phone (phone-6744b55e629fbace).\n3. Continuing to watch the video, I noticed the user checks their phone (phone-6744b55e629fbace) for the third time at 35:59. I focus on the time displayed on the phone (phone-6744b55e629fbace) and see that it is 11:54.", "referenced_object_masks": [ "phone-6744b55e629fbace" ] }, { "key": "P02-20240211-111822:9faae271512b6d2dff12a479670acb77eb6c7a4a", "video_id": "P02-20240211-111822", "question": "When zesting the second lemon, how many times did I adjust my grip on it?", "answer_choice_0": "16", "answer_choice_1": "19", "answer_choice_2": "30", "answer_choice_3": "12", "answer_choice_4": "24", "answer_id": 1, "answer": "19", "question_type": [ "Temporal Reasoning", "Counting", "Event Occurence", "Object Recognition", "Numerical Reasoning" ], "objects_of_interest": [ "phone-6744b55e629fbace", "hand blender attachment-2167c6bed0eb93dd", "hand blender-ca90da5b47137ab7", "hand blender cup-22f9b7158abd04a1", "washing up liquid bottle-825d5fd852e54610", "lid of hand blender cup-80b474eb460f7385", "sponge-ed4b1649de908d0a", "tea towel-1d373cdc4aaac937", "grater-8d788e8f028fc2bc", "plate-61db953b4a54c394", "bag of lemons-1c956539b3df3967", "punnets of raspberries-32aabb197004447a", "serrated knife-df2254f61cacc2ff", "lemon-b00bb617003dacac", "lemon2-56e4b71d5bfca0e3", "lemon3-ca1ff11afec1db47", "cutting board-dbfc7c6ebc5d3d47", "box of scales-4f8e8e0d3c43b823", "scales instructions-dfcf034f2d367837", "scales-d96dcf1060d55db3", "recipe book-728a08ba785943b9", "punnets of raspberries2-19b69198c95c5c56", "citrus juicer-44122046a14f6348", "food bin-a6d6362547eb1e8d", "lemon half-74e253ad3d7423cc", "lemon half2-1ab63128991488b9", "lemon half3-b9bae9722227adce", "container of sugar-2ebbc2ee15abf30d", "butter packet-ccf210777c7099ac", "butter packet2-6e451728af67fbb2", "stick of butter-fc1ce29145811450", "portion of butter-7b695c68d52254ec", "butter portion3-0a1aba42119aa8ea", "large mixing bowl-6a273b47425454a7", "saucepan-8e0305e33e07dc6b", "bigger saucepan-31b73dac0d238414", "lid of saucepan-fb99ded0fbc1d80a", "foam bottom of raspberry punnet-59e8f1b83bbab430", "plastic film lid-c869d8a729e2648a", "large plastic spoon-9dae0b46a533a7e2", "lid of sugar container-bf09b0c200c5d9a5", "bowl-43da25bf8c60818c", "teaspoon-1e46e1937149d3df", "kitchen roll holder-b7cf8787fcd4ac94", "kitchen towel-2f75ecd03f504e76", "egg carton-b74057bab01ab1c7", "egg-1e0a9c3f0f18b7bc", "egg2-3604159c0bf0f094", "egg3-58b911b052079207", "egg4-f1e0e13f8ebb09df", "fork-7eea868053353450", "cardboard container-e6e9d58ce4242517", "black washing up bowl-fb98537fa9fc5097", "frying pan-5fbfc013bc7c4637", "frying pan2-3cee48bd4ce2ea88", "oven mitts1-6faa4e8f0eee3903", "oven mitt2-b8b670aba479ef69", "smaller frying pan-541b208539c17909", "plate2-de20f7abcee1dca1", "plate3-403f22906468f4eb", "bowl2-d2817ac082cc7f9c", "bowl3-fe4ac808869fbce4", "lid of tupperware-4cad87cfbdda3072", "tupperware box-e7540763f2ce9433", "spatula-d292bed0be82a3cc", "bread knife-42811ef2c0816983", "cutlery-d0b68955c61b0b81", "butter knife-b63d9196d4bb5dc3", "other butter knife-0baf56dfd3c4bcd0", "knife-e0bd6d5a320bd2ad", "fork2-77e220ac85c0948b", "washing up brush-95ed7efe8b675933", "small mixing bowl-87743c3988bbbdcc", "citrus juicer guard-de5cb9b367d6534b", "glass countertop protector-969306577d8aba0b" ], "reasoning": "1. I watched the video to see the user pull the lemons (bag of lemons-1c956539b3df3967) from the refrigerator from 02:15-02:20, then set the bag on the counter at 02:24.\n3. Then, from 02:25-02:51, I watched the user open the bag of lemons (bag of lemons-1c956539b3df3967) with a knife (serrated knife-df2254f61cacc2ff). In this process the user picks out three lemons (lemon-b00bb617003dacac, lemon2-56e4b71d5bfca0e3, lemon3-ca1ff11afec1db47), putting lemon 2 (lemon2-56e4b71d5bfca0e3) back in the bag.\n5. Next, I observed the user zesting the first lemon (lemon-b00bb617003dacac) with the grater (grater-8d788e8f028fc2bc) from 02:56-04:02.\n6. The user then picks up the other lemon (lemon3-ca1ff11afec1db47) and begins zesting it with the grater (grater-8d788e8f028fc2bc) from 04:17-04:24, stopping briefly to grab a towl to the left (tea towel-1d373cdc4aaac937). In this span, the user switches their grip on the lemon 3 times.\n7, From 04:28-04:42, the user continues to grate the lemon (lemon3-ca1ff11afec1db47), adjusting their grip on it 7 times in this span.\n8. The user takes another slight break to grab something else before resuming the zesting process from 04:42-04:50, switching grip 3 times.\n9. After another pause, the user zests the lemon (lemon3-ca1ff11afec1db47) again, and switches their grip 7 times from 04-55-05:21.\n10. I then noted that the user finished zesting the lemon (lemon3-ca1ff11afec1db47) at 05:21, totaling 20 hand adjustments.", "referenced_object_masks": [ "tea towel-1d373cdc4aaac937", "lemon2-56e4b71d5bfca0e3", "lemon-b00bb617003dacac", "lemon3-ca1ff11afec1db47", "grater-8d788e8f028fc2bc", "serrated knife-df2254f61cacc2ff", "bag of lemons-1c956539b3df3967" ] }, { "key": "P02-20240211-111822:ecb486889127972614d49f1e9dc72a25977b7f04", "video_id": "P02-20240211-111822", "question": "What was the third object I cleaned using the yellow and green sponge?", "answer_choice_0": "Plate", "answer_choice_1": "Plastic lid", "answer_choice_2": "Clear cup", "answer_choice_3": "Fork", "answer_choice_4": "Hand blender", "answer_id": 4, "answer": "Hand blender", "question_type": [ "Counting", "Temporal Reasoning", "Numerical Reasoning", "Object Recognition", "Event Occurence" ], "objects_of_interest": [ "phone-6744b55e629fbace", "hand blender attachment-2167c6bed0eb93dd", "hand blender-ca90da5b47137ab7", "hand blender cup-22f9b7158abd04a1", "washing up liquid bottle-825d5fd852e54610", "lid of hand blender cup-80b474eb460f7385", "sponge-ed4b1649de908d0a", "tea towel-1d373cdc4aaac937", "grater-8d788e8f028fc2bc", "plate-61db953b4a54c394", "bag of lemons-1c956539b3df3967", "punnets of raspberries-32aabb197004447a", "serrated knife-df2254f61cacc2ff", "lemon-b00bb617003dacac", "lemon2-56e4b71d5bfca0e3", "lemon3-ca1ff11afec1db47", "cutting board-dbfc7c6ebc5d3d47", "box of scales-4f8e8e0d3c43b823", "scales instructions-dfcf034f2d367837", "scales-d96dcf1060d55db3", "recipe book-728a08ba785943b9", "punnets of raspberries2-19b69198c95c5c56", "citrus juicer-44122046a14f6348", "food bin-a6d6362547eb1e8d", "lemon half-74e253ad3d7423cc", "lemon half2-1ab63128991488b9", "lemon half3-b9bae9722227adce", "container of sugar-2ebbc2ee15abf30d", "butter packet-ccf210777c7099ac", "butter packet2-6e451728af67fbb2", "stick of butter-fc1ce29145811450", "portion of butter-7b695c68d52254ec", "butter portion3-0a1aba42119aa8ea", "large mixing bowl-6a273b47425454a7", "saucepan-8e0305e33e07dc6b", "bigger saucepan-31b73dac0d238414", "lid of saucepan-fb99ded0fbc1d80a", "foam bottom of raspberry punnet-59e8f1b83bbab430", "plastic film lid-c869d8a729e2648a", "large plastic spoon-9dae0b46a533a7e2", "lid of sugar container-bf09b0c200c5d9a5", "bowl-43da25bf8c60818c", "teaspoon-1e46e1937149d3df", "kitchen roll holder-b7cf8787fcd4ac94", "kitchen towel-2f75ecd03f504e76", "egg carton-b74057bab01ab1c7", "egg-1e0a9c3f0f18b7bc", "egg2-3604159c0bf0f094", "egg3-58b911b052079207", "egg4-f1e0e13f8ebb09df", "fork-7eea868053353450", "cardboard container-e6e9d58ce4242517", "black washing up bowl-fb98537fa9fc5097", "frying pan-5fbfc013bc7c4637", "frying pan2-3cee48bd4ce2ea88", "oven mitts1-6faa4e8f0eee3903", "oven mitt2-b8b670aba479ef69", "smaller frying pan-541b208539c17909", "plate2-de20f7abcee1dca1", "plate3-403f22906468f4eb", "bowl2-d2817ac082cc7f9c", "bowl3-fe4ac808869fbce4", "lid of tupperware-4cad87cfbdda3072", "tupperware box-e7540763f2ce9433", "spatula-d292bed0be82a3cc", "bread knife-42811ef2c0816983", "cutlery-d0b68955c61b0b81", "butter knife-b63d9196d4bb5dc3", "other butter knife-0baf56dfd3c4bcd0", "knife-e0bd6d5a320bd2ad", "fork2-77e220ac85c0948b", "washing up brush-95ed7efe8b675933", "small mixing bowl-87743c3988bbbdcc", "citrus juicer guard-de5cb9b367d6534b", "glass countertop protector-969306577d8aba0b" ], "reasoning": "1. I watched the video and saw the user first use the yellow and green sponge (sponge-ed4b1649de908d0a) to clean a clear cup at 00:29. This is the first object.\n2. I continued watching and saw the user then begin to use the yellow and green sponge (sponge-ed4b1649de908d0a) to clean a plastic lid (lid of hand blender cup-80b474eb460f7385) at 00:45. This is the second object.\n3. I continued watching the video and at 00:53 I see the user begin cleaning a hand blender (hand blender attachment-2167c6bed0eb93dd) with the yellow and green sponge (sponge-ed4b1649de908d0a). This is the third object.", "referenced_object_masks": [ "sponge-ed4b1649de908d0a", "lid of hand blender cup-80b474eb460f7385", "hand blender attachment-2167c6bed0eb93dd" ] }, { "key": "P02-20240211-111822:efbcdb765e86f45ddd8f7ae8fd3e0e1562d3b79e", "video_id": "P02-20240211-111822", "question": "What is the last item I clean in the black washing up bowl?", "answer_choice_0": "Plate", "answer_choice_1": "Cutting board", "answer_choice_2": "Cup", "answer_choice_3": "Juicer", "answer_choice_4": "Knife", "answer_id": 3, "answer": "Juicer", "question_type": [ "Action Recognition", "Object Recognition", "Temporal Reasoning", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "phone-6744b55e629fbace", "hand blender attachment-2167c6bed0eb93dd", "hand blender-ca90da5b47137ab7", "hand blender cup-22f9b7158abd04a1", "washing up liquid bottle-825d5fd852e54610", "lid of hand blender cup-80b474eb460f7385", "sponge-ed4b1649de908d0a", "tea towel-1d373cdc4aaac937", "grater-8d788e8f028fc2bc", "plate-61db953b4a54c394", "bag of lemons-1c956539b3df3967", "punnets of raspberries-32aabb197004447a", "serrated knife-df2254f61cacc2ff", "lemon-b00bb617003dacac", "lemon2-56e4b71d5bfca0e3", "lemon3-ca1ff11afec1db47", "cutting board-dbfc7c6ebc5d3d47", "box of scales-4f8e8e0d3c43b823", "scales instructions-dfcf034f2d367837", "scales-d96dcf1060d55db3", "recipe book-728a08ba785943b9", "punnets of raspberries2-19b69198c95c5c56", "citrus juicer-44122046a14f6348", "food bin-a6d6362547eb1e8d", "lemon half-74e253ad3d7423cc", "lemon half2-1ab63128991488b9", "lemon half3-b9bae9722227adce", "container of sugar-2ebbc2ee15abf30d", "butter packet-ccf210777c7099ac", "butter packet2-6e451728af67fbb2", "stick of butter-fc1ce29145811450", "portion of butter-7b695c68d52254ec", "butter portion3-0a1aba42119aa8ea", "large mixing bowl-6a273b47425454a7", "saucepan-8e0305e33e07dc6b", "bigger saucepan-31b73dac0d238414", "lid of saucepan-fb99ded0fbc1d80a", "foam bottom of raspberry punnet-59e8f1b83bbab430", "plastic film lid-c869d8a729e2648a", "large plastic spoon-9dae0b46a533a7e2", "lid of sugar container-bf09b0c200c5d9a5", "bowl-43da25bf8c60818c", "teaspoon-1e46e1937149d3df", "kitchen roll holder-b7cf8787fcd4ac94", "kitchen towel-2f75ecd03f504e76", "egg carton-b74057bab01ab1c7", "egg-1e0a9c3f0f18b7bc", "egg2-3604159c0bf0f094", "egg3-58b911b052079207", "egg4-f1e0e13f8ebb09df", "fork-7eea868053353450", "cardboard container-e6e9d58ce4242517", "black washing up bowl-fb98537fa9fc5097", "frying pan-5fbfc013bc7c4637", "frying pan2-3cee48bd4ce2ea88", "oven mitts1-6faa4e8f0eee3903", "oven mitt2-b8b670aba479ef69", "smaller frying pan-541b208539c17909", "plate2-de20f7abcee1dca1", "plate3-403f22906468f4eb", "bowl2-d2817ac082cc7f9c", "bowl3-fe4ac808869fbce4", "lid of tupperware-4cad87cfbdda3072", "tupperware box-e7540763f2ce9433", "spatula-d292bed0be82a3cc", "bread knife-42811ef2c0816983", "cutlery-d0b68955c61b0b81", "butter knife-b63d9196d4bb5dc3", "other butter knife-0baf56dfd3c4bcd0", "knife-e0bd6d5a320bd2ad", "fork2-77e220ac85c0948b", "washing up brush-95ed7efe8b675933", "small mixing bowl-87743c3988bbbdcc", "citrus juicer guard-de5cb9b367d6534b", "glass countertop protector-969306577d8aba0b" ], "reasoning": "1. I watch through the video and look for when the user begins cleaning items in the black wash up bowl (black washing up bowl-fb98537fa9fc5097). I saw that the user added soap to the black wash up bowl (black washing up bowl-fb98537fa9fc5097) at 00:21.\n2. The first item cleaned is a glass at 00:27. I watched the rest of the video, looking for the final item to be cleaned.\n3. The user then begins cleaning the juicer (citrus juicer-44122046a14f6348) at 34:08. This is the last item that gets cleaned, as the user finishes cleaning it at 34:59, and no other dishes are cleaned after that point.", "referenced_object_masks": [ "black washing up bowl-fb98537fa9fc5097", "citrus juicer-44122046a14f6348" ] }, { "key": "P02-20240211-115653:0924c9fd0f2d570da0f7ad1875a1f839dbce2c53", "video_id": "P02-20240211-115653", "question": "When did I check the smartphone for the second time?", "answer_choice_0": "06:05-06:15", "answer_choice_1": "04:20-04:25", "answer_choice_2": "08:50-08:55", "answer_choice_3": "02:30-02:35", "answer_choice_4": "10:40-10:45", "answer_id": 4, "answer": "10:40-10:45", "question_type": [ "Object Recognition", "Action Recognition", "Counting", "Temporal Reasoning" ], "objects_of_interest": [ "recording phone-82b879e1583c10f2", "large plastic spoon-56b410eb3a192282", "bowl-64c16b0e0e04861b", "kitchen roll holder-45f25d232a90dc1f", "kitchen towel-cfe0bd72554bc991", "tupperware box-25f304be17f0aba1", "puff pastry-fb9290ebaaeb982f", "block of cheddar cheese-67ec25da5435c7e5", "loaf of bread-bcb3abcad69367a1", "other packet of cheese-d9a2683b43561b72", "puff pastry roll-74f05813bd2d7a5f", "scissors-e702216bed37c119", "tea towel-b69430d711f74ec4", "grater-207bbe498a4459fc", "rolling pin-d57486f705c69885", "green bag-3ff3fa129e17ffe6", "paper cover-a7b81286039719c5", "serrated knife-828526a18cd9960e" ], "reasoning": "1. I watched the video and found the smartphone (recording phone-82b879e1583c10f2) at 00:01.\n2. I counted each time the user checked the smartphone (recording phone-82b879e1583c10f2), calculating a total of 2 at 00:01 and 10:40-10:45.\n3. Therefore, the user checked the smartphone (recording phone-82b879e1583c10f2) for the second time at 10:40-10:45.", "referenced_object_masks": [ "recording phone-82b879e1583c10f2" ] }, { "key": "P02-20240211-115653:2d0e50c0263a8c82b43808f320437a344bb17227", "video_id": "P02-20240211-115653", "question": "Were I to enter the kitchen from the dining room, where would the white tea towel be in relation to the bowl of red filling, assuming the tea towel is in its original place?", "answer_choice_0": "To the left", "answer_choice_1": "Behind it", "answer_choice_2": "To the right", "answer_choice_3": "Above it", "answer_choice_4": "Below it", "answer_id": 0, "answer": "To the left", "question_type": [ "Object Recognition", "Egocentric & Spatial Perception", "Situational Awareness" ], "objects_of_interest": [ "recording phone-82b879e1583c10f2", "large plastic spoon-56b410eb3a192282", "bowl-64c16b0e0e04861b", "kitchen roll holder-45f25d232a90dc1f", "kitchen towel-cfe0bd72554bc991", "tupperware box-25f304be17f0aba1", "puff pastry-fb9290ebaaeb982f", "block of cheddar cheese-67ec25da5435c7e5", "loaf of bread-bcb3abcad69367a1", "other packet of cheese-d9a2683b43561b72", "puff pastry roll-74f05813bd2d7a5f", "scissors-e702216bed37c119", "tea towel-b69430d711f74ec4", "grater-207bbe498a4459fc", "rolling pin-d57486f705c69885", "green bag-3ff3fa129e17ffe6", "paper cover-a7b81286039719c5", "serrated knife-828526a18cd9960e" ], "reasoning": "1. I watched the video and found the bowl of red filling (bowl-64c16b0e0e04861b) cooking at 00:06. It is cooking on the stovetop.\n2. I found the white tea towel (tea towel-b69430d711f74ec4) at 03:21. I notice it is hanging on the white radiator next to a pinkish towel .\n3. I saw the user entered the kitchen from the dining room at 00:01. Here, I realize that, from this angle, the stovetop would be to the right of the radiator.\n4. Therefore, considering the problem from the objects' perspectives, the white tea towel (tea towel-b69430d711f74ec4) would be to the left of the bowl (bowl-64c16b0e0e04861b).", "referenced_object_masks": [ "bowl-64c16b0e0e04861b", "tea towel-b69430d711f74ec4" ] }, { "key": "P02-20240211-115653:d4eeffd845fd929cf95a8c3a1eb70bd41cce8342", "video_id": "P02-20240211-115653", "question": "If I doubled the amount of times I grabbed the stirring spoon, how many times would I have grabbed it?", "answer_choice_0": "6", "answer_choice_1": "2", "answer_choice_2": "4", "answer_choice_3": "8", "answer_choice_4": "10", "answer_id": 3, "answer": "8", "question_type": [ "Object Recognition", "Action Recognition", "Counting", "Numerical Reasoning", "Counterfactual" ], "objects_of_interest": [ "recording phone-82b879e1583c10f2", "large plastic spoon-56b410eb3a192282", "bowl-64c16b0e0e04861b", "kitchen roll holder-45f25d232a90dc1f", "kitchen towel-cfe0bd72554bc991", "tupperware box-25f304be17f0aba1", "puff pastry-fb9290ebaaeb982f", "block of cheddar cheese-67ec25da5435c7e5", "loaf of bread-bcb3abcad69367a1", "other packet of cheese-d9a2683b43561b72", "puff pastry roll-74f05813bd2d7a5f", "scissors-e702216bed37c119", "tea towel-b69430d711f74ec4", "grater-207bbe498a4459fc", "rolling pin-d57486f705c69885", "green bag-3ff3fa129e17ffe6", "paper cover-a7b81286039719c5", "serrated knife-828526a18cd9960e" ], "reasoning": "1. I watched the video and found the stirring spoon (large plastic spoon-56b410eb3a192282) at 00:06.\n2. I saw and counted each time the user grabbed the spoon (large plastic spoon-56b410eb3a192282), calculating a total of 4 times at 00:06, 01:48, 05:57, and 09:32.\n3. I calculated 4*2=8.\n4. Therefore, if the user doubled the amount of times they grabbed the stirring spoon (large plastic spoon-56b410eb3a192282), they would have grabbed it 8 times.", "referenced_object_masks": [ "large plastic spoon-56b410eb3a192282" ] }, { "key": "P02-20240211-115653:dd80399e60c654f70cc8afa0d0d27ee7a5a13577", "video_id": "P02-20240211-115653", "question": "How many types of cheese did I add to the puff pastry roll?", "answer_choice_0": "4", "answer_choice_1": "2", "answer_choice_2": "1", "answer_choice_3": "0", "answer_choice_4": "3", "answer_id": 1, "answer": "2", "question_type": [ "Object Recognition", "Action Recognition", "Counting" ], "objects_of_interest": [ "recording phone-82b879e1583c10f2", "large plastic spoon-56b410eb3a192282", "bowl-64c16b0e0e04861b", "kitchen roll holder-45f25d232a90dc1f", "kitchen towel-cfe0bd72554bc991", "tupperware box-25f304be17f0aba1", "puff pastry-fb9290ebaaeb982f", "block of cheddar cheese-67ec25da5435c7e5", "loaf of bread-bcb3abcad69367a1", "other packet of cheese-d9a2683b43561b72", "puff pastry roll-74f05813bd2d7a5f", "scissors-e702216bed37c119", "tea towel-b69430d711f74ec4", "grater-207bbe498a4459fc", "rolling pin-d57486f705c69885", "green bag-3ff3fa129e17ffe6", "paper cover-a7b81286039719c5", "serrated knife-828526a18cd9960e" ], "reasoning": "1. I watched the video and found the puff pastry roll (puff pastry roll-74f05813bd2d7a5f) at 01:22.\n2. I saw the user add shredded cheese from a packet (other packet of cheese-d9a2683b43561b72) to the puff pastry roll (puff pastry roll-74f05813bd2d7a5f) at 02:42.\n3. I saw the user add more cheese to the puff pastry roll (puff pastry roll-74f05813bd2d7a5f) by using a grater (grater-207bbe498a4459fc) to shred a block of cheddar (block of cheddar cheese-67ec25da5435c7e5) at 03:44.\n4. I counted the types of cheese (other packet of cheese-d9a2683b43561b72, block of cheddar cheese-67ec25da5435c7e5) that the user added to the puff pastry roll (puff pastry roll-74f05813bd2d7a5f), calculating a total of 2.\n5. Therefore, the user added 2 types of cheese (other packet of cheese-d9a2683b43561b72, block of cheddar cheese-67ec25da5435c7e5) to the puff pastry roll (puff pastry roll-74f05813bd2d7a5f).", "referenced_object_masks": [ "grater-207bbe498a4459fc", "other packet of cheese-d9a2683b43561b72", "block of cheddar cheese-67ec25da5435c7e5", "puff pastry roll-74f05813bd2d7a5f" ] }, { "key": "P02-20240211-115653:f3d35f655de62fb140bc3d1eb70a5c4307f2c5d1", "video_id": "P02-20240211-115653", "question": "Compared to the number of horizontal cuts, how many more vertical cuts did I make into the puff pastry roll with the serrated knife?", "answer_choice_0": "16", "answer_choice_1": "14", "answer_choice_2": "20", "answer_choice_3": "12", "answer_choice_4": "18", "answer_id": 1, "answer": "14", "question_type": [ "Object Recognition", "Action Recognition", "Counting", "Egocentric & Spatial Perception", "Numerical Reasoning" ], "objects_of_interest": [ "recording phone-82b879e1583c10f2", "large plastic spoon-56b410eb3a192282", "bowl-64c16b0e0e04861b", "kitchen roll holder-45f25d232a90dc1f", "kitchen towel-cfe0bd72554bc991", "tupperware box-25f304be17f0aba1", "puff pastry-fb9290ebaaeb982f", "block of cheddar cheese-67ec25da5435c7e5", "loaf of bread-bcb3abcad69367a1", "other packet of cheese-d9a2683b43561b72", "puff pastry roll-74f05813bd2d7a5f", "scissors-e702216bed37c119", "tea towel-b69430d711f74ec4", "grater-207bbe498a4459fc", "rolling pin-d57486f705c69885", "green bag-3ff3fa129e17ffe6", "paper cover-a7b81286039719c5", "serrated knife-828526a18cd9960e" ], "reasoning": "1. I watched the video and found the puff pastry roll (puff pastry roll-74f05813bd2d7a5f) at 01:22.\n2. I found the serrated knife (serrated knife-828526a18cd9960e) at 07:56.\n3. I saw the user use the serrated knife (serrated knife-828526a18cd9960e) to cut the puff pastry roll (puff pastry roll-74f05813bd2d7a5f) vertically at 08:14.\n4. I counted the number of vertical cuts that the user made into the puff pastry roll (puff pastry roll-74f05813bd2d7a5f) with the serrated knife (serrated knife-828526a18cd9960e), calculating a total of 15.\n5. I saw the user use the serrated knife (serrated knife-828526a18cd9960e) to cut the puff pastry roll (puff pastry roll-74f05813bd2d7a5f) horizontally at 09:25.\n6. I counted the number of horizontal cuts that user made into the puff pastry roll (puff pastry roll-74f05813bd2d7a5f) with the serrated knife (serrated knife-828526a18cd9960e), calculating a total of 1.\n7. I calculated 15-1=14.\n8. Therefore, compared to the number of horizontal cuts, the user made 14 more vertical cuts into the puff pastry roll (puff pastry roll-74f05813bd2d7a5f) with the serrated knife (serrated knife-828526a18cd9960e).", "referenced_object_masks": [ "serrated knife-828526a18cd9960e", "puff pastry roll-74f05813bd2d7a5f" ] }, { "key": "P02-20240211-120927:26d00a0a2ce156097b69039a0c852fefa2e9007e", "video_id": "P02-20240211-120927", "question": "What does the food inside the fourth item I touched in the refrigerator have in common with the frontmost dessert visible when I flip through the recipe book after stirring the jam?", "answer_choice_0": "They are both yellow", "answer_choice_1": "They are both pink", "answer_choice_2": "They are both brown", "answer_choice_3": "They are both red", "answer_choice_4": "They are both orange", "answer_id": 0, "answer": "They are both yellow", "question_type": [ "Object Recognition", "Egocentric & Spatial Perception", "Action Recognition", "Counting" ], "objects_of_interest": [ "phone-243b27482388aed0", "large mixing bowl-bb8fb85c4593aea5", "baking tray-7246377423a4b5b5", "bag of flour-7f44b1b7773eb449", "tupperware lid-2a6f8cce5550c53a", "tupperware-2637df1f0730c42c", "grater-679c874181021bca", "scissors-26a9a0777eaaf0ca", "sponge-cedd35310f54c786", "baking tray2-fe0973564cbcfb93", "grated cheese packet-46c0b93d50086fd8", "offcut-ce6e0534003f1c44", "large plastic spoon-86f4f0ac59f7d81c", "tea towel-5437d0591d066039", "recipe book-4ba3415a209ba852", "saucepan-97713b1003750492", "two muffin trays-8753dd6a7d4a989f", "muffin tray1-f702138b67a5ab90", "muffin tray2-a931cbc3e64b92f2", "mini oven grips-4488937f765e4a3d", "mini oven grips2-3ebff51168f04583", "empty jar1-c5839e1c3a0f3327", "empty jar2-0d677bc73b36ea69", "rubber seal1-33726e35011431db", "rubber seal2-8b5086dc75a342d3", "puff pastry wrapper-93959b1b49184044", "roll of puff pastry-3aa8abeba9d86801", "serrated knife-9fc261df8cd4959d", "stick of butter-575f68e053486c2e", "oven gloves-afd38f8fc5a6b804", "other oven glove-2b121ead59a1341a", "puff pasty paper-326e5914b9c8855b", "rolling pen-20062fbe63c134e7", "food bin-e7d9a3695c22e44c", "used kitchen roll-df15b5c69e34917f", "glass countertop protector-889abc9fc4c9056b", "teaspoon-0c78266a10163e86", "saucer-4f70495cdd8f4dc7", "spatula-7b211b8d2dd8426e", "tupperware box-5bde95cee993d79a", "saucer2-17e51efda1827495", "cheese straw1-281812373589e956", "cheese straw2-a1a3f62fe9473cc1", "cheese straw3-6d64b51eab4c905b", "cheese straw4-e5edae14aba23e21", "two straw5-6ae035dfcac8db55", "two straw6-7025f6e2e8c44617", "cheese straw7-4578a15c6a38fcf7", "cheese straw8-ba51e65b3198287b", "cheese straw9-0e1abd3140692e8a", "cheese straw10-3bd20a10c82f14ce", "cheese straw11-ffd4a11ed619a621", "cheese straw12-a6201cf3c334da71", "cheese straw16-8d3c947a10fcdef8", "cheese straw13-8a8c1e5f59b768ec", "cheese straw14-b40f620ef1893965", "cheese straw15-cc1cb3db1692ea3f", "cheese straw17-d414993c7e9e1fe3", "cheese straw18-e2fe7d7aca3fef0b", "cheese straw19-1f4bdbde955cdb82", "cheese straw20-0c5f8ecfe662743a", "cheese straw24-b499ade6d727baba", "cheese straw21-42d31a873a3406ee", "cheese straw22-84d1ef4db454d318", "cheese straw23-6b941303a714ba84", "cheese straw26-002b23064ea14368", "cheese straw25-a5cf6d7d6f61c2ae", "cheese straw27-efbbfb6acaa5686c", "cheese straw28-7f86e27519cb022a", "cheese straw29-43ec21c830d2ac25", "cheese straw30-923dc84e193b3898", "tablespoon-c91862f8a37ffd69", "end of baguette-7d49e2a4f8418ea4", "plate-279f0aa582595079", "baguette-0818d64c6caf8f71", "bread knife-ce621c01fe05c9a9", "baguette piece-5a5f824ed819f3b7", "baguette piece2-64b1052109b9eb2e", "tub of butter-e581e1f315f53eee", "cheese-5c86336516dd4ed8", "butter knife-418756353a51e627", "tub of butter lid-ec0991326bc19677", "plate2-cdacb233d95d0bb2", "slice of cheese-5a94e59d4259c7b9", "slice of cheese2-32df7a30d1fef661", "slice of cheese3-28d109562aae57d7", "slice of cheese4-ff8db54687f85878" ], "reasoning": "1. First, I looked for the user to touch items in the refrigerator . I saw them place a packet of grated cheese (grated cheese packet-46c0b93d50086fd8) in the refrigerator at 03:43. I counted this as the first item. \n2. Next, I saw the user grab a packet of dough (puff pastry wrapper-93959b1b49184044, roll of puff pastry-3aa8abeba9d86801) from the refrigerator at 08:00. I counted this as the 2nd item. \n3. Then, at 09:57, I saw the user briefly touch an item before grabbing the butter (stick of butter-575f68e053486c2e), so I counted the butter (stick of butter-575f68e053486c2e) as the fourth item the user touched in the refrigerator . \n4. I looked for the user to stir the jam before flipping through a recipe book (recipe book-4ba3415a209ba852), and saw this happen at 23:26-23:36. \n5. Watching from 22:36, I paid attention to which dessert featured in the recipe book (recipe book-4ba3415a209ba852) was closest to the front, and found a clear image of a yellow dessert on a page closest to the front at 23:43. \n6. Since the butter (stick of butter-575f68e053486c2e) is also yellow, I determined that what the frontmost recipe visible in the recipe book (recipe book-4ba3415a209ba852) has in common with the fourth item the user touched is that they are both yellow.", "referenced_object_masks": [ "recipe book-4ba3415a209ba852", "stick of butter-575f68e053486c2e", "puff pastry wrapper-93959b1b49184044", "roll of puff pastry-3aa8abeba9d86801", "grated cheese packet-46c0b93d50086fd8" ] }, { "key": "P02-20240211-120927:35af727d29eceaaf9c048d0123abb89fb36b6770", "video_id": "P02-20240211-120927", "question": "What is the second to last item I moved on the kitchen counter before grabbing the tray and placing it next to the Phyllo dough?", "answer_choice_0": "Flour bag", "answer_choice_1": "Grater", "answer_choice_2": "Tupperware lid", "answer_choice_3": "Sponge", "answer_choice_4": "Scissors", "answer_id": 4, "answer": "Scissors", "question_type": [ "Event Occurence", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "phone-243b27482388aed0", "large mixing bowl-bb8fb85c4593aea5", "baking tray-7246377423a4b5b5", "bag of flour-7f44b1b7773eb449", "tupperware lid-2a6f8cce5550c53a", "tupperware-2637df1f0730c42c", "grater-679c874181021bca", "scissors-26a9a0777eaaf0ca", "sponge-cedd35310f54c786", "baking tray2-fe0973564cbcfb93", "grated cheese packet-46c0b93d50086fd8", "offcut-ce6e0534003f1c44", "large plastic spoon-86f4f0ac59f7d81c", "tea towel-5437d0591d066039", "recipe book-4ba3415a209ba852", "saucepan-97713b1003750492", "two muffin trays-8753dd6a7d4a989f", "muffin tray1-f702138b67a5ab90", "muffin tray2-a931cbc3e64b92f2", "mini oven grips-4488937f765e4a3d", "mini oven grips2-3ebff51168f04583", "empty jar1-c5839e1c3a0f3327", "empty jar2-0d677bc73b36ea69", "rubber seal1-33726e35011431db", "rubber seal2-8b5086dc75a342d3", "puff pastry wrapper-93959b1b49184044", "roll of puff pastry-3aa8abeba9d86801", "serrated knife-9fc261df8cd4959d", "stick of butter-575f68e053486c2e", "oven gloves-afd38f8fc5a6b804", "other oven glove-2b121ead59a1341a", "puff pasty paper-326e5914b9c8855b", "rolling pen-20062fbe63c134e7", "food bin-e7d9a3695c22e44c", "used kitchen roll-df15b5c69e34917f", "glass countertop protector-889abc9fc4c9056b", "teaspoon-0c78266a10163e86", "saucer-4f70495cdd8f4dc7", "spatula-7b211b8d2dd8426e", "tupperware box-5bde95cee993d79a", "saucer2-17e51efda1827495", "cheese straw1-281812373589e956", "cheese straw2-a1a3f62fe9473cc1", "cheese straw3-6d64b51eab4c905b", "cheese straw4-e5edae14aba23e21", "two straw5-6ae035dfcac8db55", "two straw6-7025f6e2e8c44617", "cheese straw7-4578a15c6a38fcf7", "cheese straw8-ba51e65b3198287b", "cheese straw9-0e1abd3140692e8a", "cheese straw10-3bd20a10c82f14ce", "cheese straw11-ffd4a11ed619a621", "cheese straw12-a6201cf3c334da71", "cheese straw16-8d3c947a10fcdef8", "cheese straw13-8a8c1e5f59b768ec", "cheese straw14-b40f620ef1893965", "cheese straw15-cc1cb3db1692ea3f", "cheese straw17-d414993c7e9e1fe3", "cheese straw18-e2fe7d7aca3fef0b", "cheese straw19-1f4bdbde955cdb82", "cheese straw20-0c5f8ecfe662743a", "cheese straw24-b499ade6d727baba", "cheese straw21-42d31a873a3406ee", "cheese straw22-84d1ef4db454d318", "cheese straw23-6b941303a714ba84", "cheese straw26-002b23064ea14368", "cheese straw25-a5cf6d7d6f61c2ae", "cheese straw27-efbbfb6acaa5686c", "cheese straw28-7f86e27519cb022a", "cheese straw29-43ec21c830d2ac25", "cheese straw30-923dc84e193b3898", "tablespoon-c91862f8a37ffd69", "end of baguette-7d49e2a4f8418ea4", "plate-279f0aa582595079", "baguette-0818d64c6caf8f71", "bread knife-ce621c01fe05c9a9", "baguette piece-5a5f824ed819f3b7", "baguette piece2-64b1052109b9eb2e", "tub of butter-e581e1f315f53eee", "cheese-5c86336516dd4ed8", "butter knife-418756353a51e627", "tub of butter lid-ec0991326bc19677", "plate2-cdacb233d95d0bb2", "slice of cheese-5a94e59d4259c7b9", "slice of cheese2-32df7a30d1fef661", "slice of cheese3-28d109562aae57d7", "slice of cheese4-ff8db54687f85878" ], "reasoning": "1. I watched the video and found the user placing the baking tray (baking tray2-fe0973564cbcfb93) next to the Phyllo dough at 00:32. \n2. I played the video backwards to find the last object on the counter that was moved. From 00:29 to 00:30, the user moved the grater (grater-679c874181021bca), then the scissors (scissors-26a9a0777eaaf0ca), and finally the sponge (sponge-cedd35310f54c786). \n3. Therefore, the second to last item the user moved are the scissors (scissors-26a9a0777eaaf0ca).", "referenced_object_masks": [ "scissors-26a9a0777eaaf0ca", "baking tray2-fe0973564cbcfb93", "sponge-cedd35310f54c786", "grater-679c874181021bca" ] }, { "key": "P02-20240211-120927:56d28c36ca9a8296d86296e1b8fc2eb8050747b6", "video_id": "P02-20240211-120927", "question": "If I was standing on the rug with hearts and facing in the same direction their top halves are pointing, approximately how many degrees in a clockwise direction would I turn to face the cabinet where I retrieved the muffin pans?", "answer_choice_0": "270 degrees", "answer_choice_1": "45 degrees", "answer_choice_2": "135 degrees", "answer_choice_3": "225 degrees", "answer_choice_4": "90 degrees", "answer_id": 2, "answer": "135 degrees", "question_type": [ "Egocentric & Spatial Perception", "Counterfactual", "Situational Awareness", "Object Recognition" ], "objects_of_interest": [ "phone-243b27482388aed0", "large mixing bowl-bb8fb85c4593aea5", "baking tray-7246377423a4b5b5", "bag of flour-7f44b1b7773eb449", "tupperware lid-2a6f8cce5550c53a", "tupperware-2637df1f0730c42c", "grater-679c874181021bca", "scissors-26a9a0777eaaf0ca", "sponge-cedd35310f54c786", "baking tray2-fe0973564cbcfb93", "grated cheese packet-46c0b93d50086fd8", "offcut-ce6e0534003f1c44", "large plastic spoon-86f4f0ac59f7d81c", "tea towel-5437d0591d066039", "recipe book-4ba3415a209ba852", "saucepan-97713b1003750492", "two muffin trays-8753dd6a7d4a989f", "muffin tray1-f702138b67a5ab90", "muffin tray2-a931cbc3e64b92f2", "mini oven grips-4488937f765e4a3d", "mini oven grips2-3ebff51168f04583", "empty jar1-c5839e1c3a0f3327", "empty jar2-0d677bc73b36ea69", "rubber seal1-33726e35011431db", "rubber seal2-8b5086dc75a342d3", "puff pastry wrapper-93959b1b49184044", "roll of puff pastry-3aa8abeba9d86801", "serrated knife-9fc261df8cd4959d", "stick of butter-575f68e053486c2e", "oven gloves-afd38f8fc5a6b804", "other oven glove-2b121ead59a1341a", "puff pasty paper-326e5914b9c8855b", "rolling pen-20062fbe63c134e7", "food bin-e7d9a3695c22e44c", "used kitchen roll-df15b5c69e34917f", "glass countertop protector-889abc9fc4c9056b", "teaspoon-0c78266a10163e86", "saucer-4f70495cdd8f4dc7", "spatula-7b211b8d2dd8426e", "tupperware box-5bde95cee993d79a", "saucer2-17e51efda1827495", "cheese straw1-281812373589e956", "cheese straw2-a1a3f62fe9473cc1", "cheese straw3-6d64b51eab4c905b", "cheese straw4-e5edae14aba23e21", "two straw5-6ae035dfcac8db55", "two straw6-7025f6e2e8c44617", "cheese straw7-4578a15c6a38fcf7", "cheese straw8-ba51e65b3198287b", "cheese straw9-0e1abd3140692e8a", "cheese straw10-3bd20a10c82f14ce", "cheese straw11-ffd4a11ed619a621", "cheese straw12-a6201cf3c334da71", "cheese straw16-8d3c947a10fcdef8", "cheese straw13-8a8c1e5f59b768ec", "cheese straw14-b40f620ef1893965", "cheese straw15-cc1cb3db1692ea3f", "cheese straw17-d414993c7e9e1fe3", "cheese straw18-e2fe7d7aca3fef0b", "cheese straw19-1f4bdbde955cdb82", "cheese straw20-0c5f8ecfe662743a", "cheese straw24-b499ade6d727baba", "cheese straw21-42d31a873a3406ee", "cheese straw22-84d1ef4db454d318", "cheese straw23-6b941303a714ba84", "cheese straw26-002b23064ea14368", "cheese straw25-a5cf6d7d6f61c2ae", "cheese straw27-efbbfb6acaa5686c", "cheese straw28-7f86e27519cb022a", "cheese straw29-43ec21c830d2ac25", "cheese straw30-923dc84e193b3898", "tablespoon-c91862f8a37ffd69", "end of baguette-7d49e2a4f8418ea4", "plate-279f0aa582595079", "baguette-0818d64c6caf8f71", "bread knife-ce621c01fe05c9a9", "baguette piece-5a5f824ed819f3b7", "baguette piece2-64b1052109b9eb2e", "tub of butter-e581e1f315f53eee", "cheese-5c86336516dd4ed8", "butter knife-418756353a51e627", "tub of butter lid-ec0991326bc19677", "plate2-cdacb233d95d0bb2", "slice of cheese-5a94e59d4259c7b9", "slice of cheese2-32df7a30d1fef661", "slice of cheese3-28d109562aae57d7", "slice of cheese4-ff8db54687f85878" ], "reasoning": "1. First, I looked for a rug with hearts and found a clear visual of this at the very beginning of the video at 00:00. I took note of where the rug is in relation to the rest of the space. \n2. Next, I looked for the user to retrieve the muffin pans (two muffin trays-8753dd6a7d4a989f) from a cabinet and found that they do this at 05:11. I took note that this cabinet is in front of and to the left of the rug with hearts . \n3. Noting that the top halves of the hearts are pointing away from the bottom of the rug , I then imagined the user standing on the rug . I realized that they would need to turn approximately 135 degrees in a clockwise direction in order to face the cabinet with the muffin pans (two muffin trays-8753dd6a7d4a989f).", "referenced_object_masks": [ "two muffin trays-8753dd6a7d4a989f" ] }, { "key": "P02-20240211-120927:7a4bac3a55031a4715f4fab06a43737e7ca7ae37", "video_id": "P02-20240211-120927", "question": "If I'm standing in front of the sink and turn 90 degrees to my left, approximately where would the food bin be in relation to me when the food bin is in its original location?", "answer_choice_0": "130 degrees clockwise", "answer_choice_1": "180 degrees in either direction", "answer_choice_2": "45 degrees clockwise", "answer_choice_3": "120 degrees counterclockwise", "answer_choice_4": "90 degrees counterclockwise", "answer_id": 3, "answer": "120 degrees counterclockwise", "question_type": [ "Event Occurence", "Spatial Perception", "State Changes" ], "objects_of_interest": [ "phone-243b27482388aed0", "large mixing bowl-bb8fb85c4593aea5", "baking tray-7246377423a4b5b5", "bag of flour-7f44b1b7773eb449", "tupperware lid-2a6f8cce5550c53a", "tupperware-2637df1f0730c42c", "grater-679c874181021bca", "scissors-26a9a0777eaaf0ca", "sponge-cedd35310f54c786", "baking tray2-fe0973564cbcfb93", "grated cheese packet-46c0b93d50086fd8", "offcut-ce6e0534003f1c44", "large plastic spoon-86f4f0ac59f7d81c", "tea towel-5437d0591d066039", "recipe book-4ba3415a209ba852", "saucepan-97713b1003750492", "two muffin trays-8753dd6a7d4a989f", "muffin tray1-f702138b67a5ab90", "muffin tray2-a931cbc3e64b92f2", "mini oven grips-4488937f765e4a3d", "mini oven grips2-3ebff51168f04583", "empty jar1-c5839e1c3a0f3327", "empty jar2-0d677bc73b36ea69", "rubber seal1-33726e35011431db", "rubber seal2-8b5086dc75a342d3", "puff pastry wrapper-93959b1b49184044", "roll of puff pastry-3aa8abeba9d86801", "serrated knife-9fc261df8cd4959d", "stick of butter-575f68e053486c2e", "oven gloves-afd38f8fc5a6b804", "other oven glove-2b121ead59a1341a", "puff pasty paper-326e5914b9c8855b", "rolling pen-20062fbe63c134e7", "food bin-e7d9a3695c22e44c", "used kitchen roll-df15b5c69e34917f", "glass countertop protector-889abc9fc4c9056b", "teaspoon-0c78266a10163e86", "saucer-4f70495cdd8f4dc7", "spatula-7b211b8d2dd8426e", "tupperware box-5bde95cee993d79a", "saucer2-17e51efda1827495", "cheese straw1-281812373589e956", "cheese straw2-a1a3f62fe9473cc1", "cheese straw3-6d64b51eab4c905b", "cheese straw4-e5edae14aba23e21", "two straw5-6ae035dfcac8db55", "two straw6-7025f6e2e8c44617", "cheese straw7-4578a15c6a38fcf7", "cheese straw8-ba51e65b3198287b", "cheese straw9-0e1abd3140692e8a", "cheese straw10-3bd20a10c82f14ce", "cheese straw11-ffd4a11ed619a621", "cheese straw12-a6201cf3c334da71", "cheese straw16-8d3c947a10fcdef8", "cheese straw13-8a8c1e5f59b768ec", "cheese straw14-b40f620ef1893965", "cheese straw15-cc1cb3db1692ea3f", "cheese straw17-d414993c7e9e1fe3", "cheese straw18-e2fe7d7aca3fef0b", "cheese straw19-1f4bdbde955cdb82", "cheese straw20-0c5f8ecfe662743a", "cheese straw24-b499ade6d727baba", "cheese straw21-42d31a873a3406ee", "cheese straw22-84d1ef4db454d318", "cheese straw23-6b941303a714ba84", "cheese straw26-002b23064ea14368", "cheese straw25-a5cf6d7d6f61c2ae", "cheese straw27-efbbfb6acaa5686c", "cheese straw28-7f86e27519cb022a", "cheese straw29-43ec21c830d2ac25", "cheese straw30-923dc84e193b3898", "tablespoon-c91862f8a37ffd69", "end of baguette-7d49e2a4f8418ea4", "plate-279f0aa582595079", "baguette-0818d64c6caf8f71", "bread knife-ce621c01fe05c9a9", "baguette piece-5a5f824ed819f3b7", "baguette piece2-64b1052109b9eb2e", "tub of butter-e581e1f315f53eee", "cheese-5c86336516dd4ed8", "butter knife-418756353a51e627", "tub of butter lid-ec0991326bc19677", "plate2-cdacb233d95d0bb2", "slice of cheese-5a94e59d4259c7b9", "slice of cheese2-32df7a30d1fef661", "slice of cheese3-28d109562aae57d7", "slice of cheese4-ff8db54687f85878" ], "reasoning": "1. I watched the video from the beginning to identify where the sink is located in relation to the rest of the kitchen. \n2. I identified at 00:00, that the kitchen has a U-shape design, with the sink right in the middle, in front of a window. \n3. I located the user walking towards the sink for the first time at 00:04. The user then makes a 90 degree turn to the left, and ends up in front of the stove, handling a mixing bowl (large mixing bowl-bb8fb85c4593aea5) that is placed on top of a sauce pan. Therefore, if the user is standing in front of the sink and makes a 90 degree turn to the left, the appliance directly in front will be the stove.\n4. Next, I looked for the food bin (food bin-e7d9a3695c22e44c). I found the food bin (food bin-e7d9a3695c22e44c) at 19:07, located down and directly to the right of the countertop where the user cut puff pastry paper (puff pasty paper-326e5914b9c8855b). At 19:07, the user picks up the food bin (food bin-e7d9a3695c22e44c).\n5. Since the countertop where the user cut puff pastry paper (puff pasty paper-326e5914b9c8855b) is 180 degrees away from the oven in either direction, I determined that the food bin (food bin-e7d9a3695c22e44c) would be to the left of the user if they were standing at the oven. This would make the angle approximately 120 degrees counterclockwise.", "referenced_object_masks": [ "puff pasty paper-326e5914b9c8855b", "large mixing bowl-bb8fb85c4593aea5", "food bin-e7d9a3695c22e44c" ] }, { "key": "P02-20240211-120927:8598422348ef374f565370277aab6329470bbb45", "video_id": "P02-20240211-120927", "question": "What is the fourth unique space in which I set one, or both, of the items I retrieved from behind coffee mugs?", "answer_choice_0": "A sink", "answer_choice_1": "A refrigerator", "answer_choice_2": "A cabinet", "answer_choice_3": "A drawer", "answer_choice_4": "An oven", "answer_id": 4, "answer": "An oven", "question_type": [ "Action Recognition", "Object Recognition", "Counting" ], "objects_of_interest": [ "phone-243b27482388aed0", "large mixing bowl-bb8fb85c4593aea5", "baking tray-7246377423a4b5b5", "bag of flour-7f44b1b7773eb449", "tupperware lid-2a6f8cce5550c53a", "tupperware-2637df1f0730c42c", "grater-679c874181021bca", "scissors-26a9a0777eaaf0ca", "sponge-cedd35310f54c786", "baking tray2-fe0973564cbcfb93", "grated cheese packet-46c0b93d50086fd8", "offcut-ce6e0534003f1c44", "large plastic spoon-86f4f0ac59f7d81c", "tea towel-5437d0591d066039", "recipe book-4ba3415a209ba852", "saucepan-97713b1003750492", "two muffin trays-8753dd6a7d4a989f", "muffin tray1-f702138b67a5ab90", "muffin tray2-a931cbc3e64b92f2", "mini oven grips-4488937f765e4a3d", "mini oven grips2-3ebff51168f04583", "empty jar1-c5839e1c3a0f3327", "empty jar2-0d677bc73b36ea69", "rubber seal1-33726e35011431db", "rubber seal2-8b5086dc75a342d3", "puff pastry wrapper-93959b1b49184044", "roll of puff pastry-3aa8abeba9d86801", "serrated knife-9fc261df8cd4959d", "stick of butter-575f68e053486c2e", "oven gloves-afd38f8fc5a6b804", "other oven glove-2b121ead59a1341a", "puff pasty paper-326e5914b9c8855b", "rolling pen-20062fbe63c134e7", "food bin-e7d9a3695c22e44c", "used kitchen roll-df15b5c69e34917f", "glass countertop protector-889abc9fc4c9056b", "teaspoon-0c78266a10163e86", "saucer-4f70495cdd8f4dc7", "spatula-7b211b8d2dd8426e", "tupperware box-5bde95cee993d79a", "saucer2-17e51efda1827495", "cheese straw1-281812373589e956", "cheese straw2-a1a3f62fe9473cc1", "cheese straw3-6d64b51eab4c905b", "cheese straw4-e5edae14aba23e21", "two straw5-6ae035dfcac8db55", "two straw6-7025f6e2e8c44617", "cheese straw7-4578a15c6a38fcf7", "cheese straw8-ba51e65b3198287b", "cheese straw9-0e1abd3140692e8a", "cheese straw10-3bd20a10c82f14ce", "cheese straw11-ffd4a11ed619a621", "cheese straw12-a6201cf3c334da71", "cheese straw16-8d3c947a10fcdef8", "cheese straw13-8a8c1e5f59b768ec", "cheese straw14-b40f620ef1893965", "cheese straw15-cc1cb3db1692ea3f", "cheese straw17-d414993c7e9e1fe3", "cheese straw18-e2fe7d7aca3fef0b", "cheese straw19-1f4bdbde955cdb82", "cheese straw20-0c5f8ecfe662743a", "cheese straw24-b499ade6d727baba", "cheese straw21-42d31a873a3406ee", "cheese straw22-84d1ef4db454d318", "cheese straw23-6b941303a714ba84", "cheese straw26-002b23064ea14368", "cheese straw25-a5cf6d7d6f61c2ae", "cheese straw27-efbbfb6acaa5686c", "cheese straw28-7f86e27519cb022a", "cheese straw29-43ec21c830d2ac25", "cheese straw30-923dc84e193b3898", "tablespoon-c91862f8a37ffd69", "end of baguette-7d49e2a4f8418ea4", "plate-279f0aa582595079", "baguette-0818d64c6caf8f71", "bread knife-ce621c01fe05c9a9", "baguette piece-5a5f824ed819f3b7", "baguette piece2-64b1052109b9eb2e", "tub of butter-e581e1f315f53eee", "cheese-5c86336516dd4ed8", "butter knife-418756353a51e627", "tub of butter lid-ec0991326bc19677", "plate2-cdacb233d95d0bb2", "slice of cheese-5a94e59d4259c7b9", "slice of cheese2-32df7a30d1fef661", "slice of cheese3-28d109562aae57d7", "slice of cheese4-ff8db54687f85878" ], "reasoning": "1. First, I looked for the user to retrieve items from behind coffee mugs. I found that they grab two glass jars (empty jar1-c5839e1c3a0f3327, empty jar2-0d677bc73b36ea69) from behind several coffee mugs in a cabinet at 06:23-06:28. \n2. Next, I watched to see where the user would place the glass jars (empty jar1-c5839e1c3a0f3327, empty jar2-0d677bc73b36ea69). I saw them place one (empty jar1-c5839e1c3a0f3327) on a counter at 06:33 and counted the counter as the first unique place. \n3. Then, at 06:39, I saw the user place both glass jars (empty jar1-c5839e1c3a0f3327, empty jar2-0d677bc73b36ea69) into the sink , and counted that as the second unique place. \n4. At 07:24, I saw the user place both glass jars (empty jar1-c5839e1c3a0f3327, empty jar2-0d677bc73b36ea69) on the drying area next to the sink , and counted that as the third unique space. \n5. Finally, at 22:27, I saw the user grab the jars (empty jar1-c5839e1c3a0f3327, empty jar2-0d677bc73b36ea69) from the drying area , and place them in the oven at 22:50-22:52. I counted this as the fourth unique place, making an \"oven\" the answer.", "referenced_object_masks": [ "empty jar1-c5839e1c3a0f3327", "empty jar2-0d677bc73b36ea69" ] }, { "key": "P02-20240211-120927:8a6d51e9e8ccc25c002ff3268051af3af6057877", "video_id": "P02-20240211-120927", "question": "How many guests can I feed if I serve each two cheese sticks and one puff pastry?", "answer_choice_0": "31", "answer_choice_1": "30", "answer_choice_2": "18", "answer_choice_3": "15", "answer_choice_4": "13", "answer_id": 3, "answer": "15", "question_type": [ "Counting", "Event Occurence", "Goal Reasoning", "Numerical Reasoning", "Object Recognition" ], "objects_of_interest": [ "phone-243b27482388aed0", "large mixing bowl-bb8fb85c4593aea5", "baking tray-7246377423a4b5b5", "bag of flour-7f44b1b7773eb449", "tupperware lid-2a6f8cce5550c53a", "tupperware-2637df1f0730c42c", "grater-679c874181021bca", "scissors-26a9a0777eaaf0ca", "sponge-cedd35310f54c786", "baking tray2-fe0973564cbcfb93", "grated cheese packet-46c0b93d50086fd8", "offcut-ce6e0534003f1c44", "large plastic spoon-86f4f0ac59f7d81c", "tea towel-5437d0591d066039", "recipe book-4ba3415a209ba852", "saucepan-97713b1003750492", "two muffin trays-8753dd6a7d4a989f", "muffin tray1-f702138b67a5ab90", "muffin tray2-a931cbc3e64b92f2", "mini oven grips-4488937f765e4a3d", "mini oven grips2-3ebff51168f04583", "empty jar1-c5839e1c3a0f3327", "empty jar2-0d677bc73b36ea69", "rubber seal1-33726e35011431db", "rubber seal2-8b5086dc75a342d3", "puff pastry wrapper-93959b1b49184044", "roll of puff pastry-3aa8abeba9d86801", "serrated knife-9fc261df8cd4959d", "stick of butter-575f68e053486c2e", "oven gloves-afd38f8fc5a6b804", "other oven glove-2b121ead59a1341a", "puff pasty paper-326e5914b9c8855b", "rolling pen-20062fbe63c134e7", "food bin-e7d9a3695c22e44c", "used kitchen roll-df15b5c69e34917f", "glass countertop protector-889abc9fc4c9056b", "teaspoon-0c78266a10163e86", "saucer-4f70495cdd8f4dc7", "spatula-7b211b8d2dd8426e", "tupperware box-5bde95cee993d79a", "saucer2-17e51efda1827495", "cheese straw1-281812373589e956", "cheese straw2-a1a3f62fe9473cc1", "cheese straw3-6d64b51eab4c905b", "cheese straw4-e5edae14aba23e21", "two straw5-6ae035dfcac8db55", "two straw6-7025f6e2e8c44617", "cheese straw7-4578a15c6a38fcf7", "cheese straw8-ba51e65b3198287b", "cheese straw9-0e1abd3140692e8a", "cheese straw10-3bd20a10c82f14ce", "cheese straw11-ffd4a11ed619a621", "cheese straw12-a6201cf3c334da71", "cheese straw16-8d3c947a10fcdef8", "cheese straw13-8a8c1e5f59b768ec", "cheese straw14-b40f620ef1893965", "cheese straw15-cc1cb3db1692ea3f", "cheese straw17-d414993c7e9e1fe3", "cheese straw18-e2fe7d7aca3fef0b", "cheese straw19-1f4bdbde955cdb82", "cheese straw20-0c5f8ecfe662743a", "cheese straw24-b499ade6d727baba", "cheese straw21-42d31a873a3406ee", "cheese straw22-84d1ef4db454d318", "cheese straw23-6b941303a714ba84", "cheese straw26-002b23064ea14368", "cheese straw25-a5cf6d7d6f61c2ae", "cheese straw27-efbbfb6acaa5686c", "cheese straw28-7f86e27519cb022a", "cheese straw29-43ec21c830d2ac25", "cheese straw30-923dc84e193b3898", "tablespoon-c91862f8a37ffd69", "end of baguette-7d49e2a4f8418ea4", "plate-279f0aa582595079", "baguette-0818d64c6caf8f71", "bread knife-ce621c01fe05c9a9", "baguette piece-5a5f824ed819f3b7", "baguette piece2-64b1052109b9eb2e", "tub of butter-e581e1f315f53eee", "cheese-5c86336516dd4ed8", "butter knife-418756353a51e627", "tub of butter lid-ec0991326bc19677", "plate2-cdacb233d95d0bb2", "slice of cheese-5a94e59d4259c7b9", "slice of cheese2-32df7a30d1fef661", "slice of cheese3-28d109562aae57d7", "slice of cheese4-ff8db54687f85878" ], "reasoning": "1. I looked first at how many cheese sticks the user baked. From 00:33 to 03:40, the user is placing the raw cheese sticks on a baking tray (baking tray-7246377423a4b5b5), and I counted 31 cheese sticks in total. \n2. I then looked at the amount of puff pastries the user baked. \nAt 40:34, I counted 18 completed puff pastries on the kitchen counter. \n3. I divided the total number of cheese sticks by 2, which equals 15.5.\n4. Since the user wants to offer each guest a plate with two cheese sticks and one puff pastry, the subject can only feed 15 guests, since two cheese sticks per person equals 30 cheese sticks and he only has 31. \n5. Since the user baked 18 puff pastries, this will be enough to offer each of the 15 guests one puffed pastry and have some left over.", "referenced_object_masks": [ "baking tray-7246377423a4b5b5" ] }, { "key": "P02-20240211-120927:90b653f69e3f7a613752cac4380a78b4847e72eb", "video_id": "P02-20240211-120927", "question": "What time was it when I finished setting up the food on the kitchen counter?", "answer_choice_0": "12:58 A.M", "answer_choice_1": "2:00 P.M", "answer_choice_2": "1:00 P.M", "answer_choice_3": "1:00 A.M", "answer_choice_4": "12:57 P.M", "answer_id": 2, "answer": "1:00 P.M", "question_type": [ "Counting", "Event Occurence", "Numerical Reasoning" ], "objects_of_interest": [ "phone-243b27482388aed0", "large mixing bowl-bb8fb85c4593aea5", "baking tray-7246377423a4b5b5", "bag of flour-7f44b1b7773eb449", "tupperware lid-2a6f8cce5550c53a", "tupperware-2637df1f0730c42c", "grater-679c874181021bca", "scissors-26a9a0777eaaf0ca", "sponge-cedd35310f54c786", "baking tray2-fe0973564cbcfb93", "grated cheese packet-46c0b93d50086fd8", "offcut-ce6e0534003f1c44", "large plastic spoon-86f4f0ac59f7d81c", "tea towel-5437d0591d066039", "recipe book-4ba3415a209ba852", "saucepan-97713b1003750492", "two muffin trays-8753dd6a7d4a989f", "muffin tray1-f702138b67a5ab90", "muffin tray2-a931cbc3e64b92f2", "mini oven grips-4488937f765e4a3d", "mini oven grips2-3ebff51168f04583", "empty jar1-c5839e1c3a0f3327", "empty jar2-0d677bc73b36ea69", "rubber seal1-33726e35011431db", "rubber seal2-8b5086dc75a342d3", "puff pastry wrapper-93959b1b49184044", "roll of puff pastry-3aa8abeba9d86801", "serrated knife-9fc261df8cd4959d", "stick of butter-575f68e053486c2e", "oven gloves-afd38f8fc5a6b804", "other oven glove-2b121ead59a1341a", "puff pasty paper-326e5914b9c8855b", "rolling pen-20062fbe63c134e7", "food bin-e7d9a3695c22e44c", "used kitchen roll-df15b5c69e34917f", "glass countertop protector-889abc9fc4c9056b", "teaspoon-0c78266a10163e86", "saucer-4f70495cdd8f4dc7", "spatula-7b211b8d2dd8426e", "tupperware box-5bde95cee993d79a", "saucer2-17e51efda1827495", "cheese straw1-281812373589e956", "cheese straw2-a1a3f62fe9473cc1", "cheese straw3-6d64b51eab4c905b", "cheese straw4-e5edae14aba23e21", "two straw5-6ae035dfcac8db55", "two straw6-7025f6e2e8c44617", "cheese straw7-4578a15c6a38fcf7", "cheese straw8-ba51e65b3198287b", "cheese straw9-0e1abd3140692e8a", "cheese straw10-3bd20a10c82f14ce", "cheese straw11-ffd4a11ed619a621", "cheese straw12-a6201cf3c334da71", "cheese straw16-8d3c947a10fcdef8", "cheese straw13-8a8c1e5f59b768ec", "cheese straw14-b40f620ef1893965", "cheese straw15-cc1cb3db1692ea3f", "cheese straw17-d414993c7e9e1fe3", "cheese straw18-e2fe7d7aca3fef0b", "cheese straw19-1f4bdbde955cdb82", "cheese straw20-0c5f8ecfe662743a", "cheese straw24-b499ade6d727baba", "cheese straw21-42d31a873a3406ee", "cheese straw22-84d1ef4db454d318", "cheese straw23-6b941303a714ba84", "cheese straw26-002b23064ea14368", "cheese straw25-a5cf6d7d6f61c2ae", "cheese straw27-efbbfb6acaa5686c", "cheese straw28-7f86e27519cb022a", "cheese straw29-43ec21c830d2ac25", "cheese straw30-923dc84e193b3898", "tablespoon-c91862f8a37ffd69", "end of baguette-7d49e2a4f8418ea4", "plate-279f0aa582595079", "baguette-0818d64c6caf8f71", "bread knife-ce621c01fe05c9a9", "baguette piece-5a5f824ed819f3b7", "baguette piece2-64b1052109b9eb2e", "tub of butter-e581e1f315f53eee", "cheese-5c86336516dd4ed8", "butter knife-418756353a51e627", "tub of butter lid-ec0991326bc19677", "plate2-cdacb233d95d0bb2", "slice of cheese-5a94e59d4259c7b9", "slice of cheese2-32df7a30d1fef661", "slice of cheese3-28d109562aae57d7", "slice of cheese4-ff8db54687f85878" ], "reasoning": "1. First, I looked through the video for a moment where the time could be seen. At the 48:20 time stamp, the user grabs a cell phone (phone-243b27482388aed0) and on the screen it shows it was 12:57. Because it was daylight, it is safe to assume it was 12:57 pm.\n2. I kept looking from that moment on until the user finished setting up the food on the kitchen counter. This happened at the very end of the video, at the 51:56 timestamp specifically. \n3. I subtracted the 48:20 timestamp, which is the first time I saw it was 12:57 pm, from the 51:56 timestamp, which is when the user finished setting up the food, to calculate the amount of minutes that passed between the two events. This totaled 3 minutes and 36 seconds. \n4. Finally, I added the 3 minutes and 36 seconds to 12:57 pm, which indicates it was 36 seconds past 1:00 pm when the subject finished setting up the food on the kitchen counter. Since the cell phone (phone-243b27482388aed0) clock does not include seconds, it is safe to assume that it was 1:00 pm when the user was done.", "referenced_object_masks": [ "phone-243b27482388aed0" ] }, { "key": "P02-20240211-120927:9627856497b77fe65f0ea2d5dbee5b02374dd87f", "video_id": "P02-20240211-120927", "question": "An image of what type of beverage do I obscure with the pan of cheesesticks I took out of the oven the least?", "answer_choice_0": "Orange Juice", "answer_choice_1": "Red wine", "answer_choice_2": "Beer", "answer_choice_3": "Champagne", "answer_choice_4": "Coffee", "answer_id": 1, "answer": "Red wine", "question_type": [ "Action Recognition", "Object Recognition", "Counting" ], "objects_of_interest": [ "phone-243b27482388aed0", "large mixing bowl-bb8fb85c4593aea5", "baking tray-7246377423a4b5b5", "bag of flour-7f44b1b7773eb449", "tupperware lid-2a6f8cce5550c53a", "tupperware-2637df1f0730c42c", "grater-679c874181021bca", "scissors-26a9a0777eaaf0ca", "sponge-cedd35310f54c786", "baking tray2-fe0973564cbcfb93", "grated cheese packet-46c0b93d50086fd8", "offcut-ce6e0534003f1c44", "large plastic spoon-86f4f0ac59f7d81c", "tea towel-5437d0591d066039", "recipe book-4ba3415a209ba852", "saucepan-97713b1003750492", "two muffin trays-8753dd6a7d4a989f", "muffin tray1-f702138b67a5ab90", "muffin tray2-a931cbc3e64b92f2", "mini oven grips-4488937f765e4a3d", "mini oven grips2-3ebff51168f04583", "empty jar1-c5839e1c3a0f3327", "empty jar2-0d677bc73b36ea69", "rubber seal1-33726e35011431db", "rubber seal2-8b5086dc75a342d3", "puff pastry wrapper-93959b1b49184044", "roll of puff pastry-3aa8abeba9d86801", "serrated knife-9fc261df8cd4959d", "stick of butter-575f68e053486c2e", "oven gloves-afd38f8fc5a6b804", "other oven glove-2b121ead59a1341a", "puff pasty paper-326e5914b9c8855b", "rolling pen-20062fbe63c134e7", "food bin-e7d9a3695c22e44c", "used kitchen roll-df15b5c69e34917f", "glass countertop protector-889abc9fc4c9056b", "teaspoon-0c78266a10163e86", "saucer-4f70495cdd8f4dc7", "spatula-7b211b8d2dd8426e", "tupperware box-5bde95cee993d79a", "saucer2-17e51efda1827495", "cheese straw1-281812373589e956", "cheese straw2-a1a3f62fe9473cc1", "cheese straw3-6d64b51eab4c905b", "cheese straw4-e5edae14aba23e21", "two straw5-6ae035dfcac8db55", "two straw6-7025f6e2e8c44617", "cheese straw7-4578a15c6a38fcf7", "cheese straw8-ba51e65b3198287b", "cheese straw9-0e1abd3140692e8a", "cheese straw10-3bd20a10c82f14ce", "cheese straw11-ffd4a11ed619a621", "cheese straw12-a6201cf3c334da71", "cheese straw16-8d3c947a10fcdef8", "cheese straw13-8a8c1e5f59b768ec", "cheese straw14-b40f620ef1893965", "cheese straw15-cc1cb3db1692ea3f", "cheese straw17-d414993c7e9e1fe3", "cheese straw18-e2fe7d7aca3fef0b", "cheese straw19-1f4bdbde955cdb82", "cheese straw20-0c5f8ecfe662743a", "cheese straw24-b499ade6d727baba", "cheese straw21-42d31a873a3406ee", "cheese straw22-84d1ef4db454d318", "cheese straw23-6b941303a714ba84", "cheese straw26-002b23064ea14368", "cheese straw25-a5cf6d7d6f61c2ae", "cheese straw27-efbbfb6acaa5686c", "cheese straw28-7f86e27519cb022a", "cheese straw29-43ec21c830d2ac25", "cheese straw30-923dc84e193b3898", "tablespoon-c91862f8a37ffd69", "end of baguette-7d49e2a4f8418ea4", "plate-279f0aa582595079", "baguette-0818d64c6caf8f71", "bread knife-ce621c01fe05c9a9", "baguette piece-5a5f824ed819f3b7", "baguette piece2-64b1052109b9eb2e", "tub of butter-e581e1f315f53eee", "cheese-5c86336516dd4ed8", "butter knife-418756353a51e627", "tub of butter lid-ec0991326bc19677", "plate2-cdacb233d95d0bb2", "slice of cheese-5a94e59d4259c7b9", "slice of cheese2-32df7a30d1fef661", "slice of cheese3-28d109562aae57d7", "slice of cheese4-ff8db54687f85878" ], "reasoning": "1. First, I had to identify the pan of cheesesticks the user took out of the oven the least amount of times. From 13:50-14:04, the user shuffles around the pans (baking tray-7246377423a4b5b5, baking tray2-fe0973564cbcfb93) with cheesesticks in the oven , removing the grey pan (baking tray-7246377423a4b5b5) a total of 2 times and the black pan (baking tray2-fe0973564cbcfb93) a total of 3 times. \n2. From 20:48-20:51, I saw the user take both pans baking tray-7246377423a4b5b5, baking tray2-fe0973564cbcfb93) out of the oven a single time, meaning that the grey pan (baking tray-7246377423a4b5b5) is the pan (baking tray-7246377423a4b5b5) the user removes from the oven the least amount of times. \n3. Finally, I looked for the user to place the grey pan (baking tray-7246377423a4b5b5) on top of the image of a beverage and found that the user puts the pan (baking tray-7246377423a4b5b5) on a glass countertop protector with the image of a glass of red wine on its corner at 21:23.", "referenced_object_masks": [ "baking tray-7246377423a4b5b5", "baking tray2-fe0973564cbcfb93" ] }, { "key": "P02-20240211-120927:dfac491b83b917bcef669b66dec122dd6179b267", "video_id": "P02-20240211-120927", "question": "What is the ratio between the dough sticks I placed on the first baking tray horizontally versus the ones I placed vertically?", "answer_choice_0": "4:13", "answer_choice_1": "4:9", "answer_choice_2": "9:4", "answer_choice_3": "4:11", "answer_choice_4": "9:13", "answer_id": 3, "answer": "4:11", "question_type": [ "Event Occurence", "Numerical Reasoning", "Object Recognition" ], "objects_of_interest": [ "phone-243b27482388aed0", "large mixing bowl-bb8fb85c4593aea5", "baking tray-7246377423a4b5b5", "bag of flour-7f44b1b7773eb449", "tupperware lid-2a6f8cce5550c53a", "tupperware-2637df1f0730c42c", "grater-679c874181021bca", "scissors-26a9a0777eaaf0ca", "sponge-cedd35310f54c786", "baking tray2-fe0973564cbcfb93", "grated cheese packet-46c0b93d50086fd8", "offcut-ce6e0534003f1c44", "large plastic spoon-86f4f0ac59f7d81c", "tea towel-5437d0591d066039", "recipe book-4ba3415a209ba852", "saucepan-97713b1003750492", "two muffin trays-8753dd6a7d4a989f", "muffin tray1-f702138b67a5ab90", "muffin tray2-a931cbc3e64b92f2", "mini oven grips-4488937f765e4a3d", "mini oven grips2-3ebff51168f04583", "empty jar1-c5839e1c3a0f3327", "empty jar2-0d677bc73b36ea69", "rubber seal1-33726e35011431db", "rubber seal2-8b5086dc75a342d3", "puff pastry wrapper-93959b1b49184044", "roll of puff pastry-3aa8abeba9d86801", "serrated knife-9fc261df8cd4959d", "stick of butter-575f68e053486c2e", "oven gloves-afd38f8fc5a6b804", "other oven glove-2b121ead59a1341a", "puff pasty paper-326e5914b9c8855b", "rolling pen-20062fbe63c134e7", "food bin-e7d9a3695c22e44c", "used kitchen roll-df15b5c69e34917f", "glass countertop protector-889abc9fc4c9056b", "teaspoon-0c78266a10163e86", "saucer-4f70495cdd8f4dc7", "spatula-7b211b8d2dd8426e", "tupperware box-5bde95cee993d79a", "saucer2-17e51efda1827495", "cheese straw1-281812373589e956", "cheese straw2-a1a3f62fe9473cc1", "cheese straw3-6d64b51eab4c905b", "cheese straw4-e5edae14aba23e21", "two straw5-6ae035dfcac8db55", "two straw6-7025f6e2e8c44617", "cheese straw7-4578a15c6a38fcf7", "cheese straw8-ba51e65b3198287b", "cheese straw9-0e1abd3140692e8a", "cheese straw10-3bd20a10c82f14ce", "cheese straw11-ffd4a11ed619a621", "cheese straw12-a6201cf3c334da71", "cheese straw16-8d3c947a10fcdef8", "cheese straw13-8a8c1e5f59b768ec", "cheese straw14-b40f620ef1893965", "cheese straw15-cc1cb3db1692ea3f", "cheese straw17-d414993c7e9e1fe3", "cheese straw18-e2fe7d7aca3fef0b", "cheese straw19-1f4bdbde955cdb82", "cheese straw20-0c5f8ecfe662743a", "cheese straw24-b499ade6d727baba", "cheese straw21-42d31a873a3406ee", "cheese straw22-84d1ef4db454d318", "cheese straw23-6b941303a714ba84", "cheese straw26-002b23064ea14368", "cheese straw25-a5cf6d7d6f61c2ae", "cheese straw27-efbbfb6acaa5686c", "cheese straw28-7f86e27519cb022a", "cheese straw29-43ec21c830d2ac25", "cheese straw30-923dc84e193b3898", "tablespoon-c91862f8a37ffd69", "end of baguette-7d49e2a4f8418ea4", "plate-279f0aa582595079", "baguette-0818d64c6caf8f71", "bread knife-ce621c01fe05c9a9", "baguette piece-5a5f824ed819f3b7", "baguette piece2-64b1052109b9eb2e", "tub of butter-e581e1f315f53eee", "cheese-5c86336516dd4ed8", "butter knife-418756353a51e627", "tub of butter lid-ec0991326bc19677", "plate2-cdacb233d95d0bb2", "slice of cheese-5a94e59d4259c7b9", "slice of cheese2-32df7a30d1fef661", "slice of cheese3-28d109562aae57d7", "slice of cheese4-ff8db54687f85878" ], "reasoning": "1. I watched the video to locate the moment the user started placing the dough sticks on the first baking tray (baking tray-7246377423a4b5b5), which happens between 00:35 and 01:44. \n2. I counted the amount of dough sticks the user placed horizontally on the tray (baking tray-7246377423a4b5b5) versus the ones the subject placed vertically. I counted 4 placed horizontally versus 9 placed vertically, for a total of 13 dough sticks.\n3. I continued watching to see if the user added any more dough sticks to the first tray (baking tray-7246377423a4b5b5). From 03:13 to 03:19, the user adds two more dough sticks to the first tray vertically. These are the last dough sticks the user adds to the first tray (baking tray-7246377423a4b5b5).\n4. The ratio of horizontal dough sticks to the vertical dough sticks is 4:11.", "referenced_object_masks": [ "baking tray-7246377423a4b5b5" ] }, { "key": "P02-20240211-120927:e5a2ffbc068517748f058e8e348188925771aaaa", "video_id": "P02-20240211-120927", "question": "From where do I retrieve the second unique object I move counter-clockwise with my right hand?", "answer_choice_0": "A drawer", "answer_choice_1": "The sink", "answer_choice_2": "The refrigerator", "answer_choice_3": "The oven", "answer_choice_4": "A cabinet", "answer_id": 2, "answer": "The refrigerator", "question_type": [ "Egocentric & Spatial Perception", "Action Recognition", "Object Recognition" ], "objects_of_interest": [ "phone-243b27482388aed0", "large mixing bowl-bb8fb85c4593aea5", "baking tray-7246377423a4b5b5", "bag of flour-7f44b1b7773eb449", "tupperware lid-2a6f8cce5550c53a", "tupperware-2637df1f0730c42c", "grater-679c874181021bca", "scissors-26a9a0777eaaf0ca", "sponge-cedd35310f54c786", "baking tray2-fe0973564cbcfb93", "grated cheese packet-46c0b93d50086fd8", "offcut-ce6e0534003f1c44", "large plastic spoon-86f4f0ac59f7d81c", "tea towel-5437d0591d066039", "recipe book-4ba3415a209ba852", "saucepan-97713b1003750492", "two muffin trays-8753dd6a7d4a989f", "muffin tray1-f702138b67a5ab90", "muffin tray2-a931cbc3e64b92f2", "mini oven grips-4488937f765e4a3d", "mini oven grips2-3ebff51168f04583", "empty jar1-c5839e1c3a0f3327", "empty jar2-0d677bc73b36ea69", "rubber seal1-33726e35011431db", "rubber seal2-8b5086dc75a342d3", "puff pastry wrapper-93959b1b49184044", "roll of puff pastry-3aa8abeba9d86801", "serrated knife-9fc261df8cd4959d", "stick of butter-575f68e053486c2e", "oven gloves-afd38f8fc5a6b804", "other oven glove-2b121ead59a1341a", "puff pasty paper-326e5914b9c8855b", "rolling pen-20062fbe63c134e7", "food bin-e7d9a3695c22e44c", "used kitchen roll-df15b5c69e34917f", "glass countertop protector-889abc9fc4c9056b", "teaspoon-0c78266a10163e86", "saucer-4f70495cdd8f4dc7", "spatula-7b211b8d2dd8426e", "tupperware box-5bde95cee993d79a", "saucer2-17e51efda1827495", "cheese straw1-281812373589e956", "cheese straw2-a1a3f62fe9473cc1", "cheese straw3-6d64b51eab4c905b", "cheese straw4-e5edae14aba23e21", "two straw5-6ae035dfcac8db55", "two straw6-7025f6e2e8c44617", "cheese straw7-4578a15c6a38fcf7", "cheese straw8-ba51e65b3198287b", "cheese straw9-0e1abd3140692e8a", "cheese straw10-3bd20a10c82f14ce", "cheese straw11-ffd4a11ed619a621", "cheese straw12-a6201cf3c334da71", "cheese straw16-8d3c947a10fcdef8", "cheese straw13-8a8c1e5f59b768ec", "cheese straw14-b40f620ef1893965", "cheese straw15-cc1cb3db1692ea3f", "cheese straw17-d414993c7e9e1fe3", "cheese straw18-e2fe7d7aca3fef0b", "cheese straw19-1f4bdbde955cdb82", "cheese straw20-0c5f8ecfe662743a", "cheese straw24-b499ade6d727baba", "cheese straw21-42d31a873a3406ee", "cheese straw22-84d1ef4db454d318", "cheese straw23-6b941303a714ba84", "cheese straw26-002b23064ea14368", "cheese straw25-a5cf6d7d6f61c2ae", "cheese straw27-efbbfb6acaa5686c", "cheese straw28-7f86e27519cb022a", "cheese straw29-43ec21c830d2ac25", "cheese straw30-923dc84e193b3898", "tablespoon-c91862f8a37ffd69", "end of baguette-7d49e2a4f8418ea4", "plate-279f0aa582595079", "baguette-0818d64c6caf8f71", "bread knife-ce621c01fe05c9a9", "baguette piece-5a5f824ed819f3b7", "baguette piece2-64b1052109b9eb2e", "tub of butter-e581e1f315f53eee", "cheese-5c86336516dd4ed8", "butter knife-418756353a51e627", "tub of butter lid-ec0991326bc19677", "plate2-cdacb233d95d0bb2", "slice of cheese-5a94e59d4259c7b9", "slice of cheese2-32df7a30d1fef661", "slice of cheese3-28d109562aae57d7", "slice of cheese4-ff8db54687f85878" ], "reasoning": "1. I watched the video and looked for the user to move an object in a counter-clockwise circular direction. They do this for the first time when they stir the jam , performing the counter-clockwise motion 3 times, always with the same spoon (large plastic spoon-86f4f0ac59f7d81c) in their right hand, at 03:53, 04:20, and 07:45.\n2. Next, I looked for the user to move a second object in a counter-clockwise circular motion with their right hand, and saw that they do this multiple times from 10:07-10:36 with a stick of butter (stick of butter-575f68e053486c2e). \n3. I then went back and looked for the user to retrieve the butter (stick of butter-575f68e053486c2e) and saw them grab it from the refrigerator at 09:57.", "referenced_object_masks": [ "stick of butter-575f68e053486c2e", "large plastic spoon-86f4f0ac59f7d81c" ] }, { "key": "P02-20240211-141313:08f50874cbc9fcfb80103ddc95f32bec7f1beae0", "video_id": "P02-20240211-141313", "question": "What is the inverted ratio of the number of cheese straw groups separated by paper towel in the first tupperware container to the number of tarts in the rightmost muffin tray as the tupperware is being packed?", "answer_choice_0": "1:2", "answer_choice_1": "3:5", "answer_choice_2": "2:1", "answer_choice_3": "3:1", "answer_choice_4": "2:3", "answer_id": 2, "answer": "2:1", "question_type": [ "Counting", "Numerical Reasoning", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "serrated knife-60cf5135ef7afc2e", "plate1-894f66f3439f9895", "empty packaging1-abf74550309926c4", "box-da932b373d45576c", "empty packaging2-b150954d5fc6570c", "hand blender-721dea9a6a7307ac", "citrus juicer-c857fa2762ca3680", "citrus juicer guard-756401bd3085acf1", "tea towel-b869d30560eadbb9", "blue cloth-c7d3d4a38b341f7d", "green cloth-c4c589a84251c391", "tea towel2-f5f8e7cc20582cd1", "hand blender attachment-1cf0e440d388e023", "cutting board-b297db957087d00f", "tablespoon-bd29fcdcdc219888", "teaspoon-9e58ff82a2b8bcd7", "black washing up bowl-a94762e89cf55c0d", "washing up liquid bottle-54a7f7ef5869669a", "rolling pen-777d57e5b8a8d1c6", "grater-8ccd82718960f41e", "hand blender cup-215dff0892a23060", "hand blender cup lid-252316a7585aacf5", "stack of colanders-845a245e3239cd67", "saucer-7625e366914cb2f9", "bowl-63193a7b18c63c67", "scissors-b2ea14634cf86dd3", "fork-7a206faccb94b2b2", "kitchen towel holder-7660f7bc5f068eae", "piece of kitchen towel-f740d4c6740682e3", "tupperware lid-70fc85232cec14b5", "cheese straws-d7560f1fc41b778e", "cheese straws2-b046b87fa1f0e0b2", "cheese straws3-4388eb170a6c07a3", "cheese straws4-9b4fa02070fb3ee7", "cheese straws5-db13821576a3234a", "cheese straws6-141c3f94b437a579", "cheese straws7-58b603cb02a3d93b", "cheese straws8-bf1a9e3b1c95daab", "cheese straws9-29030461bbe5bb0a", "cheese straws10-452ee213912ed91f", "piece of kitchen towel2-32b7b781f220f18c", "tupperware-7cca882f645c61bb", "tupperware2-ad03df2f3b800b2e", "tupperware lid2-1fdb135dcebc1070", "cheese straws11-a1f9c506fdc50427", "some cheese straws-ef35e37460f6def2", "cheese straws13-860d866a01d15c48", "cheese straws14-7a3ff9be505828c8", "cheese straws15-b86ec0ee2c9a648b", "cheese straws16-77a6f4dd7702688b", "cheese straws17-69013db73b97bb03", "tupperware3-bf1775b117f4f898", "tupperware4-d687ebec477cd306", "tupperware5-9e426892fdb82695", "tupperware lid5-c30d504a77a98f18", "tart1-95417e179d2dbc2d", "tart2-18472503d11536bd", "tart3-d4862dd226c3493d", "tupperware lid3-6ed5d28d11abae6a", "tart4-6ee86ad5ee7ae462", "tart6-6104f7e51d3757c8", "tart7-119a7ea9e5ca6fe2", "tart8-5de6071a86967ab3", "tart9-019767a4de0a77a1", "saucer2-cef73356046354a8", "tupperware6-e25ae9bc858ae13c", "tupperware7-486a9185105bb78b", "tupperware lid7-bfe937803be3f967", "tupperware lid6-c86cfdeadfb62ab8", "tart10-773fa939c4e3682e", "tart11-7500b08699129e6d", "tart12-60995b13a0b0f2e3", "tart13-788595d1f21e2aff", "tart14-00b6ccd0f4477170", "tart15-39caf2792a9c7713", "tart16-84c0e1584b193347", "tart17-38f8bf0cbc2d000d", "yoghurt-92e9bc37860f8014", "lid of yoghurt-9d3e859b5bfd22a9", "cling film box-2803b28bce61587d", "tablespoon2-793978b9e0b2239e", "sponge-9e7e751606140eb5", "large mixing bowl-de2cacdb1e189ec0", "teaspoon2-8f9e973b79662102", "piece of cling film-e3a5c0d96aeba6ec", "object in fridge-af0e03b56dde3a0d", "measuring jug-ff048110b78b0144", "jam jar-ceb21eea5f819900", "piece of kitchen towel3-7ad2cc6f17a94467", "jam jar2-3596c81be19c6f73", "butter knife-246bec285e32a315", "baking trays-b6854d9c54c7edac", "teaspoon3-ead6fbe11f2c5b70", "spatula-4dca59671cad6995", "pan gripper-5b12f1742b4cc731", "pan gripper2-36ccee4dbb3ec3cc", "large plastic spoon-b2fb0b96bd089aee", "baking tray2-d6cdb26743456365", "plate2-b86b86614b47c004", "large saucepan-69a2ca9fa5530d85", "saucer3-6ae864bef179cfec", "bread knife-c7af07bcc3cdfceb", "food bin-f7ba3ea4e82192e9", "lid of saucepan-5f7969a5044beefa", "muffin tray1-1617228fcb5787e2", "muffin tray2-6067517d419c17cd", "washing up brush-e8c2e8d218d9a011", "drain protector-a6ff8b48edfce6ac", "recipe book-da21f089852f9162", "cardboard insert-bc8f137b942e0b29", "scales-f4654fedabef9df9", "scale instructions-1252a27eff3951a8", "scales box-5bfb8cb9558e9fa9", "packet of tomatoes-472e26daa314566e", "vegetable drawer-3a5ba00d07d7568b" ], "reasoning": "1. First, I looked to find the first tupperware container (tupperware-7cca882f645c61bb) being packed with cheese straws (cheese straws11-a1f9c506fdc50427) which begins at 03:22.\n2. The user makes levels with paper towels (piece of kitchen towel-f740d4c6740682e3) at 03:22 and 03:27, for a total of two levels.\n3. I then counted the tarts (tart7-119a7ea9e5ca6fe2, tart9-019767a4de0a77a1, tart10-773fa939c4e3682e, tart12-60995b13a0b0f2e3) in the tart pan on the right, for a total of four tarts.\n4. The ratio of levels of paper towels (piece of kitchen towel-f740d4c6740682e3) containing cheese straws (cheese straws11-a1f9c506fdc50427) to tarts (tart7-119a7ea9e5ca6fe2, tart9-019767a4de0a77a1, tart10-773fa939c4e3682e, tart12-60995b13a0b0f2e3) is 2:4, or 1:2.\n5. The inverse ratio is 2:1.", "referenced_object_masks": [ "piece of kitchen towel-f740d4c6740682e3", "cheese straws11-a1f9c506fdc50427", "tupperware-7cca882f645c61bb", "tart7-119a7ea9e5ca6fe2", "tart9-019767a4de0a77a1", "tart10-773fa939c4e3682e", "tart12-60995b13a0b0f2e3" ] }, { "key": "P02-20240211-141313:14dde6a7128e17e6a2fcd25416fdc7477a531a82", "video_id": "P02-20240211-141313", "question": "From first to last, in what order did I place the following items - clear plastic cup, citrus juicer, cutting board, hand blender attachment - into the cabinet next to the dishwasher?", "answer_choice_0": "Hand blender attachment, citrus juicer, clear plastic cup, cutting board", "answer_choice_1": "Citrus juicer, clear plastic cup, hand blender attachment, cutting board", "answer_choice_2": "Citrus juicer, hand blender attachment, cutting board, clear plastic cup", "answer_choice_3": "Clear plastic cup, citrus juicer, cutting board, hand blender attachement", "answer_choice_4": "Hand blender attachment, cutting board, citrus juicer, clear plastic cup", "answer_id": 2, "answer": "Citrus juicer, hand blender attachment, cutting board, clear plastic cup", "question_type": [ "Object Recognition", "Temporal Reasoning", "Event Occurence" ], "objects_of_interest": [ "serrated knife-60cf5135ef7afc2e", "plate1-894f66f3439f9895", "empty packaging1-abf74550309926c4", "box-da932b373d45576c", "empty packaging2-b150954d5fc6570c", "hand blender-721dea9a6a7307ac", "citrus juicer-c857fa2762ca3680", "citrus juicer guard-756401bd3085acf1", "tea towel-b869d30560eadbb9", "blue cloth-c7d3d4a38b341f7d", "green cloth-c4c589a84251c391", "tea towel2-f5f8e7cc20582cd1", "hand blender attachment-1cf0e440d388e023", "cutting board-b297db957087d00f", "tablespoon-bd29fcdcdc219888", "teaspoon-9e58ff82a2b8bcd7", "black washing up bowl-a94762e89cf55c0d", "washing up liquid bottle-54a7f7ef5869669a", "rolling pen-777d57e5b8a8d1c6", "grater-8ccd82718960f41e", "hand blender cup-215dff0892a23060", "hand blender cup lid-252316a7585aacf5", "stack of colanders-845a245e3239cd67", "saucer-7625e366914cb2f9", "bowl-63193a7b18c63c67", "scissors-b2ea14634cf86dd3", "fork-7a206faccb94b2b2", "kitchen towel holder-7660f7bc5f068eae", "piece of kitchen towel-f740d4c6740682e3", "tupperware lid-70fc85232cec14b5", "cheese straws-d7560f1fc41b778e", "cheese straws2-b046b87fa1f0e0b2", "cheese straws3-4388eb170a6c07a3", "cheese straws4-9b4fa02070fb3ee7", "cheese straws5-db13821576a3234a", "cheese straws6-141c3f94b437a579", "cheese straws7-58b603cb02a3d93b", "cheese straws8-bf1a9e3b1c95daab", "cheese straws9-29030461bbe5bb0a", "cheese straws10-452ee213912ed91f", "piece of kitchen towel2-32b7b781f220f18c", "tupperware-7cca882f645c61bb", "tupperware2-ad03df2f3b800b2e", "tupperware lid2-1fdb135dcebc1070", "cheese straws11-a1f9c506fdc50427", "some cheese straws-ef35e37460f6def2", "cheese straws13-860d866a01d15c48", "cheese straws14-7a3ff9be505828c8", "cheese straws15-b86ec0ee2c9a648b", "cheese straws16-77a6f4dd7702688b", "cheese straws17-69013db73b97bb03", "tupperware3-bf1775b117f4f898", "tupperware4-d687ebec477cd306", "tupperware5-9e426892fdb82695", "tupperware lid5-c30d504a77a98f18", "tart1-95417e179d2dbc2d", "tart2-18472503d11536bd", "tart3-d4862dd226c3493d", "tupperware lid3-6ed5d28d11abae6a", "tart4-6ee86ad5ee7ae462", "tart6-6104f7e51d3757c8", "tart7-119a7ea9e5ca6fe2", "tart8-5de6071a86967ab3", "tart9-019767a4de0a77a1", "saucer2-cef73356046354a8", "tupperware6-e25ae9bc858ae13c", "tupperware7-486a9185105bb78b", "tupperware lid7-bfe937803be3f967", "tupperware lid6-c86cfdeadfb62ab8", "tart10-773fa939c4e3682e", "tart11-7500b08699129e6d", "tart12-60995b13a0b0f2e3", "tart13-788595d1f21e2aff", "tart14-00b6ccd0f4477170", "tart15-39caf2792a9c7713", "tart16-84c0e1584b193347", "tart17-38f8bf0cbc2d000d", "yoghurt-92e9bc37860f8014", "lid of yoghurt-9d3e859b5bfd22a9", "cling film box-2803b28bce61587d", "tablespoon2-793978b9e0b2239e", "sponge-9e7e751606140eb5", "large mixing bowl-de2cacdb1e189ec0", "teaspoon2-8f9e973b79662102", "piece of cling film-e3a5c0d96aeba6ec", "object in fridge-af0e03b56dde3a0d", "measuring jug-ff048110b78b0144", "jam jar-ceb21eea5f819900", "piece of kitchen towel3-7ad2cc6f17a94467", "jam jar2-3596c81be19c6f73", "butter knife-246bec285e32a315", "baking trays-b6854d9c54c7edac", "teaspoon3-ead6fbe11f2c5b70", "spatula-4dca59671cad6995", "pan gripper-5b12f1742b4cc731", "pan gripper2-36ccee4dbb3ec3cc", "large plastic spoon-b2fb0b96bd089aee", "baking tray2-d6cdb26743456365", "plate2-b86b86614b47c004", "large saucepan-69a2ca9fa5530d85", "saucer3-6ae864bef179cfec", "bread knife-c7af07bcc3cdfceb", "food bin-f7ba3ea4e82192e9", "lid of saucepan-5f7969a5044beefa", "muffin tray1-1617228fcb5787e2", "muffin tray2-6067517d419c17cd", "washing up brush-e8c2e8d218d9a011", "drain protector-a6ff8b48edfce6ac", "recipe book-da21f089852f9162", "cardboard insert-bc8f137b942e0b29", "scales-f4654fedabef9df9", "scale instructions-1252a27eff3951a8", "scales box-5bfb8cb9558e9fa9", "packet of tomatoes-472e26daa314566e", "vegetable drawer-3a5ba00d07d7568b" ], "reasoning": "1. I watched the video to determine when the cabinet next to the dishwasher was opened, and objects were placed into it. This occurred at 01:11. \n2. From 01:11-02:25, I watched as objects were placed into the cabinet, taking note of the order in which they were put away. First, I observed the citrus juicer (citrus juicer-c857fa2762ca3680) being placed into the cabinet at 01:14. \n3. Second, I saw that the hand blender attachment (hand blender attachment-1cf0e440d388e023) was placed in the cabinet at 01:25-01:26. \n4. Third, I saw the cutting board (cutting board-b297db957087d00f) be placed into the cabinet from 01:36-01:38. \n5. Finally, I saw the clear plastic cup be placed into the cabinet from 02:24-02:25.", "referenced_object_masks": [ "hand blender attachment-1cf0e440d388e023", "cutting board-b297db957087d00f", "citrus juicer-c857fa2762ca3680" ] }, { "key": "P02-20240211-141313:51db24925d262ad1aff4481b3b69fc2f4972e339", "video_id": "P02-20240211-141313", "question": "What is in front of the food bin when it is in its third stationary position?", "answer_choice_0": "A box of plastic wrap", "answer_choice_1": "A saucepan", "answer_choice_2": "A trash can", "answer_choice_3": "A blue and white cutting board", "answer_choice_4": "A drying rack", "answer_id": 3, "answer": "A blue and white cutting board", "question_type": [ "Situational Awareness", "State Changes", "Counting", "Object Recognition" ], "objects_of_interest": [ "serrated knife-60cf5135ef7afc2e", "plate1-894f66f3439f9895", "empty packaging1-abf74550309926c4", "box-da932b373d45576c", "empty packaging2-b150954d5fc6570c", "hand blender-721dea9a6a7307ac", "citrus juicer-c857fa2762ca3680", "citrus juicer guard-756401bd3085acf1", "tea towel-b869d30560eadbb9", "blue cloth-c7d3d4a38b341f7d", "green cloth-c4c589a84251c391", "tea towel2-f5f8e7cc20582cd1", "hand blender attachment-1cf0e440d388e023", "cutting board-b297db957087d00f", "tablespoon-bd29fcdcdc219888", "teaspoon-9e58ff82a2b8bcd7", "black washing up bowl-a94762e89cf55c0d", "washing up liquid bottle-54a7f7ef5869669a", "rolling pen-777d57e5b8a8d1c6", "grater-8ccd82718960f41e", "hand blender cup-215dff0892a23060", "hand blender cup lid-252316a7585aacf5", "stack of colanders-845a245e3239cd67", "saucer-7625e366914cb2f9", "bowl-63193a7b18c63c67", "scissors-b2ea14634cf86dd3", "fork-7a206faccb94b2b2", "kitchen towel holder-7660f7bc5f068eae", "piece of kitchen towel-f740d4c6740682e3", "tupperware lid-70fc85232cec14b5", "cheese straws-d7560f1fc41b778e", "cheese straws2-b046b87fa1f0e0b2", "cheese straws3-4388eb170a6c07a3", "cheese straws4-9b4fa02070fb3ee7", "cheese straws5-db13821576a3234a", "cheese straws6-141c3f94b437a579", "cheese straws7-58b603cb02a3d93b", "cheese straws8-bf1a9e3b1c95daab", "cheese straws9-29030461bbe5bb0a", "cheese straws10-452ee213912ed91f", "piece of kitchen towel2-32b7b781f220f18c", "tupperware-7cca882f645c61bb", "tupperware2-ad03df2f3b800b2e", "tupperware lid2-1fdb135dcebc1070", "cheese straws11-a1f9c506fdc50427", "some cheese straws-ef35e37460f6def2", "cheese straws13-860d866a01d15c48", "cheese straws14-7a3ff9be505828c8", "cheese straws15-b86ec0ee2c9a648b", "cheese straws16-77a6f4dd7702688b", "cheese straws17-69013db73b97bb03", "tupperware3-bf1775b117f4f898", "tupperware4-d687ebec477cd306", "tupperware5-9e426892fdb82695", "tupperware lid5-c30d504a77a98f18", "tart1-95417e179d2dbc2d", "tart2-18472503d11536bd", "tart3-d4862dd226c3493d", "tupperware lid3-6ed5d28d11abae6a", "tart4-6ee86ad5ee7ae462", "tart6-6104f7e51d3757c8", "tart7-119a7ea9e5ca6fe2", "tart8-5de6071a86967ab3", "tart9-019767a4de0a77a1", "saucer2-cef73356046354a8", "tupperware6-e25ae9bc858ae13c", "tupperware7-486a9185105bb78b", "tupperware lid7-bfe937803be3f967", "tupperware lid6-c86cfdeadfb62ab8", "tart10-773fa939c4e3682e", "tart11-7500b08699129e6d", "tart12-60995b13a0b0f2e3", "tart13-788595d1f21e2aff", "tart14-00b6ccd0f4477170", "tart15-39caf2792a9c7713", "tart16-84c0e1584b193347", "tart17-38f8bf0cbc2d000d", "yoghurt-92e9bc37860f8014", "lid of yoghurt-9d3e859b5bfd22a9", "cling film box-2803b28bce61587d", "tablespoon2-793978b9e0b2239e", "sponge-9e7e751606140eb5", "large mixing bowl-de2cacdb1e189ec0", "teaspoon2-8f9e973b79662102", "piece of cling film-e3a5c0d96aeba6ec", "object in fridge-af0e03b56dde3a0d", "measuring jug-ff048110b78b0144", "jam jar-ceb21eea5f819900", "piece of kitchen towel3-7ad2cc6f17a94467", "jam jar2-3596c81be19c6f73", "butter knife-246bec285e32a315", "baking trays-b6854d9c54c7edac", "teaspoon3-ead6fbe11f2c5b70", "spatula-4dca59671cad6995", "pan gripper-5b12f1742b4cc731", "pan gripper2-36ccee4dbb3ec3cc", "large plastic spoon-b2fb0b96bd089aee", "baking tray2-d6cdb26743456365", "plate2-b86b86614b47c004", "large saucepan-69a2ca9fa5530d85", "saucer3-6ae864bef179cfec", "bread knife-c7af07bcc3cdfceb", "food bin-f7ba3ea4e82192e9", "lid of saucepan-5f7969a5044beefa", "muffin tray1-1617228fcb5787e2", "muffin tray2-6067517d419c17cd", "washing up brush-e8c2e8d218d9a011", "drain protector-a6ff8b48edfce6ac", "recipe book-da21f089852f9162", "cardboard insert-bc8f137b942e0b29", "scales-f4654fedabef9df9", "scale instructions-1252a27eff3951a8", "scales box-5bfb8cb9558e9fa9", "packet of tomatoes-472e26daa314566e", "vegetable drawer-3a5ba00d07d7568b" ], "reasoning": "1. I watched the video looking for the food bin (food bin-f7ba3ea4e82192e9) which can be seen on the floor at 15:54, making the floor its first stationary position.\n2. At 15:55, the user picks up the food bin (food bin-f7ba3ea4e82192e9) and puts it down at 15:56 on a white cutting board on the counter, making this the second stationary position. \n3. The user does not pick the food bin (food bin-f7ba3ea4e82192e9) up again until 21:29.\n4. The user puts the food bin (food bin-f7ba3ea4e82192e9) down again at 22:02, this time on the counter, the third stationary position.\n5. In front of the food bin (food bin-f7ba3ea4e82192e9) in this position is a blue and white cutting board .", "referenced_object_masks": [ "food bin-f7ba3ea4e82192e9" ] }, { "key": "P02-20240211-141313:5f98a34f26705d190b082f9b9149c149c0bd60ed", "video_id": "P02-20240211-141313", "question": "What object is between the spatula and the black baking tray before I wash them?", "answer_choice_0": "A dish towel", "answer_choice_1": "A bowl", "answer_choice_2": "A paper towel", "answer_choice_3": "A gray baking tray", "answer_choice_4": "A spoon", "answer_id": 3, "answer": "A gray baking tray", "question_type": [ "Egocentric & Spatial Perception", "Temporal Reasoning" ], "objects_of_interest": [ "serrated knife-60cf5135ef7afc2e", "plate1-894f66f3439f9895", "empty packaging1-abf74550309926c4", "box-da932b373d45576c", "empty packaging2-b150954d5fc6570c", "hand blender-721dea9a6a7307ac", "citrus juicer-c857fa2762ca3680", "citrus juicer guard-756401bd3085acf1", "tea towel-b869d30560eadbb9", "blue cloth-c7d3d4a38b341f7d", "green cloth-c4c589a84251c391", "tea towel2-f5f8e7cc20582cd1", "hand blender attachment-1cf0e440d388e023", "cutting board-b297db957087d00f", "tablespoon-bd29fcdcdc219888", "teaspoon-9e58ff82a2b8bcd7", "black washing up bowl-a94762e89cf55c0d", "washing up liquid bottle-54a7f7ef5869669a", "rolling pen-777d57e5b8a8d1c6", "grater-8ccd82718960f41e", "hand blender cup-215dff0892a23060", "hand blender cup lid-252316a7585aacf5", "stack of colanders-845a245e3239cd67", "saucer-7625e366914cb2f9", "bowl-63193a7b18c63c67", "scissors-b2ea14634cf86dd3", "fork-7a206faccb94b2b2", "kitchen towel holder-7660f7bc5f068eae", "piece of kitchen towel-f740d4c6740682e3", "tupperware lid-70fc85232cec14b5", "cheese straws-d7560f1fc41b778e", "cheese straws2-b046b87fa1f0e0b2", "cheese straws3-4388eb170a6c07a3", "cheese straws4-9b4fa02070fb3ee7", "cheese straws5-db13821576a3234a", "cheese straws6-141c3f94b437a579", "cheese straws7-58b603cb02a3d93b", "cheese straws8-bf1a9e3b1c95daab", "cheese straws9-29030461bbe5bb0a", "cheese straws10-452ee213912ed91f", "piece of kitchen towel2-32b7b781f220f18c", "tupperware-7cca882f645c61bb", "tupperware2-ad03df2f3b800b2e", "tupperware lid2-1fdb135dcebc1070", "cheese straws11-a1f9c506fdc50427", "some cheese straws-ef35e37460f6def2", "cheese straws13-860d866a01d15c48", "cheese straws14-7a3ff9be505828c8", "cheese straws15-b86ec0ee2c9a648b", "cheese straws16-77a6f4dd7702688b", "cheese straws17-69013db73b97bb03", "tupperware3-bf1775b117f4f898", "tupperware4-d687ebec477cd306", "tupperware5-9e426892fdb82695", "tupperware lid5-c30d504a77a98f18", "tart1-95417e179d2dbc2d", "tart2-18472503d11536bd", "tart3-d4862dd226c3493d", "tupperware lid3-6ed5d28d11abae6a", "tart4-6ee86ad5ee7ae462", "tart6-6104f7e51d3757c8", "tart7-119a7ea9e5ca6fe2", "tart8-5de6071a86967ab3", "tart9-019767a4de0a77a1", "saucer2-cef73356046354a8", "tupperware6-e25ae9bc858ae13c", "tupperware7-486a9185105bb78b", "tupperware lid7-bfe937803be3f967", "tupperware lid6-c86cfdeadfb62ab8", "tart10-773fa939c4e3682e", "tart11-7500b08699129e6d", "tart12-60995b13a0b0f2e3", "tart13-788595d1f21e2aff", "tart14-00b6ccd0f4477170", "tart15-39caf2792a9c7713", "tart16-84c0e1584b193347", "tart17-38f8bf0cbc2d000d", "yoghurt-92e9bc37860f8014", "lid of yoghurt-9d3e859b5bfd22a9", "cling film box-2803b28bce61587d", "tablespoon2-793978b9e0b2239e", "sponge-9e7e751606140eb5", "large mixing bowl-de2cacdb1e189ec0", "teaspoon2-8f9e973b79662102", "piece of cling film-e3a5c0d96aeba6ec", "object in fridge-af0e03b56dde3a0d", "measuring jug-ff048110b78b0144", "jam jar-ceb21eea5f819900", "piece of kitchen towel3-7ad2cc6f17a94467", "jam jar2-3596c81be19c6f73", "butter knife-246bec285e32a315", "baking trays-b6854d9c54c7edac", "teaspoon3-ead6fbe11f2c5b70", "spatula-4dca59671cad6995", "pan gripper-5b12f1742b4cc731", "pan gripper2-36ccee4dbb3ec3cc", "large plastic spoon-b2fb0b96bd089aee", "baking tray2-d6cdb26743456365", "plate2-b86b86614b47c004", "large saucepan-69a2ca9fa5530d85", "saucer3-6ae864bef179cfec", "bread knife-c7af07bcc3cdfceb", "food bin-f7ba3ea4e82192e9", "lid of saucepan-5f7969a5044beefa", "muffin tray1-1617228fcb5787e2", "muffin tray2-6067517d419c17cd", "washing up brush-e8c2e8d218d9a011", "drain protector-a6ff8b48edfce6ac", "recipe book-da21f089852f9162", "cardboard insert-bc8f137b942e0b29", "scales-f4654fedabef9df9", "scale instructions-1252a27eff3951a8", "scales box-5bfb8cb9558e9fa9", "packet of tomatoes-472e26daa314566e", "vegetable drawer-3a5ba00d07d7568b" ], "reasoning": "1. I looked for the user to wash the spatula (spatula-4dca59671cad6995), which occurs at 13:33.\n2. The spatula (spatula-4dca59671cad6995) had been lying on a gray baking tray (baking trays-b6854d9c54c7edac).\n3. I waited for the user to wash the gray baking tray (baking trays-b6854d9c54c7edac), which occurs at 14:20.\n4. When the user removes the gray baking tray (baking trays-b6854d9c54c7edac), the black baking tray (baking tray2-d6cdb26743456365) is revealed beneath.\n5. This makes the gray baking tray (baking trays-b6854d9c54c7edac) the object in between the spatula (spatula-4dca59671cad6995) and the black baking tray (baking tray2-d6cdb26743456365).", "referenced_object_masks": [ "baking trays-b6854d9c54c7edac", "spatula-4dca59671cad6995", "baking tray2-d6cdb26743456365" ] }, { "key": "P02-20240211-141313:77ae040f9d94b5981f8644ecb2384af2e9d1bce7", "video_id": "P02-20240211-141313", "question": "If the third tart and the eighth tart removed from the muffin tins were to be switched in the order I removed them, from which cup of which muffin tin would the new third tart have been removed?", "answer_choice_0": "The left cup of the third row of the right tin", "answer_choice_1": "The middle cup of the third row of the right tin", "answer_choice_2": "The left cup of the second row of the left tin", "answer_choice_3": "The middle cup of the second row of the left tin", "answer_choice_4": "The left cup of the top row of the left tin", "answer_id": 0, "answer": "The left cup of the third row of the right tin", "question_type": [ "Counterfactual", "Spatial Perception", "Counting" ], "objects_of_interest": [ "serrated knife-60cf5135ef7afc2e", "plate1-894f66f3439f9895", "empty packaging1-abf74550309926c4", "box-da932b373d45576c", "empty packaging2-b150954d5fc6570c", "hand blender-721dea9a6a7307ac", "citrus juicer-c857fa2762ca3680", "citrus juicer guard-756401bd3085acf1", "tea towel-b869d30560eadbb9", "blue cloth-c7d3d4a38b341f7d", "green cloth-c4c589a84251c391", "tea towel2-f5f8e7cc20582cd1", "hand blender attachment-1cf0e440d388e023", "cutting board-b297db957087d00f", "tablespoon-bd29fcdcdc219888", "teaspoon-9e58ff82a2b8bcd7", "black washing up bowl-a94762e89cf55c0d", "washing up liquid bottle-54a7f7ef5869669a", "rolling pen-777d57e5b8a8d1c6", "grater-8ccd82718960f41e", "hand blender cup-215dff0892a23060", "hand blender cup lid-252316a7585aacf5", "stack of colanders-845a245e3239cd67", "saucer-7625e366914cb2f9", "bowl-63193a7b18c63c67", "scissors-b2ea14634cf86dd3", "fork-7a206faccb94b2b2", "kitchen towel holder-7660f7bc5f068eae", "piece of kitchen towel-f740d4c6740682e3", "tupperware lid-70fc85232cec14b5", "cheese straws-d7560f1fc41b778e", "cheese straws2-b046b87fa1f0e0b2", "cheese straws3-4388eb170a6c07a3", "cheese straws4-9b4fa02070fb3ee7", "cheese straws5-db13821576a3234a", "cheese straws6-141c3f94b437a579", "cheese straws7-58b603cb02a3d93b", "cheese straws8-bf1a9e3b1c95daab", "cheese straws9-29030461bbe5bb0a", "cheese straws10-452ee213912ed91f", "piece of kitchen towel2-32b7b781f220f18c", "tupperware-7cca882f645c61bb", "tupperware2-ad03df2f3b800b2e", "tupperware lid2-1fdb135dcebc1070", "cheese straws11-a1f9c506fdc50427", "some cheese straws-ef35e37460f6def2", "cheese straws13-860d866a01d15c48", "cheese straws14-7a3ff9be505828c8", "cheese straws15-b86ec0ee2c9a648b", "cheese straws16-77a6f4dd7702688b", "cheese straws17-69013db73b97bb03", "tupperware3-bf1775b117f4f898", "tupperware4-d687ebec477cd306", "tupperware5-9e426892fdb82695", "tupperware lid5-c30d504a77a98f18", "tart1-95417e179d2dbc2d", "tart2-18472503d11536bd", "tart3-d4862dd226c3493d", "tupperware lid3-6ed5d28d11abae6a", "tart4-6ee86ad5ee7ae462", "tart6-6104f7e51d3757c8", "tart7-119a7ea9e5ca6fe2", "tart8-5de6071a86967ab3", "tart9-019767a4de0a77a1", "saucer2-cef73356046354a8", "tupperware6-e25ae9bc858ae13c", "tupperware7-486a9185105bb78b", "tupperware lid7-bfe937803be3f967", "tupperware lid6-c86cfdeadfb62ab8", "tart10-773fa939c4e3682e", "tart11-7500b08699129e6d", "tart12-60995b13a0b0f2e3", "tart13-788595d1f21e2aff", "tart14-00b6ccd0f4477170", "tart15-39caf2792a9c7713", "tart16-84c0e1584b193347", "tart17-38f8bf0cbc2d000d", "yoghurt-92e9bc37860f8014", "lid of yoghurt-9d3e859b5bfd22a9", "cling film box-2803b28bce61587d", "tablespoon2-793978b9e0b2239e", "sponge-9e7e751606140eb5", "large mixing bowl-de2cacdb1e189ec0", "teaspoon2-8f9e973b79662102", "piece of cling film-e3a5c0d96aeba6ec", "object in fridge-af0e03b56dde3a0d", "measuring jug-ff048110b78b0144", "jam jar-ceb21eea5f819900", "piece of kitchen towel3-7ad2cc6f17a94467", "jam jar2-3596c81be19c6f73", "butter knife-246bec285e32a315", "baking trays-b6854d9c54c7edac", "teaspoon3-ead6fbe11f2c5b70", "spatula-4dca59671cad6995", "pan gripper-5b12f1742b4cc731", "pan gripper2-36ccee4dbb3ec3cc", "large plastic spoon-b2fb0b96bd089aee", "baking tray2-d6cdb26743456365", "plate2-b86b86614b47c004", "large saucepan-69a2ca9fa5530d85", "saucer3-6ae864bef179cfec", "bread knife-c7af07bcc3cdfceb", "food bin-f7ba3ea4e82192e9", "lid of saucepan-5f7969a5044beefa", "muffin tray1-1617228fcb5787e2", "muffin tray2-6067517d419c17cd", "washing up brush-e8c2e8d218d9a011", "drain protector-a6ff8b48edfce6ac", "recipe book-da21f089852f9162", "cardboard insert-bc8f137b942e0b29", "scales-f4654fedabef9df9", "scale instructions-1252a27eff3951a8", "scales box-5bfb8cb9558e9fa9", "packet of tomatoes-472e26daa314566e", "vegetable drawer-3a5ba00d07d7568b" ], "reasoning": "1. I watched the video while the tarts were being removed from the muffin tins , counting to identify the third (tart3-d4862dd226c3493d) and the eigthth (tart9-019767a4de0a77a1) tart removed. \n2. I identified that the third tart (tart3-d4862dd226c3493d) was removed from the top left cup of the left muffin tin at 05:05. \n3. I then continued to watch, counting the number of tarts as they were removed, until identifying the eighth tart removed (tart9-019767a4de0a77a1) from the left cup of the third row of the right muffin tin . \n4. I then imagined that the order in which I removed these tarts (tart3-d4862dd226c3493d, tart9-019767a4de0a77a1) was swapped, making tart #8 now tart #3. Therefore, in this scenario, tart 3 (tart3-d4862dd226c3493d) would be removed from the left cup of the third row of the right muffin tin .", "referenced_object_masks": [ "tart3-d4862dd226c3493d", "tart9-019767a4de0a77a1" ] }, { "key": "P02-20240211-141313:8d69134a31ffaafd53608ff9b6cbecbcc7325f47", "video_id": "P02-20240211-141313", "question": "What in the house am I walking towards when I lick the yogurt spoon?", "answer_choice_0": "The sink", "answer_choice_1": "The glass door", "answer_choice_2": "The stove", "answer_choice_3": "The radiator", "answer_choice_4": "The table", "answer_id": 0, "answer": "The sink", "question_type": [ "Object Recognition", "Action Recognition", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "serrated knife-60cf5135ef7afc2e", "plate1-894f66f3439f9895", "empty packaging1-abf74550309926c4", "box-da932b373d45576c", "empty packaging2-b150954d5fc6570c", "hand blender-721dea9a6a7307ac", "citrus juicer-c857fa2762ca3680", "citrus juicer guard-756401bd3085acf1", "tea towel-b869d30560eadbb9", "blue cloth-c7d3d4a38b341f7d", "green cloth-c4c589a84251c391", "tea towel2-f5f8e7cc20582cd1", "hand blender attachment-1cf0e440d388e023", "cutting board-b297db957087d00f", "tablespoon-bd29fcdcdc219888", "teaspoon-9e58ff82a2b8bcd7", "black washing up bowl-a94762e89cf55c0d", "washing up liquid bottle-54a7f7ef5869669a", "rolling pen-777d57e5b8a8d1c6", "grater-8ccd82718960f41e", "hand blender cup-215dff0892a23060", "hand blender cup lid-252316a7585aacf5", "stack of colanders-845a245e3239cd67", "saucer-7625e366914cb2f9", "bowl-63193a7b18c63c67", "scissors-b2ea14634cf86dd3", "fork-7a206faccb94b2b2", "kitchen towel holder-7660f7bc5f068eae", "piece of kitchen towel-f740d4c6740682e3", "tupperware lid-70fc85232cec14b5", "cheese straws-d7560f1fc41b778e", "cheese straws2-b046b87fa1f0e0b2", "cheese straws3-4388eb170a6c07a3", "cheese straws4-9b4fa02070fb3ee7", "cheese straws5-db13821576a3234a", "cheese straws6-141c3f94b437a579", "cheese straws7-58b603cb02a3d93b", "cheese straws8-bf1a9e3b1c95daab", "cheese straws9-29030461bbe5bb0a", "cheese straws10-452ee213912ed91f", "piece of kitchen towel2-32b7b781f220f18c", "tupperware-7cca882f645c61bb", "tupperware2-ad03df2f3b800b2e", "tupperware lid2-1fdb135dcebc1070", "cheese straws11-a1f9c506fdc50427", "some cheese straws-ef35e37460f6def2", "cheese straws13-860d866a01d15c48", "cheese straws14-7a3ff9be505828c8", "cheese straws15-b86ec0ee2c9a648b", "cheese straws16-77a6f4dd7702688b", "cheese straws17-69013db73b97bb03", "tupperware3-bf1775b117f4f898", "tupperware4-d687ebec477cd306", "tupperware5-9e426892fdb82695", "tupperware lid5-c30d504a77a98f18", "tart1-95417e179d2dbc2d", "tart2-18472503d11536bd", "tart3-d4862dd226c3493d", "tupperware lid3-6ed5d28d11abae6a", "tart4-6ee86ad5ee7ae462", "tart6-6104f7e51d3757c8", "tart7-119a7ea9e5ca6fe2", "tart8-5de6071a86967ab3", "tart9-019767a4de0a77a1", "saucer2-cef73356046354a8", "tupperware6-e25ae9bc858ae13c", "tupperware7-486a9185105bb78b", "tupperware lid7-bfe937803be3f967", "tupperware lid6-c86cfdeadfb62ab8", "tart10-773fa939c4e3682e", "tart11-7500b08699129e6d", "tart12-60995b13a0b0f2e3", "tart13-788595d1f21e2aff", "tart14-00b6ccd0f4477170", "tart15-39caf2792a9c7713", "tart16-84c0e1584b193347", "tart17-38f8bf0cbc2d000d", "yoghurt-92e9bc37860f8014", "lid of yoghurt-9d3e859b5bfd22a9", "cling film box-2803b28bce61587d", "tablespoon2-793978b9e0b2239e", "sponge-9e7e751606140eb5", "large mixing bowl-de2cacdb1e189ec0", "teaspoon2-8f9e973b79662102", "piece of cling film-e3a5c0d96aeba6ec", "object in fridge-af0e03b56dde3a0d", "measuring jug-ff048110b78b0144", "jam jar-ceb21eea5f819900", "piece of kitchen towel3-7ad2cc6f17a94467", "jam jar2-3596c81be19c6f73", "butter knife-246bec285e32a315", "baking trays-b6854d9c54c7edac", "teaspoon3-ead6fbe11f2c5b70", "spatula-4dca59671cad6995", "pan gripper-5b12f1742b4cc731", "pan gripper2-36ccee4dbb3ec3cc", "large plastic spoon-b2fb0b96bd089aee", "baking tray2-d6cdb26743456365", "plate2-b86b86614b47c004", "large saucepan-69a2ca9fa5530d85", "saucer3-6ae864bef179cfec", "bread knife-c7af07bcc3cdfceb", "food bin-f7ba3ea4e82192e9", "lid of saucepan-5f7969a5044beefa", "muffin tray1-1617228fcb5787e2", "muffin tray2-6067517d419c17cd", "washing up brush-e8c2e8d218d9a011", "drain protector-a6ff8b48edfce6ac", "recipe book-da21f089852f9162", "cardboard insert-bc8f137b942e0b29", "scales-f4654fedabef9df9", "scale instructions-1252a27eff3951a8", "scales box-5bfb8cb9558e9fa9", "packet of tomatoes-472e26daa314566e", "vegetable drawer-3a5ba00d07d7568b" ], "reasoning": "1. I watched for the user to get out yogurt (yoghurt-92e9bc37860f8014), which they retrieve from the refrigerator at 07:46.\n2. I then looked for the user to get a spoon (tablespoon2-793978b9e0b2239e), which they retrieve from a drawer at 08:15.\n3. The user scoops yogurt (yoghurt-92e9bc37860f8014) into a bowl with the spoon (tablespoon2-793978b9e0b2239e) from 08:19-08:26.\n4. The user then licks the spoon (tablespoon2-793978b9e0b2239e) at 08:29 while walking towards the sink.", "referenced_object_masks": [ "tablespoon2-793978b9e0b2239e", "yoghurt-92e9bc37860f8014" ] }, { "key": "P02-20240211-141313:c5482d587c736f0729e949cce810b224750a1997", "video_id": "P02-20240211-141313", "question": "What number would I get if I multiplied the number of spoonfuls of yogurt I scooped out of the yogurt container, with the number of spoonfuls of red fruit puree that I scooped into the yogurt container?", "answer_choice_0": "16", "answer_choice_1": "24", "answer_choice_2": "27", "answer_choice_3": "20", "answer_choice_4": "15", "answer_id": 2, "answer": "27", "question_type": [ "Counting", "Numerical Reasoning", "Object Recognition" ], "objects_of_interest": [ "serrated knife-60cf5135ef7afc2e", "plate1-894f66f3439f9895", "empty packaging1-abf74550309926c4", "box-da932b373d45576c", "empty packaging2-b150954d5fc6570c", "hand blender-721dea9a6a7307ac", "citrus juicer-c857fa2762ca3680", "citrus juicer guard-756401bd3085acf1", "tea towel-b869d30560eadbb9", "blue cloth-c7d3d4a38b341f7d", "green cloth-c4c589a84251c391", "tea towel2-f5f8e7cc20582cd1", "hand blender attachment-1cf0e440d388e023", "cutting board-b297db957087d00f", "tablespoon-bd29fcdcdc219888", "teaspoon-9e58ff82a2b8bcd7", "black washing up bowl-a94762e89cf55c0d", "washing up liquid bottle-54a7f7ef5869669a", "rolling pen-777d57e5b8a8d1c6", "grater-8ccd82718960f41e", "hand blender cup-215dff0892a23060", "hand blender cup lid-252316a7585aacf5", "stack of colanders-845a245e3239cd67", "saucer-7625e366914cb2f9", "bowl-63193a7b18c63c67", "scissors-b2ea14634cf86dd3", "fork-7a206faccb94b2b2", "kitchen towel holder-7660f7bc5f068eae", "piece of kitchen towel-f740d4c6740682e3", "tupperware lid-70fc85232cec14b5", "cheese straws-d7560f1fc41b778e", "cheese straws2-b046b87fa1f0e0b2", "cheese straws3-4388eb170a6c07a3", "cheese straws4-9b4fa02070fb3ee7", "cheese straws5-db13821576a3234a", "cheese straws6-141c3f94b437a579", "cheese straws7-58b603cb02a3d93b", "cheese straws8-bf1a9e3b1c95daab", "cheese straws9-29030461bbe5bb0a", "cheese straws10-452ee213912ed91f", "piece of kitchen towel2-32b7b781f220f18c", "tupperware-7cca882f645c61bb", "tupperware2-ad03df2f3b800b2e", "tupperware lid2-1fdb135dcebc1070", "cheese straws11-a1f9c506fdc50427", "some cheese straws-ef35e37460f6def2", "cheese straws13-860d866a01d15c48", "cheese straws14-7a3ff9be505828c8", "cheese straws15-b86ec0ee2c9a648b", "cheese straws16-77a6f4dd7702688b", "cheese straws17-69013db73b97bb03", "tupperware3-bf1775b117f4f898", "tupperware4-d687ebec477cd306", "tupperware5-9e426892fdb82695", "tupperware lid5-c30d504a77a98f18", "tart1-95417e179d2dbc2d", "tart2-18472503d11536bd", "tart3-d4862dd226c3493d", "tupperware lid3-6ed5d28d11abae6a", "tart4-6ee86ad5ee7ae462", "tart6-6104f7e51d3757c8", "tart7-119a7ea9e5ca6fe2", "tart8-5de6071a86967ab3", "tart9-019767a4de0a77a1", "saucer2-cef73356046354a8", "tupperware6-e25ae9bc858ae13c", "tupperware7-486a9185105bb78b", "tupperware lid7-bfe937803be3f967", "tupperware lid6-c86cfdeadfb62ab8", "tart10-773fa939c4e3682e", "tart11-7500b08699129e6d", "tart12-60995b13a0b0f2e3", "tart13-788595d1f21e2aff", "tart14-00b6ccd0f4477170", "tart15-39caf2792a9c7713", "tart16-84c0e1584b193347", "tart17-38f8bf0cbc2d000d", "yoghurt-92e9bc37860f8014", "lid of yoghurt-9d3e859b5bfd22a9", "cling film box-2803b28bce61587d", "tablespoon2-793978b9e0b2239e", "sponge-9e7e751606140eb5", "large mixing bowl-de2cacdb1e189ec0", "teaspoon2-8f9e973b79662102", "piece of cling film-e3a5c0d96aeba6ec", "object in fridge-af0e03b56dde3a0d", "measuring jug-ff048110b78b0144", "jam jar-ceb21eea5f819900", "piece of kitchen towel3-7ad2cc6f17a94467", "jam jar2-3596c81be19c6f73", "butter knife-246bec285e32a315", "baking trays-b6854d9c54c7edac", "teaspoon3-ead6fbe11f2c5b70", "spatula-4dca59671cad6995", "pan gripper-5b12f1742b4cc731", "pan gripper2-36ccee4dbb3ec3cc", "large plastic spoon-b2fb0b96bd089aee", "baking tray2-d6cdb26743456365", "plate2-b86b86614b47c004", "large saucepan-69a2ca9fa5530d85", "saucer3-6ae864bef179cfec", "bread knife-c7af07bcc3cdfceb", "food bin-f7ba3ea4e82192e9", "lid of saucepan-5f7969a5044beefa", "muffin tray1-1617228fcb5787e2", "muffin tray2-6067517d419c17cd", "washing up brush-e8c2e8d218d9a011", "drain protector-a6ff8b48edfce6ac", "recipe book-da21f089852f9162", "cardboard insert-bc8f137b942e0b29", "scales-f4654fedabef9df9", "scale instructions-1252a27eff3951a8", "scales box-5bfb8cb9558e9fa9", "packet of tomatoes-472e26daa314566e", "vegetable drawer-3a5ba00d07d7568b" ], "reasoning": "1. I watched the video for the part where the person scooped yogurt and a bowl of fruit puree with a spoon. I first observed the person scooping yogurt at 08:17. \n2. I then counted the number of scoops of yogurt that the person removes from the bowl: between 08:17 and 08:27, I counted 3 scoops, before the person cleans the spoon and retrieves the fruit puree at 08:55. \n3. The person then holds the bowl of puree over the yogurt and 08:57, and from this point until 09:33, I counted 9 spoonfuls of puree spooned into the yogurt. \n4. I then multiplied the number of yogurt scoops, 3, by puree scoops, 9, to get the answer of 27.", "referenced_object_masks": [] }, { "key": "P02-20240211-141313:e77785af5b4f59091f16dc5b42d8c5e66ff645f9", "video_id": "P02-20240211-141313", "question": "What is the fifth action I used my right hand for after putting away the cheese grater?", "answer_choice_0": "Adjusting faucet", "answer_choice_1": "Turning sink knob", "answer_choice_2": "Drying clear plastic cup", "answer_choice_3": "Putting clear plastic cup away", "answer_choice_4": "Grabbing clear plastic cup", "answer_id": 0, "answer": "Adjusting faucet", "question_type": [ "Counting", "Event Occurence", "Object Recognition" ], "objects_of_interest": [ "serrated knife-60cf5135ef7afc2e", "plate1-894f66f3439f9895", "empty packaging1-abf74550309926c4", "box-da932b373d45576c", "empty packaging2-b150954d5fc6570c", "hand blender-721dea9a6a7307ac", "citrus juicer-c857fa2762ca3680", "citrus juicer guard-756401bd3085acf1", "tea towel-b869d30560eadbb9", "blue cloth-c7d3d4a38b341f7d", "green cloth-c4c589a84251c391", "tea towel2-f5f8e7cc20582cd1", "hand blender attachment-1cf0e440d388e023", "cutting board-b297db957087d00f", "tablespoon-bd29fcdcdc219888", "teaspoon-9e58ff82a2b8bcd7", "black washing up bowl-a94762e89cf55c0d", "washing up liquid bottle-54a7f7ef5869669a", "rolling pen-777d57e5b8a8d1c6", "grater-8ccd82718960f41e", "hand blender cup-215dff0892a23060", "hand blender cup lid-252316a7585aacf5", "stack of colanders-845a245e3239cd67", "saucer-7625e366914cb2f9", "bowl-63193a7b18c63c67", "scissors-b2ea14634cf86dd3", "fork-7a206faccb94b2b2", "kitchen towel holder-7660f7bc5f068eae", "piece of kitchen towel-f740d4c6740682e3", "tupperware lid-70fc85232cec14b5", "cheese straws-d7560f1fc41b778e", "cheese straws2-b046b87fa1f0e0b2", "cheese straws3-4388eb170a6c07a3", "cheese straws4-9b4fa02070fb3ee7", "cheese straws5-db13821576a3234a", "cheese straws6-141c3f94b437a579", "cheese straws7-58b603cb02a3d93b", "cheese straws8-bf1a9e3b1c95daab", "cheese straws9-29030461bbe5bb0a", "cheese straws10-452ee213912ed91f", "piece of kitchen towel2-32b7b781f220f18c", "tupperware-7cca882f645c61bb", "tupperware2-ad03df2f3b800b2e", "tupperware lid2-1fdb135dcebc1070", "cheese straws11-a1f9c506fdc50427", "some cheese straws-ef35e37460f6def2", "cheese straws13-860d866a01d15c48", "cheese straws14-7a3ff9be505828c8", "cheese straws15-b86ec0ee2c9a648b", "cheese straws16-77a6f4dd7702688b", "cheese straws17-69013db73b97bb03", "tupperware3-bf1775b117f4f898", "tupperware4-d687ebec477cd306", "tupperware5-9e426892fdb82695", "tupperware lid5-c30d504a77a98f18", "tart1-95417e179d2dbc2d", "tart2-18472503d11536bd", "tart3-d4862dd226c3493d", "tupperware lid3-6ed5d28d11abae6a", "tart4-6ee86ad5ee7ae462", "tart6-6104f7e51d3757c8", "tart7-119a7ea9e5ca6fe2", "tart8-5de6071a86967ab3", "tart9-019767a4de0a77a1", "saucer2-cef73356046354a8", "tupperware6-e25ae9bc858ae13c", "tupperware7-486a9185105bb78b", "tupperware lid7-bfe937803be3f967", "tupperware lid6-c86cfdeadfb62ab8", "tart10-773fa939c4e3682e", "tart11-7500b08699129e6d", "tart12-60995b13a0b0f2e3", "tart13-788595d1f21e2aff", "tart14-00b6ccd0f4477170", "tart15-39caf2792a9c7713", "tart16-84c0e1584b193347", "tart17-38f8bf0cbc2d000d", "yoghurt-92e9bc37860f8014", "lid of yoghurt-9d3e859b5bfd22a9", "cling film box-2803b28bce61587d", "tablespoon2-793978b9e0b2239e", "sponge-9e7e751606140eb5", "large mixing bowl-de2cacdb1e189ec0", "teaspoon2-8f9e973b79662102", "piece of cling film-e3a5c0d96aeba6ec", "object in fridge-af0e03b56dde3a0d", "measuring jug-ff048110b78b0144", "jam jar-ceb21eea5f819900", "piece of kitchen towel3-7ad2cc6f17a94467", "jam jar2-3596c81be19c6f73", "butter knife-246bec285e32a315", "baking trays-b6854d9c54c7edac", "teaspoon3-ead6fbe11f2c5b70", "spatula-4dca59671cad6995", "pan gripper-5b12f1742b4cc731", "pan gripper2-36ccee4dbb3ec3cc", "large plastic spoon-b2fb0b96bd089aee", "baking tray2-d6cdb26743456365", "plate2-b86b86614b47c004", "large saucepan-69a2ca9fa5530d85", "saucer3-6ae864bef179cfec", "bread knife-c7af07bcc3cdfceb", "food bin-f7ba3ea4e82192e9", "lid of saucepan-5f7969a5044beefa", "muffin tray1-1617228fcb5787e2", "muffin tray2-6067517d419c17cd", "washing up brush-e8c2e8d218d9a011", "drain protector-a6ff8b48edfce6ac", "recipe book-da21f089852f9162", "cardboard insert-bc8f137b942e0b29", "scales-f4654fedabef9df9", "scale instructions-1252a27eff3951a8", "scales box-5bfb8cb9558e9fa9", "packet of tomatoes-472e26daa314566e", "vegetable drawer-3a5ba00d07d7568b" ], "reasoning": "1. I watched the video to first identify when the cheese grater (grater-8ccd82718960f41e) was handled and put away - I observed this action from 02:04-02:06. \n2. I then paid attention to the actions that the subject performs with their right hand - first, at 02:07, I noticed them pick up a clear plastic cup (hand blender cup-215dff0892a23060), passing it to their right hand. \n3. Second, I watched the subject run their right hand under the water at 02:09. \n4. Third, I watched them use their right hand to twist the sink knob from 02:09-02:12. \n5. Fourth, I watched the run their hand under the water again at 02:13. \n6. Fifth, I observed the person move the sink faucet rightward over the sink at 02:17, making this the correct answer.", "referenced_object_masks": [ "grater-8ccd82718960f41e", "hand blender cup-215dff0892a23060" ] }, { "key": "P02-20240211-141313:f06c29712de10434c767b0a941e147d33536e6bd", "video_id": "P02-20240211-141313", "question": "What is in front of the object I take out of the dish drainer after putting away the citrus juicer?", "answer_choice_0": "A plate", "answer_choice_1": "A spatula", "answer_choice_2": "A hand blender attachment", "answer_choice_3": "A cutting board", "answer_choice_4": "A bowl", "answer_id": 4, "answer": "A bowl", "question_type": [ "Action Recognition", "Object Recognition", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "serrated knife-60cf5135ef7afc2e", "plate1-894f66f3439f9895", "empty packaging1-abf74550309926c4", "box-da932b373d45576c", "empty packaging2-b150954d5fc6570c", "hand blender-721dea9a6a7307ac", "citrus juicer-c857fa2762ca3680", "citrus juicer guard-756401bd3085acf1", "tea towel-b869d30560eadbb9", "blue cloth-c7d3d4a38b341f7d", "green cloth-c4c589a84251c391", "tea towel2-f5f8e7cc20582cd1", "hand blender attachment-1cf0e440d388e023", "cutting board-b297db957087d00f", "tablespoon-bd29fcdcdc219888", "teaspoon-9e58ff82a2b8bcd7", "black washing up bowl-a94762e89cf55c0d", "washing up liquid bottle-54a7f7ef5869669a", "rolling pen-777d57e5b8a8d1c6", "grater-8ccd82718960f41e", "hand blender cup-215dff0892a23060", "hand blender cup lid-252316a7585aacf5", "stack of colanders-845a245e3239cd67", "saucer-7625e366914cb2f9", "bowl-63193a7b18c63c67", "scissors-b2ea14634cf86dd3", "fork-7a206faccb94b2b2", "kitchen towel holder-7660f7bc5f068eae", "piece of kitchen towel-f740d4c6740682e3", "tupperware lid-70fc85232cec14b5", "cheese straws-d7560f1fc41b778e", "cheese straws2-b046b87fa1f0e0b2", "cheese straws3-4388eb170a6c07a3", "cheese straws4-9b4fa02070fb3ee7", "cheese straws5-db13821576a3234a", "cheese straws6-141c3f94b437a579", "cheese straws7-58b603cb02a3d93b", "cheese straws8-bf1a9e3b1c95daab", "cheese straws9-29030461bbe5bb0a", "cheese straws10-452ee213912ed91f", "piece of kitchen towel2-32b7b781f220f18c", "tupperware-7cca882f645c61bb", "tupperware2-ad03df2f3b800b2e", "tupperware lid2-1fdb135dcebc1070", "cheese straws11-a1f9c506fdc50427", "some cheese straws-ef35e37460f6def2", "cheese straws13-860d866a01d15c48", "cheese straws14-7a3ff9be505828c8", "cheese straws15-b86ec0ee2c9a648b", "cheese straws16-77a6f4dd7702688b", "cheese straws17-69013db73b97bb03", "tupperware3-bf1775b117f4f898", "tupperware4-d687ebec477cd306", "tupperware5-9e426892fdb82695", "tupperware lid5-c30d504a77a98f18", "tart1-95417e179d2dbc2d", "tart2-18472503d11536bd", "tart3-d4862dd226c3493d", "tupperware lid3-6ed5d28d11abae6a", "tart4-6ee86ad5ee7ae462", "tart6-6104f7e51d3757c8", "tart7-119a7ea9e5ca6fe2", "tart8-5de6071a86967ab3", "tart9-019767a4de0a77a1", "saucer2-cef73356046354a8", "tupperware6-e25ae9bc858ae13c", "tupperware7-486a9185105bb78b", "tupperware lid7-bfe937803be3f967", "tupperware lid6-c86cfdeadfb62ab8", "tart10-773fa939c4e3682e", "tart11-7500b08699129e6d", "tart12-60995b13a0b0f2e3", "tart13-788595d1f21e2aff", "tart14-00b6ccd0f4477170", "tart15-39caf2792a9c7713", "tart16-84c0e1584b193347", "tart17-38f8bf0cbc2d000d", "yoghurt-92e9bc37860f8014", "lid of yoghurt-9d3e859b5bfd22a9", "cling film box-2803b28bce61587d", "tablespoon2-793978b9e0b2239e", "sponge-9e7e751606140eb5", "large mixing bowl-de2cacdb1e189ec0", "teaspoon2-8f9e973b79662102", "piece of cling film-e3a5c0d96aeba6ec", "object in fridge-af0e03b56dde3a0d", "measuring jug-ff048110b78b0144", "jam jar-ceb21eea5f819900", "piece of kitchen towel3-7ad2cc6f17a94467", "jam jar2-3596c81be19c6f73", "butter knife-246bec285e32a315", "baking trays-b6854d9c54c7edac", "teaspoon3-ead6fbe11f2c5b70", "spatula-4dca59671cad6995", "pan gripper-5b12f1742b4cc731", "pan gripper2-36ccee4dbb3ec3cc", "large plastic spoon-b2fb0b96bd089aee", "baking tray2-d6cdb26743456365", "plate2-b86b86614b47c004", "large saucepan-69a2ca9fa5530d85", "saucer3-6ae864bef179cfec", "bread knife-c7af07bcc3cdfceb", "food bin-f7ba3ea4e82192e9", "lid of saucepan-5f7969a5044beefa", "muffin tray1-1617228fcb5787e2", "muffin tray2-6067517d419c17cd", "washing up brush-e8c2e8d218d9a011", "drain protector-a6ff8b48edfce6ac", "recipe book-da21f089852f9162", "cardboard insert-bc8f137b942e0b29", "scales-f4654fedabef9df9", "scale instructions-1252a27eff3951a8", "scales box-5bfb8cb9558e9fa9", "packet of tomatoes-472e26daa314566e", "vegetable drawer-3a5ba00d07d7568b" ], "reasoning": "1. I watched for the user pick up the citrus juicer (citrus juicer-c857fa2762ca3680), which they do at 00:36.\n2. The user then puts the citrus juicer (citrus juicer-c857fa2762ca3680) away in the cupboard at 01:15.\n3. I looked to see what the next object the user took out of the dish drainer was, and it was a hand blender attachment (hand blender attachment-1cf0e440d388e023) at 01:18.\n4. The object directly in front of the hand blender attachment (hand blender attachment-1cf0e440d388e023) while it was in the dish drainer is a bowl (bowl-63193a7b18c63c67).", "referenced_object_masks": [ "hand blender attachment-1cf0e440d388e023", "bowl-63193a7b18c63c67", "citrus juicer-c857fa2762ca3680" ] }, { "key": "P02-20240211-141313:fa4ec754f0ab277bf875949a595f8e00418f78b4", "video_id": "P02-20240211-141313", "question": "What color is the third object that I picked up off the dining table with my right hand?", "answer_choice_0": "Blue", "answer_choice_1": "Red", "answer_choice_2": "White", "answer_choice_3": "Silver", "answer_choice_4": "Brown", "answer_id": 0, "answer": "Blue", "question_type": [ "Counting", "Object Recognition" ], "objects_of_interest": [ "serrated knife-60cf5135ef7afc2e", "plate1-894f66f3439f9895", "empty packaging1-abf74550309926c4", "box-da932b373d45576c", "empty packaging2-b150954d5fc6570c", "hand blender-721dea9a6a7307ac", "citrus juicer-c857fa2762ca3680", "citrus juicer guard-756401bd3085acf1", "tea towel-b869d30560eadbb9", "blue cloth-c7d3d4a38b341f7d", "green cloth-c4c589a84251c391", "tea towel2-f5f8e7cc20582cd1", "hand blender attachment-1cf0e440d388e023", "cutting board-b297db957087d00f", "tablespoon-bd29fcdcdc219888", "teaspoon-9e58ff82a2b8bcd7", "black washing up bowl-a94762e89cf55c0d", "washing up liquid bottle-54a7f7ef5869669a", "rolling pen-777d57e5b8a8d1c6", "grater-8ccd82718960f41e", "hand blender cup-215dff0892a23060", "hand blender cup lid-252316a7585aacf5", "stack of colanders-845a245e3239cd67", "saucer-7625e366914cb2f9", "bowl-63193a7b18c63c67", "scissors-b2ea14634cf86dd3", "fork-7a206faccb94b2b2", "kitchen towel holder-7660f7bc5f068eae", "piece of kitchen towel-f740d4c6740682e3", "tupperware lid-70fc85232cec14b5", "cheese straws-d7560f1fc41b778e", "cheese straws2-b046b87fa1f0e0b2", "cheese straws3-4388eb170a6c07a3", "cheese straws4-9b4fa02070fb3ee7", "cheese straws5-db13821576a3234a", "cheese straws6-141c3f94b437a579", "cheese straws7-58b603cb02a3d93b", "cheese straws8-bf1a9e3b1c95daab", "cheese straws9-29030461bbe5bb0a", "cheese straws10-452ee213912ed91f", "piece of kitchen towel2-32b7b781f220f18c", "tupperware-7cca882f645c61bb", "tupperware2-ad03df2f3b800b2e", "tupperware lid2-1fdb135dcebc1070", "cheese straws11-a1f9c506fdc50427", "some cheese straws-ef35e37460f6def2", "cheese straws13-860d866a01d15c48", "cheese straws14-7a3ff9be505828c8", "cheese straws15-b86ec0ee2c9a648b", "cheese straws16-77a6f4dd7702688b", "cheese straws17-69013db73b97bb03", "tupperware3-bf1775b117f4f898", "tupperware4-d687ebec477cd306", "tupperware5-9e426892fdb82695", "tupperware lid5-c30d504a77a98f18", "tart1-95417e179d2dbc2d", "tart2-18472503d11536bd", "tart3-d4862dd226c3493d", "tupperware lid3-6ed5d28d11abae6a", "tart4-6ee86ad5ee7ae462", "tart6-6104f7e51d3757c8", "tart7-119a7ea9e5ca6fe2", "tart8-5de6071a86967ab3", "tart9-019767a4de0a77a1", "saucer2-cef73356046354a8", "tupperware6-e25ae9bc858ae13c", "tupperware7-486a9185105bb78b", "tupperware lid7-bfe937803be3f967", "tupperware lid6-c86cfdeadfb62ab8", "tart10-773fa939c4e3682e", "tart11-7500b08699129e6d", "tart12-60995b13a0b0f2e3", "tart13-788595d1f21e2aff", "tart14-00b6ccd0f4477170", "tart15-39caf2792a9c7713", "tart16-84c0e1584b193347", "tart17-38f8bf0cbc2d000d", "yoghurt-92e9bc37860f8014", "lid of yoghurt-9d3e859b5bfd22a9", "cling film box-2803b28bce61587d", "tablespoon2-793978b9e0b2239e", "sponge-9e7e751606140eb5", "large mixing bowl-de2cacdb1e189ec0", "teaspoon2-8f9e973b79662102", "piece of cling film-e3a5c0d96aeba6ec", "object in fridge-af0e03b56dde3a0d", "measuring jug-ff048110b78b0144", "jam jar-ceb21eea5f819900", "piece of kitchen towel3-7ad2cc6f17a94467", "jam jar2-3596c81be19c6f73", "butter knife-246bec285e32a315", "baking trays-b6854d9c54c7edac", "teaspoon3-ead6fbe11f2c5b70", "spatula-4dca59671cad6995", "pan gripper-5b12f1742b4cc731", "pan gripper2-36ccee4dbb3ec3cc", "large plastic spoon-b2fb0b96bd089aee", "baking tray2-d6cdb26743456365", "plate2-b86b86614b47c004", "large saucepan-69a2ca9fa5530d85", "saucer3-6ae864bef179cfec", "bread knife-c7af07bcc3cdfceb", "food bin-f7ba3ea4e82192e9", "lid of saucepan-5f7969a5044beefa", "muffin tray1-1617228fcb5787e2", "muffin tray2-6067517d419c17cd", "washing up brush-e8c2e8d218d9a011", "drain protector-a6ff8b48edfce6ac", "recipe book-da21f089852f9162", "cardboard insert-bc8f137b942e0b29", "scales-f4654fedabef9df9", "scale instructions-1252a27eff3951a8", "scales box-5bfb8cb9558e9fa9", "packet of tomatoes-472e26daa314566e", "vegetable drawer-3a5ba00d07d7568b" ], "reasoning": "1. I watched the video to observe the subject as they picked things up of the dining table, paying attention specifically to their right hand. The sequence starts at 0:14.\n2. At 0:14, I observed the person picking up a red-handled kitchen knife (serrated knife-60cf5135ef7afc2e) with their right hand, counting this as the first item.\n3. At 0:16, I watched the person pick up a blue plastic wrapper with their right hand (empty packaging1-abf74550309926c4), marking this as the second item.\n4. At 0:18, I watched the person pick up a second blue plastic wrapper off the table (empty packaging2-b150954d5fc6570c), counting this as the third item, and noting its color, blue, to get the correct answer.", "referenced_object_masks": [ "empty packaging1-abf74550309926c4", "serrated knife-60cf5135ef7afc2e", "empty packaging2-b150954d5fc6570c" ] }, { "key": "P02-20240211-175138:17b6f887040a6abd2aba9da6d5d2b96ca0c69245", "video_id": "P02-20240211-175138", "question": "When I wash the first batch of mushrooms, at what timecode are there precisely half as many mushrooms to the right of the sink as there are inside the sink?", "answer_choice_0": "04:32", "answer_choice_1": "03:46", "answer_choice_2": "01:28", "answer_choice_3": "03:13", "answer_choice_4": "02:27", "answer_id": 3, "answer": "03:13", "question_type": [ "Object Recognition", "Counting", "Numerical Reasoning", "Action Recognition", "Egocentric & Spatial Perception", "Temporal Reasoning" ], "objects_of_interest": [ "phone-e3c4c3cf4aacbafc", "recipe sheet-f1cad958e3f385f1", "plastic wrapper-1ada86e0171adb72", "pint glass-08273ca31b54ecea", "washing up liquid bottle-c581844420f23634", "cutting board-7d550d8e6d273150", "washing up bowl-45987bf991af86ad", "tupperware-c40f43f40f56f379", "packet of mushrooms1-dd1a59533ca0096a", "packet of mushrooms2-aa346d7d5e41892a", "tomatoes-e576112019072510", "packet of red onions-f5cbe588f24087fb", "baking tray1-4b716abaad27b7a7", "baking tray2-0b081f43b592c0f7", "box of foil roll-70796fde4537a552", "sheet of foil1-3d4b9dfa0a998a34", "plastic packaging of mushroom1-aa73afc1022b2f75", "mushroom1-b7de39c106df7218", "mushroom2-5b96a9ebf3b2e85d", "mushroom3-10c6533bb2a246bd", "mushroom4-ddd4918d43691f83", "mushroom5-e4cbf25c87731613", "mushroom6-ece92bfdf17cb502", "mushroom7-9f0cb68ade84de50", "mushroom8-24cc3c38ee4390b0", "mushroom9-5f3481321e53599d", "mushroom10-f320f5291c0fd1e2", "mushroom11-133563985d9a1d90", "mushroom12-b55dae0a1ec4c15b", "mushroom13-77387330a3e829bb", "mushroom14-4bc2a42075fa5052", "mushroom15-0aeb457828737a29", "mushroom16-a074b128f2857f3a", "mushroom17-3e6e3ebad690017a", "mushroom18-f4d8b8299a4b1fa7", "mushroom19-fcbd68479317b78a", "mushroom20-657591b4dfdff79f", "mushroom21-c5ee43cbfc384730", "towel-8d4b397557064586", "large mixing bowl-b2a18fcfa935ccc2", "cayennne pepper spice jar-ece5eeec002ee167", "cumin spice jar-53f87bf27776c074", "paprika spice jar-83c73927d4330f44", "garam masala spice jar-c7f15589b1219053", "ginger spice jar-4b58e369a7018d9d", "basket of spices-3ea1aac946265309", "ginger powder packet-f0c060929e848ec6", "coriander packet-2730f85fc6ce667a", "clip-fcbb34fcb84208b9", "teaspoon-b67daef958db7592", "cumin spice jar lid-9c893bf858830c9e", "paprika spice jar lid-a396e6d7f772d8f6", "cayenne pepper spice jar-a400e8a00b25d418", "cayenne pepper spice jar lid-69020437dd0b21fa", "salt bottle-6b7c4d0fcac0f02b", "olive oil bottle-4443d2de7a2f3d1d", "tablespoon-e10d8cc98abebd20", "olive oil bottle lid-d649041ef00753e3", "serrated knife-55dbeabd977bc7b1", "bag of lemons-e72f4acb16b83a07", "lemon-e59acc2de411a652", "chef's knife-73111e3b9f93ddcf", "lemon half-a165dd867eef32d0", "food bin-2d940b254a258670", "sheet of foil2-1c47789129d95778", "red onion1-2baf306bd300c710", "red onion2-d163ea9e77bba3b5", "kitchen roll holder-16717aa38d9c6761", "kitchen towel sheet-4efc7069a149c8fb", "red onion1 half-d39fbc223c29f506", "red onion2 half-8cd4d1448a5540ff", "bowl-0ab274b6aefb8ab0", "paring knife-2491d9a022c695ee", "pepper mill-bfda19f3fe2dd99f", "pepper mill lid-4a2edd3b5c496f7a", "tablespoon2-a34ed0373b19c312", "minced garlic jar-85f46278462bdd06", "saucepan-4eb82fdc5e07dcea", "lid of saucepan-aae7fa5ab65189ce", "measuring jug-743b6f9e58fdab13", "sugar container-a1754f76c055c1f1", "sugar container lid-d653df1b6dd3628c", "tablespoon3-c92d1d368b93f83e", "can of something-94b4bbeb5fd47100", "worcestershire sauce bottle-8452f56b46feb4a0", "teaspoon2-c71c546ac4a2553f", "large plastic spoon-7987e29555606842", "worcestershire sauce bottle lid-26b8e908be753b5f", "lid of minced garlic jar-5b6f5163609dbfc1", "muffin tray1-6003a2b3b7d8ae6f", "muffin tray2-f7111da630b333e9", "plate1-357de90612e03533", "plate2-e35aff14357524e7", "plate3-19fd3f43fc135ffd", "teaspoon3-826e32548eff839a", "tablespoon4-f841e6911011217f", "bread knife-fbb4458baa763eb1", "butter knife-5c375f1ce479ea96", "spatula-45f1e2690b505215", "pan gripper1-05895977da0731ea", "pan gripper2-297e812676d64723" ], "reasoning": "1. I look for the time I begin to wash the first batch of mushrooms (packet of mushrooms1-dd1a59533ca0096a), and I find it from 02:24-03:46.\n2. The user washes 12 mushrooms, the first (mushroom1-b7de39c106df7218) at 02:27, the second (mushroom2-5b96a9ebf3b2e85d) at 02:32, the third (mushroom3-10c6533bb2a246bd) at 02:41, the fourth (mushroom4-ddd4918d43691f83) at 02:44, the fifth (mushroom5-e4cbf25c87731613) at 02:47, the sixth (mushroom6-ece92bfdf17cb502) at 02:53, the seventh (mushroom7-9f0cb68ade84de50) at 03:15, the eighth (mushroom8-24cc3c38ee4390b0) at 03:24, the ninth (mushroom9-5f3481321e53599d) 03:28, the tenth at (mushroom10-f320f5291c0fd1e2) 03:34, the eleventh (mushroom11-133563985d9a1d90) at 03:38, and the twelfth (mushroom12-b55dae0a1ec4c15b) at 03:41. The user places the final mushroom (mushroom12-b55dae0a1ec4c15b) to the right of the sink at \n03:45.\n3. I try to find the time the user places the sixth mushroom (mushroom6-ece92bfdf17cb502) to the right of the sink, since at this time, there should be exactly half to the right of the sink and half inside the sink.\n4. I examine the timecode at 02:53, and realize that the user places this mushroom (mushroom6-ece92bfdf17cb502) to the right of the sink at 03:13.\n5. Therefore, 03:13 is the answer.", "referenced_object_masks": [ "mushroom3-10c6533bb2a246bd", "mushroom9-5f3481321e53599d", "mushroom2-5b96a9ebf3b2e85d", "mushroom4-ddd4918d43691f83", "mushroom7-9f0cb68ade84de50", "mushroom1-b7de39c106df7218", "mushroom10-f320f5291c0fd1e2", "mushroom6-ece92bfdf17cb502", "mushroom12-b55dae0a1ec4c15b", "packet of mushrooms1-dd1a59533ca0096a", "mushroom5-e4cbf25c87731613", "mushroom8-24cc3c38ee4390b0", "mushroom11-133563985d9a1d90" ] }, { "key": "P02-20240211-175138:1de042222f6a2ece65cdcb591be033c7dba8f0b0", "video_id": "P02-20240211-175138", "question": "What is written on the fifth bag I touch when I'm sorting through bags looking for coriander?", "answer_choice_0": "\"Ginger\"", "answer_choice_1": "\"Garam Masala\"", "answer_choice_2": "\"Chilli Flakes\"", "answer_choice_3": "\"Garlic Paste\"", "answer_choice_4": "\"Coriander\"", "answer_id": 2, "answer": "\"Chilli Flakes\"", "question_type": [ "Counting", "Object Recognition", "Reading" ], "objects_of_interest": [ "phone-e3c4c3cf4aacbafc", "recipe sheet-f1cad958e3f385f1", "plastic wrapper-1ada86e0171adb72", "pint glass-08273ca31b54ecea", "washing up liquid bottle-c581844420f23634", "cutting board-7d550d8e6d273150", "washing up bowl-45987bf991af86ad", "tupperware-c40f43f40f56f379", "packet of mushrooms1-dd1a59533ca0096a", "packet of mushrooms2-aa346d7d5e41892a", "tomatoes-e576112019072510", "packet of red onions-f5cbe588f24087fb", "baking tray1-4b716abaad27b7a7", "baking tray2-0b081f43b592c0f7", "box of foil roll-70796fde4537a552", "sheet of foil1-3d4b9dfa0a998a34", "plastic packaging of mushroom1-aa73afc1022b2f75", "mushroom1-b7de39c106df7218", "mushroom2-5b96a9ebf3b2e85d", "mushroom3-10c6533bb2a246bd", "mushroom4-ddd4918d43691f83", "mushroom5-e4cbf25c87731613", "mushroom6-ece92bfdf17cb502", "mushroom7-9f0cb68ade84de50", "mushroom8-24cc3c38ee4390b0", "mushroom9-5f3481321e53599d", "mushroom10-f320f5291c0fd1e2", "mushroom11-133563985d9a1d90", "mushroom12-b55dae0a1ec4c15b", "mushroom13-77387330a3e829bb", "mushroom14-4bc2a42075fa5052", "mushroom15-0aeb457828737a29", "mushroom16-a074b128f2857f3a", "mushroom17-3e6e3ebad690017a", "mushroom18-f4d8b8299a4b1fa7", "mushroom19-fcbd68479317b78a", "mushroom20-657591b4dfdff79f", "mushroom21-c5ee43cbfc384730", "towel-8d4b397557064586", "large mixing bowl-b2a18fcfa935ccc2", "cayennne pepper spice jar-ece5eeec002ee167", "cumin spice jar-53f87bf27776c074", "paprika spice jar-83c73927d4330f44", "garam masala spice jar-c7f15589b1219053", "ginger spice jar-4b58e369a7018d9d", "basket of spices-3ea1aac946265309", "ginger powder packet-f0c060929e848ec6", "coriander packet-2730f85fc6ce667a", "clip-fcbb34fcb84208b9", "teaspoon-b67daef958db7592", "cumin spice jar lid-9c893bf858830c9e", "paprika spice jar lid-a396e6d7f772d8f6", "cayenne pepper spice jar-a400e8a00b25d418", "cayenne pepper spice jar lid-69020437dd0b21fa", "salt bottle-6b7c4d0fcac0f02b", "olive oil bottle-4443d2de7a2f3d1d", "tablespoon-e10d8cc98abebd20", "olive oil bottle lid-d649041ef00753e3", "serrated knife-55dbeabd977bc7b1", "bag of lemons-e72f4acb16b83a07", "lemon-e59acc2de411a652", "chef's knife-73111e3b9f93ddcf", "lemon half-a165dd867eef32d0", "food bin-2d940b254a258670", "sheet of foil2-1c47789129d95778", "red onion1-2baf306bd300c710", "red onion2-d163ea9e77bba3b5", "kitchen roll holder-16717aa38d9c6761", "kitchen towel sheet-4efc7069a149c8fb", "red onion1 half-d39fbc223c29f506", "red onion2 half-8cd4d1448a5540ff", "bowl-0ab274b6aefb8ab0", "paring knife-2491d9a022c695ee", "pepper mill-bfda19f3fe2dd99f", "pepper mill lid-4a2edd3b5c496f7a", "tablespoon2-a34ed0373b19c312", "minced garlic jar-85f46278462bdd06", "saucepan-4eb82fdc5e07dcea", "lid of saucepan-aae7fa5ab65189ce", "measuring jug-743b6f9e58fdab13", "sugar container-a1754f76c055c1f1", "sugar container lid-d653df1b6dd3628c", "tablespoon3-c92d1d368b93f83e", "can of something-94b4bbeb5fd47100", "worcestershire sauce bottle-8452f56b46feb4a0", "teaspoon2-c71c546ac4a2553f", "large plastic spoon-7987e29555606842", "worcestershire sauce bottle lid-26b8e908be753b5f", "lid of minced garlic jar-5b6f5163609dbfc1", "muffin tray1-6003a2b3b7d8ae6f", "muffin tray2-f7111da630b333e9", "plate1-357de90612e03533", "plate2-e35aff14357524e7", "plate3-19fd3f43fc135ffd", "teaspoon3-826e32548eff839a", "tablespoon4-f841e6911011217f", "bread knife-fbb4458baa763eb1", "butter knife-5c375f1ce479ea96", "spatula-45f1e2690b505215", "pan gripper1-05895977da0731ea", "pan gripper2-297e812676d64723" ], "reasoning": "1. I look for the time the user looks for coriander (coriander packet-2730f85fc6ce667a). From 06:20-06:42, the user searches through an assortment of bags held in a plasket basket (basket of spices-3ea1aac946265309). They flip through the bags, eventually choosing a bag of coriander (coriander packet-2730f85fc6ce667a).\n2. The first bag the user touches is a bag of ginger (ginger powder packet-f0c060929e848ec6) at 06:20.\n3. The second bag the user touches is the bag of coriander (coriander packet-2730f85fc6ce667a) at 06:26.\n4. The third bag the user touches is a bag of garam masala powder at 06:29.\n5. The fourth bag the user touches is another bag of garam masala powder at 06:30.\n6. The fifth bag the user touches is a bag of chili flakes .\n7. This fifth bag reads \"Chilli Flakes.\"\n8. Therefore, the answer is \"Chilli Flakes.\"", "referenced_object_masks": [ "basket of spices-3ea1aac946265309", "coriander packet-2730f85fc6ce667a", "ginger powder packet-f0c060929e848ec6" ] }, { "key": "P02-20240211-175138:3c1068ce80fb236f7838ab7571e6f72e6580d5b9", "video_id": "P02-20240211-175138", "question": "What do I turn to look at after I put down the recipe sheet for the final time?", "answer_choice_0": "The stove", "answer_choice_1": "The countertop", "answer_choice_2": "The closet", "answer_choice_3": "The table", "answer_choice_4": "The sink", "answer_id": 1, "answer": "The countertop", "question_type": [ "Egocentric & Spatial Perception", "Action Recognition", "Event Occurence", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "phone-e3c4c3cf4aacbafc", "recipe sheet-f1cad958e3f385f1", "plastic wrapper-1ada86e0171adb72", "pint glass-08273ca31b54ecea", "washing up liquid bottle-c581844420f23634", "cutting board-7d550d8e6d273150", "washing up bowl-45987bf991af86ad", "tupperware-c40f43f40f56f379", "packet of mushrooms1-dd1a59533ca0096a", "packet of mushrooms2-aa346d7d5e41892a", "tomatoes-e576112019072510", "packet of red onions-f5cbe588f24087fb", "baking tray1-4b716abaad27b7a7", "baking tray2-0b081f43b592c0f7", "box of foil roll-70796fde4537a552", "sheet of foil1-3d4b9dfa0a998a34", "plastic packaging of mushroom1-aa73afc1022b2f75", "mushroom1-b7de39c106df7218", "mushroom2-5b96a9ebf3b2e85d", "mushroom3-10c6533bb2a246bd", "mushroom4-ddd4918d43691f83", "mushroom5-e4cbf25c87731613", "mushroom6-ece92bfdf17cb502", "mushroom7-9f0cb68ade84de50", "mushroom8-24cc3c38ee4390b0", "mushroom9-5f3481321e53599d", "mushroom10-f320f5291c0fd1e2", "mushroom11-133563985d9a1d90", "mushroom12-b55dae0a1ec4c15b", "mushroom13-77387330a3e829bb", "mushroom14-4bc2a42075fa5052", "mushroom15-0aeb457828737a29", "mushroom16-a074b128f2857f3a", "mushroom17-3e6e3ebad690017a", "mushroom18-f4d8b8299a4b1fa7", "mushroom19-fcbd68479317b78a", "mushroom20-657591b4dfdff79f", "mushroom21-c5ee43cbfc384730", "towel-8d4b397557064586", "large mixing bowl-b2a18fcfa935ccc2", "cayennne pepper spice jar-ece5eeec002ee167", "cumin spice jar-53f87bf27776c074", "paprika spice jar-83c73927d4330f44", "garam masala spice jar-c7f15589b1219053", "ginger spice jar-4b58e369a7018d9d", "basket of spices-3ea1aac946265309", "ginger powder packet-f0c060929e848ec6", "coriander packet-2730f85fc6ce667a", "clip-fcbb34fcb84208b9", "teaspoon-b67daef958db7592", "cumin spice jar lid-9c893bf858830c9e", "paprika spice jar lid-a396e6d7f772d8f6", "cayenne pepper spice jar-a400e8a00b25d418", "cayenne pepper spice jar lid-69020437dd0b21fa", "salt bottle-6b7c4d0fcac0f02b", "olive oil bottle-4443d2de7a2f3d1d", "tablespoon-e10d8cc98abebd20", "olive oil bottle lid-d649041ef00753e3", "serrated knife-55dbeabd977bc7b1", "bag of lemons-e72f4acb16b83a07", "lemon-e59acc2de411a652", "chef's knife-73111e3b9f93ddcf", "lemon half-a165dd867eef32d0", "food bin-2d940b254a258670", "sheet of foil2-1c47789129d95778", "red onion1-2baf306bd300c710", "red onion2-d163ea9e77bba3b5", "kitchen roll holder-16717aa38d9c6761", "kitchen towel sheet-4efc7069a149c8fb", "red onion1 half-d39fbc223c29f506", "red onion2 half-8cd4d1448a5540ff", "bowl-0ab274b6aefb8ab0", "paring knife-2491d9a022c695ee", "pepper mill-bfda19f3fe2dd99f", "pepper mill lid-4a2edd3b5c496f7a", "tablespoon2-a34ed0373b19c312", "minced garlic jar-85f46278462bdd06", "saucepan-4eb82fdc5e07dcea", "lid of saucepan-aae7fa5ab65189ce", "measuring jug-743b6f9e58fdab13", "sugar container-a1754f76c055c1f1", "sugar container lid-d653df1b6dd3628c", "tablespoon3-c92d1d368b93f83e", "can of something-94b4bbeb5fd47100", "worcestershire sauce bottle-8452f56b46feb4a0", "teaspoon2-c71c546ac4a2553f", "large plastic spoon-7987e29555606842", "worcestershire sauce bottle lid-26b8e908be753b5f", "lid of minced garlic jar-5b6f5163609dbfc1", "muffin tray1-6003a2b3b7d8ae6f", "muffin tray2-f7111da630b333e9", "plate1-357de90612e03533", "plate2-e35aff14357524e7", "plate3-19fd3f43fc135ffd", "teaspoon3-826e32548eff839a", "tablespoon4-f841e6911011217f", "bread knife-fbb4458baa763eb1", "butter knife-5c375f1ce479ea96", "spatula-45f1e2690b505215", "pan gripper1-05895977da0731ea", "pan gripper2-297e812676d64723" ], "reasoning": "1. I watched the video, looking for when the user looked at the recipe sheet (recipe sheet-f1cad958e3f385f1) for the final time. I saw them pick it up at 25:54.\n2. I noticed the user looked at the recipe sheet (recipe sheet-f1cad958e3f385f1) until 26:31, when they set it down. After this point, I did not notice the user picking up the recipe sheet (recipe sheet-f1cad958e3f385f1) again.\n3. At 26:31, I watched to see where the user looked next. After a moment, the user turned to face the countertop at 26:32, pausing to look down at it. They remain looking at it until 26:34. Owing to this information, I determined that the countertop is the final answer.", "referenced_object_masks": [ "recipe sheet-f1cad958e3f385f1" ] }, { "key": "P02-20240211-175138:50ea1875b25de654345e9a2ec4181d8459b9bbdd", "video_id": "P02-20240211-175138", "question": "Just before I closed the closet door, what was the basket of spices next to as I picked it up?", "answer_choice_0": "The mushrooms", "answer_choice_1": "A glass", "answer_choice_2": "The food bin", "answer_choice_3": "A bowl", "answer_choice_4": "The stove", "answer_id": 1, "answer": "A glass", "question_type": [ "Egocentric & Spatial Perception", "Action Recognition", "Object Recognition", "Situational Awareness", "State Changes" ], "objects_of_interest": [ "phone-e3c4c3cf4aacbafc", "recipe sheet-f1cad958e3f385f1", "plastic wrapper-1ada86e0171adb72", "pint glass-08273ca31b54ecea", "washing up liquid bottle-c581844420f23634", "cutting board-7d550d8e6d273150", "washing up bowl-45987bf991af86ad", "tupperware-c40f43f40f56f379", "packet of mushrooms1-dd1a59533ca0096a", "packet of mushrooms2-aa346d7d5e41892a", "tomatoes-e576112019072510", "packet of red onions-f5cbe588f24087fb", "baking tray1-4b716abaad27b7a7", "baking tray2-0b081f43b592c0f7", "box of foil roll-70796fde4537a552", "sheet of foil1-3d4b9dfa0a998a34", "plastic packaging of mushroom1-aa73afc1022b2f75", "mushroom1-b7de39c106df7218", "mushroom2-5b96a9ebf3b2e85d", "mushroom3-10c6533bb2a246bd", "mushroom4-ddd4918d43691f83", "mushroom5-e4cbf25c87731613", "mushroom6-ece92bfdf17cb502", "mushroom7-9f0cb68ade84de50", "mushroom8-24cc3c38ee4390b0", "mushroom9-5f3481321e53599d", "mushroom10-f320f5291c0fd1e2", "mushroom11-133563985d9a1d90", "mushroom12-b55dae0a1ec4c15b", "mushroom13-77387330a3e829bb", "mushroom14-4bc2a42075fa5052", "mushroom15-0aeb457828737a29", "mushroom16-a074b128f2857f3a", "mushroom17-3e6e3ebad690017a", "mushroom18-f4d8b8299a4b1fa7", "mushroom19-fcbd68479317b78a", "mushroom20-657591b4dfdff79f", "mushroom21-c5ee43cbfc384730", "towel-8d4b397557064586", "large mixing bowl-b2a18fcfa935ccc2", "cayennne pepper spice jar-ece5eeec002ee167", "cumin spice jar-53f87bf27776c074", "paprika spice jar-83c73927d4330f44", "garam masala spice jar-c7f15589b1219053", "ginger spice jar-4b58e369a7018d9d", "basket of spices-3ea1aac946265309", "ginger powder packet-f0c060929e848ec6", "coriander packet-2730f85fc6ce667a", "clip-fcbb34fcb84208b9", "teaspoon-b67daef958db7592", "cumin spice jar lid-9c893bf858830c9e", "paprika spice jar lid-a396e6d7f772d8f6", "cayenne pepper spice jar-a400e8a00b25d418", "cayenne pepper spice jar lid-69020437dd0b21fa", "salt bottle-6b7c4d0fcac0f02b", "olive oil bottle-4443d2de7a2f3d1d", "tablespoon-e10d8cc98abebd20", "olive oil bottle lid-d649041ef00753e3", "serrated knife-55dbeabd977bc7b1", "bag of lemons-e72f4acb16b83a07", "lemon-e59acc2de411a652", "chef's knife-73111e3b9f93ddcf", "lemon half-a165dd867eef32d0", "food bin-2d940b254a258670", "sheet of foil2-1c47789129d95778", "red onion1-2baf306bd300c710", "red onion2-d163ea9e77bba3b5", "kitchen roll holder-16717aa38d9c6761", "kitchen towel sheet-4efc7069a149c8fb", "red onion1 half-d39fbc223c29f506", "red onion2 half-8cd4d1448a5540ff", "bowl-0ab274b6aefb8ab0", "paring knife-2491d9a022c695ee", "pepper mill-bfda19f3fe2dd99f", "pepper mill lid-4a2edd3b5c496f7a", "tablespoon2-a34ed0373b19c312", "minced garlic jar-85f46278462bdd06", "saucepan-4eb82fdc5e07dcea", "lid of saucepan-aae7fa5ab65189ce", "measuring jug-743b6f9e58fdab13", "sugar container-a1754f76c055c1f1", "sugar container lid-d653df1b6dd3628c", "tablespoon3-c92d1d368b93f83e", "can of something-94b4bbeb5fd47100", "worcestershire sauce bottle-8452f56b46feb4a0", "teaspoon2-c71c546ac4a2553f", "large plastic spoon-7987e29555606842", "worcestershire sauce bottle lid-26b8e908be753b5f", "lid of minced garlic jar-5b6f5163609dbfc1", "muffin tray1-6003a2b3b7d8ae6f", "muffin tray2-f7111da630b333e9", "plate1-357de90612e03533", "plate2-e35aff14357524e7", "plate3-19fd3f43fc135ffd", "teaspoon3-826e32548eff839a", "tablespoon4-f841e6911011217f", "bread knife-fbb4458baa763eb1", "butter knife-5c375f1ce479ea96", "spatula-45f1e2690b505215", "pan gripper1-05895977da0731ea", "pan gripper2-297e812676d64723" ], "reasoning": "1. I watched the video and looked for when the user closed the closet door . I found that the user did this at 16:03.\n2. Then, I looked back at the video, searching for when the user picked up the basket of spices (basket of spices-3ea1aac946265309) just before closing the closet door at 16:03. I found this moment at 15:58, when they picked it up from its place on the counter.\n3. In order to answer the question, I looked to see what was around the basket of spices (basket of spices-3ea1aac946265309) at 15:58. On the counter, I noticed it was adjacent to a pint glass (pint glass-08273ca31b54ecea). No other object was closer, making a pint glass (pint glass-08273ca31b54ecea) the answer.", "referenced_object_masks": [ "basket of spices-3ea1aac946265309", "pint glass-08273ca31b54ecea" ] }, { "key": "P02-20240211-175138:526e61cdc06a6b193ef780a1dc8f9822b6bcda8d", "video_id": "P02-20240211-175138", "question": "How many tomatoes do I cut before moving a bowl holding sliced onions and grape tomatoes closer to me?", "answer_choice_0": "2", "answer_choice_1": "7", "answer_choice_2": "4", "answer_choice_3": "6", "answer_choice_4": "5", "answer_id": 2, "answer": "4", "question_type": [ "Counting", "Action Recognition", "Egocentric & Spatial Perception", "Object Recognition" ], "objects_of_interest": [ "phone-e3c4c3cf4aacbafc", "recipe sheet-f1cad958e3f385f1", "plastic wrapper-1ada86e0171adb72", "pint glass-08273ca31b54ecea", "washing up liquid bottle-c581844420f23634", "cutting board-7d550d8e6d273150", "washing up bowl-45987bf991af86ad", "tupperware-c40f43f40f56f379", "packet of mushrooms1-dd1a59533ca0096a", "packet of mushrooms2-aa346d7d5e41892a", "tomatoes-e576112019072510", "packet of red onions-f5cbe588f24087fb", "baking tray1-4b716abaad27b7a7", "baking tray2-0b081f43b592c0f7", "box of foil roll-70796fde4537a552", "sheet of foil1-3d4b9dfa0a998a34", "plastic packaging of mushroom1-aa73afc1022b2f75", "mushroom1-b7de39c106df7218", "mushroom2-5b96a9ebf3b2e85d", "mushroom3-10c6533bb2a246bd", "mushroom4-ddd4918d43691f83", "mushroom5-e4cbf25c87731613", "mushroom6-ece92bfdf17cb502", "mushroom7-9f0cb68ade84de50", "mushroom8-24cc3c38ee4390b0", "mushroom9-5f3481321e53599d", "mushroom10-f320f5291c0fd1e2", "mushroom11-133563985d9a1d90", "mushroom12-b55dae0a1ec4c15b", "mushroom13-77387330a3e829bb", "mushroom14-4bc2a42075fa5052", "mushroom15-0aeb457828737a29", "mushroom16-a074b128f2857f3a", "mushroom17-3e6e3ebad690017a", "mushroom18-f4d8b8299a4b1fa7", "mushroom19-fcbd68479317b78a", "mushroom20-657591b4dfdff79f", "mushroom21-c5ee43cbfc384730", "towel-8d4b397557064586", "large mixing bowl-b2a18fcfa935ccc2", "cayennne pepper spice jar-ece5eeec002ee167", "cumin spice jar-53f87bf27776c074", "paprika spice jar-83c73927d4330f44", "garam masala spice jar-c7f15589b1219053", "ginger spice jar-4b58e369a7018d9d", "basket of spices-3ea1aac946265309", "ginger powder packet-f0c060929e848ec6", "coriander packet-2730f85fc6ce667a", "clip-fcbb34fcb84208b9", "teaspoon-b67daef958db7592", "cumin spice jar lid-9c893bf858830c9e", "paprika spice jar lid-a396e6d7f772d8f6", "cayenne pepper spice jar-a400e8a00b25d418", "cayenne pepper spice jar lid-69020437dd0b21fa", "salt bottle-6b7c4d0fcac0f02b", "olive oil bottle-4443d2de7a2f3d1d", "tablespoon-e10d8cc98abebd20", "olive oil bottle lid-d649041ef00753e3", "serrated knife-55dbeabd977bc7b1", "bag of lemons-e72f4acb16b83a07", "lemon-e59acc2de411a652", "chef's knife-73111e3b9f93ddcf", "lemon half-a165dd867eef32d0", "food bin-2d940b254a258670", "sheet of foil2-1c47789129d95778", "red onion1-2baf306bd300c710", "red onion2-d163ea9e77bba3b5", "kitchen roll holder-16717aa38d9c6761", "kitchen towel sheet-4efc7069a149c8fb", "red onion1 half-d39fbc223c29f506", "red onion2 half-8cd4d1448a5540ff", "bowl-0ab274b6aefb8ab0", "paring knife-2491d9a022c695ee", "pepper mill-bfda19f3fe2dd99f", "pepper mill lid-4a2edd3b5c496f7a", "tablespoon2-a34ed0373b19c312", "minced garlic jar-85f46278462bdd06", "saucepan-4eb82fdc5e07dcea", "lid of saucepan-aae7fa5ab65189ce", "measuring jug-743b6f9e58fdab13", "sugar container-a1754f76c055c1f1", "sugar container lid-d653df1b6dd3628c", "tablespoon3-c92d1d368b93f83e", "can of something-94b4bbeb5fd47100", "worcestershire sauce bottle-8452f56b46feb4a0", "teaspoon2-c71c546ac4a2553f", "large plastic spoon-7987e29555606842", "worcestershire sauce bottle lid-26b8e908be753b5f", "lid of minced garlic jar-5b6f5163609dbfc1", "muffin tray1-6003a2b3b7d8ae6f", "muffin tray2-f7111da630b333e9", "plate1-357de90612e03533", "plate2-e35aff14357524e7", "plate3-19fd3f43fc135ffd", "teaspoon3-826e32548eff839a", "tablespoon4-f841e6911011217f", "bread knife-fbb4458baa763eb1", "butter knife-5c375f1ce479ea96", "spatula-45f1e2690b505215", "pan gripper1-05895977da0731ea", "pan gripper2-297e812676d64723" ], "reasoning": "1. I watched the video, looking for when the user began to cut tomatoes (tomatoes-e576112019072510). I found this moment at 20:01, when they had the tomatoes (tomatoes-e576112019072510) on a cutting board (cutting board-7d550d8e6d273150).\n2. Then, I watched to see how many tomatoes (tomatoes-e576112019072510) they cut before moving a bowl (bowl-0ab274b6aefb8ab0) which contains sliced onions . At this point, at 20:01, I noticed the bowl (bowl-0ab274b6aefb8ab0) to the left of the user on the counter.\n3. The user finished cutting individual tomatoes (tomatoes-e576112019072510) at 20:10, 20:21, 20:46, and 20:59. \n4. Then, as the user placed a cut tomato (tomatoes-e576112019072510) into the bowl with the others and the sliced onions , I saw that they moved the bowl (bowl-0ab274b6aefb8ab0) closer to them. Since 4 were cut, I determined that was the final answer.", "referenced_object_masks": [ "bowl-0ab274b6aefb8ab0", "cutting board-7d550d8e6d273150", "tomatoes-e576112019072510" ] }, { "key": "P02-20240211-175138:ab05be45b8c333c9882820f25bfcb4595aa1b79f", "video_id": "P02-20240211-175138", "question": "After I held the paper recipe in my hand for the third time, what is the first item I touched?", "answer_choice_0": "A salt bottle", "answer_choice_1": "A clip", "answer_choice_2": "A towel", "answer_choice_3": "A baking tray", "answer_choice_4": "A paprika spice jar", "answer_id": 4, "answer": "A paprika spice jar", "question_type": [ "Action Recognition", "Counting", "Object Recognition" ], "objects_of_interest": [ "phone-e3c4c3cf4aacbafc", "recipe sheet-f1cad958e3f385f1", "plastic wrapper-1ada86e0171adb72", "pint glass-08273ca31b54ecea", "washing up liquid bottle-c581844420f23634", "cutting board-7d550d8e6d273150", "washing up bowl-45987bf991af86ad", "tupperware-c40f43f40f56f379", "packet of mushrooms1-dd1a59533ca0096a", "packet of mushrooms2-aa346d7d5e41892a", "tomatoes-e576112019072510", "packet of red onions-f5cbe588f24087fb", "baking tray1-4b716abaad27b7a7", "baking tray2-0b081f43b592c0f7", "box of foil roll-70796fde4537a552", "sheet of foil1-3d4b9dfa0a998a34", "plastic packaging of mushroom1-aa73afc1022b2f75", "mushroom1-b7de39c106df7218", "mushroom2-5b96a9ebf3b2e85d", "mushroom3-10c6533bb2a246bd", "mushroom4-ddd4918d43691f83", "mushroom5-e4cbf25c87731613", "mushroom6-ece92bfdf17cb502", "mushroom7-9f0cb68ade84de50", "mushroom8-24cc3c38ee4390b0", "mushroom9-5f3481321e53599d", "mushroom10-f320f5291c0fd1e2", "mushroom11-133563985d9a1d90", "mushroom12-b55dae0a1ec4c15b", "mushroom13-77387330a3e829bb", "mushroom14-4bc2a42075fa5052", "mushroom15-0aeb457828737a29", "mushroom16-a074b128f2857f3a", "mushroom17-3e6e3ebad690017a", "mushroom18-f4d8b8299a4b1fa7", "mushroom19-fcbd68479317b78a", "mushroom20-657591b4dfdff79f", "mushroom21-c5ee43cbfc384730", "towel-8d4b397557064586", "large mixing bowl-b2a18fcfa935ccc2", "cayennne pepper spice jar-ece5eeec002ee167", "cumin spice jar-53f87bf27776c074", "paprika spice jar-83c73927d4330f44", "garam masala spice jar-c7f15589b1219053", "ginger spice jar-4b58e369a7018d9d", "basket of spices-3ea1aac946265309", "ginger powder packet-f0c060929e848ec6", "coriander packet-2730f85fc6ce667a", "clip-fcbb34fcb84208b9", "teaspoon-b67daef958db7592", "cumin spice jar lid-9c893bf858830c9e", "paprika spice jar lid-a396e6d7f772d8f6", "cayenne pepper spice jar-a400e8a00b25d418", "cayenne pepper spice jar lid-69020437dd0b21fa", "salt bottle-6b7c4d0fcac0f02b", "olive oil bottle-4443d2de7a2f3d1d", "tablespoon-e10d8cc98abebd20", "olive oil bottle lid-d649041ef00753e3", "serrated knife-55dbeabd977bc7b1", "bag of lemons-e72f4acb16b83a07", "lemon-e59acc2de411a652", "chef's knife-73111e3b9f93ddcf", "lemon half-a165dd867eef32d0", "food bin-2d940b254a258670", "sheet of foil2-1c47789129d95778", "red onion1-2baf306bd300c710", "red onion2-d163ea9e77bba3b5", "kitchen roll holder-16717aa38d9c6761", "kitchen towel sheet-4efc7069a149c8fb", "red onion1 half-d39fbc223c29f506", "red onion2 half-8cd4d1448a5540ff", "bowl-0ab274b6aefb8ab0", "paring knife-2491d9a022c695ee", "pepper mill-bfda19f3fe2dd99f", "pepper mill lid-4a2edd3b5c496f7a", "tablespoon2-a34ed0373b19c312", "minced garlic jar-85f46278462bdd06", "saucepan-4eb82fdc5e07dcea", "lid of saucepan-aae7fa5ab65189ce", "measuring jug-743b6f9e58fdab13", "sugar container-a1754f76c055c1f1", "sugar container lid-d653df1b6dd3628c", "tablespoon3-c92d1d368b93f83e", "can of something-94b4bbeb5fd47100", "worcestershire sauce bottle-8452f56b46feb4a0", "teaspoon2-c71c546ac4a2553f", "large plastic spoon-7987e29555606842", "worcestershire sauce bottle lid-26b8e908be753b5f", "lid of minced garlic jar-5b6f5163609dbfc1", "muffin tray1-6003a2b3b7d8ae6f", "muffin tray2-f7111da630b333e9", "plate1-357de90612e03533", "plate2-e35aff14357524e7", "plate3-19fd3f43fc135ffd", "teaspoon3-826e32548eff839a", "tablespoon4-f841e6911011217f", "bread knife-fbb4458baa763eb1", "butter knife-5c375f1ce479ea96", "spatula-45f1e2690b505215", "pan gripper1-05895977da0731ea", "pan gripper2-297e812676d64723" ], "reasoning": "1. I watched the video and looked for the times when the user held the recipe sheet (recipe sheet-f1cad958e3f385f1). Right after the video I began, I noticed the user holding the recipe sheet (recipe sheet-f1cad958e3f385f1) in their left hand at 00:06. I counted this as the first instance.\n2. I noticed the second time the user held the recipe sheet (recipe sheet-f1cad958e3f385f1) in their hand at 01:33, when they picked it up to look at it.\n3. Continuing to watch, I found the third instance of the user holding the recipe sheet (recipe sheet-f1cad958e3f385f1) at 06:03.\n4. Since this was the third instance at 06:03, I then watched to see what item they picked up next. At 06:10, I saw them reach into the back of the open closet. Then, for a brief moment, they pick up the paprika spice jar (paprika spice jar-83c73927d4330f44) at 06:11. I determined that was the final answer.", "referenced_object_masks": [ "recipe sheet-f1cad958e3f385f1", "paprika spice jar-83c73927d4330f44" ] }, { "key": "P02-20240211-175138:c9ffb3567cf13f94e920333d5f51975a74533c01", "video_id": "P02-20240211-175138", "question": "If I had taken 1 minute and 30 seconds longer in between juicing the two lemon halves, how long would it have taken me from starting to juice the first and finishing the second?", "answer_choice_0": "13 minutes and 58 seconds", "answer_choice_1": "15 minutes and 25 seconds", "answer_choice_2": "16 minutes and 43 seconds", "answer_choice_3": "14 minutes and 18 seconds", "answer_choice_4": "17 minutes and 2 seconds", "answer_id": 1, "answer": "15 minutes and 25 seconds", "question_type": [ "Action Recognition", "Numerical Reasoning", "Counterfactual", "Counting", "Object Recognition" ], "objects_of_interest": [ "phone-e3c4c3cf4aacbafc", "recipe sheet-f1cad958e3f385f1", "plastic wrapper-1ada86e0171adb72", "pint glass-08273ca31b54ecea", "washing up liquid bottle-c581844420f23634", "cutting board-7d550d8e6d273150", "washing up bowl-45987bf991af86ad", "tupperware-c40f43f40f56f379", "packet of mushrooms1-dd1a59533ca0096a", "packet of mushrooms2-aa346d7d5e41892a", "tomatoes-e576112019072510", "packet of red onions-f5cbe588f24087fb", "baking tray1-4b716abaad27b7a7", "baking tray2-0b081f43b592c0f7", "box of foil roll-70796fde4537a552", "sheet of foil1-3d4b9dfa0a998a34", "plastic packaging of mushroom1-aa73afc1022b2f75", "mushroom1-b7de39c106df7218", "mushroom2-5b96a9ebf3b2e85d", "mushroom3-10c6533bb2a246bd", "mushroom4-ddd4918d43691f83", "mushroom5-e4cbf25c87731613", "mushroom6-ece92bfdf17cb502", "mushroom7-9f0cb68ade84de50", "mushroom8-24cc3c38ee4390b0", "mushroom9-5f3481321e53599d", "mushroom10-f320f5291c0fd1e2", "mushroom11-133563985d9a1d90", "mushroom12-b55dae0a1ec4c15b", "mushroom13-77387330a3e829bb", "mushroom14-4bc2a42075fa5052", "mushroom15-0aeb457828737a29", "mushroom16-a074b128f2857f3a", "mushroom17-3e6e3ebad690017a", "mushroom18-f4d8b8299a4b1fa7", "mushroom19-fcbd68479317b78a", "mushroom20-657591b4dfdff79f", "mushroom21-c5ee43cbfc384730", "towel-8d4b397557064586", "large mixing bowl-b2a18fcfa935ccc2", "cayennne pepper spice jar-ece5eeec002ee167", "cumin spice jar-53f87bf27776c074", "paprika spice jar-83c73927d4330f44", "garam masala spice jar-c7f15589b1219053", "ginger spice jar-4b58e369a7018d9d", "basket of spices-3ea1aac946265309", "ginger powder packet-f0c060929e848ec6", "coriander packet-2730f85fc6ce667a", "clip-fcbb34fcb84208b9", "teaspoon-b67daef958db7592", "cumin spice jar lid-9c893bf858830c9e", "paprika spice jar lid-a396e6d7f772d8f6", "cayenne pepper spice jar-a400e8a00b25d418", "cayenne pepper spice jar lid-69020437dd0b21fa", "salt bottle-6b7c4d0fcac0f02b", "olive oil bottle-4443d2de7a2f3d1d", "tablespoon-e10d8cc98abebd20", "olive oil bottle lid-d649041ef00753e3", "serrated knife-55dbeabd977bc7b1", "bag of lemons-e72f4acb16b83a07", "lemon-e59acc2de411a652", "chef's knife-73111e3b9f93ddcf", "lemon half-a165dd867eef32d0", "food bin-2d940b254a258670", "sheet of foil2-1c47789129d95778", "red onion1-2baf306bd300c710", "red onion2-d163ea9e77bba3b5", "kitchen roll holder-16717aa38d9c6761", "kitchen towel sheet-4efc7069a149c8fb", "red onion1 half-d39fbc223c29f506", "red onion2 half-8cd4d1448a5540ff", "bowl-0ab274b6aefb8ab0", "paring knife-2491d9a022c695ee", "pepper mill-bfda19f3fe2dd99f", "pepper mill lid-4a2edd3b5c496f7a", "tablespoon2-a34ed0373b19c312", "minced garlic jar-85f46278462bdd06", "saucepan-4eb82fdc5e07dcea", "lid of saucepan-aae7fa5ab65189ce", "measuring jug-743b6f9e58fdab13", "sugar container-a1754f76c055c1f1", "sugar container lid-d653df1b6dd3628c", "tablespoon3-c92d1d368b93f83e", "can of something-94b4bbeb5fd47100", "worcestershire sauce bottle-8452f56b46feb4a0", "teaspoon2-c71c546ac4a2553f", "large plastic spoon-7987e29555606842", "worcestershire sauce bottle lid-26b8e908be753b5f", "lid of minced garlic jar-5b6f5163609dbfc1", "muffin tray1-6003a2b3b7d8ae6f", "muffin tray2-f7111da630b333e9", "plate1-357de90612e03533", "plate2-e35aff14357524e7", "plate3-19fd3f43fc135ffd", "teaspoon3-826e32548eff839a", "tablespoon4-f841e6911011217f", "bread knife-fbb4458baa763eb1", "butter knife-5c375f1ce479ea96", "spatula-45f1e2690b505215", "pan gripper1-05895977da0731ea", "pan gripper2-297e812676d64723" ], "reasoning": "1. I watched the video, preparing to figure out how long it took between juicing the two halves of a lemon (lemon-e59acc2de411a652). I found the user cutting the lemon (lemon-e59acc2de411a652) at 09:41.\n2. At 09:49, I noticed the user grab one half of the lemon (lemon half-a165dd867eef32d0) and juice it into the bowl (large mixing bowl-b2a18fcfa935ccc2).\n3. Then, I continued watching, looking for the time when the user juiced the second half of the lemon . For much of the video, the lemon half sits stationary on the countertop. Finally, I noticed the user picked up the lemon half in order to juice it at 23:28. They finish juicing it into the small bowl (bowl-0ab274b6aefb8ab0) at 23:44.\n4. I subtracted 09:49 from 23:44 in order to get a total gap of 13 minutes and 55 seconds.\n5. Then, I added 1 minute and 30 seconds to 13 minutes and 55 seconds, getting a final total of 15 minutes and 25 seconds.", "referenced_object_masks": [ "lemon half-a165dd867eef32d0", "large mixing bowl-b2a18fcfa935ccc2", "bowl-0ab274b6aefb8ab0", "lemon-e59acc2de411a652" ] }, { "key": "P02-20240211-175138:e9e321d8cfa7b60a5b823d907bcf9c889a4ea034", "video_id": "P02-20240211-175138", "question": "When I am first grabbing ingredients from the fridge, which ingredient do I grab from one of the bottom fridge drawers that is deeper inside the drawer than all the others?", "answer_choice_0": "Lemons", "answer_choice_1": "Apples", "answer_choice_2": "Red onions", "answer_choice_3": "Mushrooms", "answer_choice_4": "Cherry tomatoes", "answer_id": 2, "answer": "Red onions", "question_type": [ "Egocentric & Spatial Perception", "Temporal Reasoning", "Counting", "Object Recognition", "Action Recognition", "Situational Awareness" ], "objects_of_interest": [ "phone-e3c4c3cf4aacbafc", "recipe sheet-f1cad958e3f385f1", "plastic wrapper-1ada86e0171adb72", "pint glass-08273ca31b54ecea", "washing up liquid bottle-c581844420f23634", "cutting board-7d550d8e6d273150", "washing up bowl-45987bf991af86ad", "tupperware-c40f43f40f56f379", "packet of mushrooms1-dd1a59533ca0096a", "packet of mushrooms2-aa346d7d5e41892a", "tomatoes-e576112019072510", "packet of red onions-f5cbe588f24087fb", "baking tray1-4b716abaad27b7a7", "baking tray2-0b081f43b592c0f7", "box of foil roll-70796fde4537a552", "sheet of foil1-3d4b9dfa0a998a34", "plastic packaging of mushroom1-aa73afc1022b2f75", "mushroom1-b7de39c106df7218", "mushroom2-5b96a9ebf3b2e85d", "mushroom3-10c6533bb2a246bd", "mushroom4-ddd4918d43691f83", "mushroom5-e4cbf25c87731613", "mushroom6-ece92bfdf17cb502", "mushroom7-9f0cb68ade84de50", "mushroom8-24cc3c38ee4390b0", "mushroom9-5f3481321e53599d", "mushroom10-f320f5291c0fd1e2", "mushroom11-133563985d9a1d90", "mushroom12-b55dae0a1ec4c15b", "mushroom13-77387330a3e829bb", "mushroom14-4bc2a42075fa5052", "mushroom15-0aeb457828737a29", "mushroom16-a074b128f2857f3a", "mushroom17-3e6e3ebad690017a", "mushroom18-f4d8b8299a4b1fa7", "mushroom19-fcbd68479317b78a", "mushroom20-657591b4dfdff79f", "mushroom21-c5ee43cbfc384730", "towel-8d4b397557064586", "large mixing bowl-b2a18fcfa935ccc2", "cayennne pepper spice jar-ece5eeec002ee167", "cumin spice jar-53f87bf27776c074", "paprika spice jar-83c73927d4330f44", "garam masala spice jar-c7f15589b1219053", "ginger spice jar-4b58e369a7018d9d", "basket of spices-3ea1aac946265309", "ginger powder packet-f0c060929e848ec6", "coriander packet-2730f85fc6ce667a", "clip-fcbb34fcb84208b9", "teaspoon-b67daef958db7592", "cumin spice jar lid-9c893bf858830c9e", "paprika spice jar lid-a396e6d7f772d8f6", "cayenne pepper spice jar-a400e8a00b25d418", "cayenne pepper spice jar lid-69020437dd0b21fa", "salt bottle-6b7c4d0fcac0f02b", "olive oil bottle-4443d2de7a2f3d1d", "tablespoon-e10d8cc98abebd20", "olive oil bottle lid-d649041ef00753e3", "serrated knife-55dbeabd977bc7b1", "bag of lemons-e72f4acb16b83a07", "lemon-e59acc2de411a652", "chef's knife-73111e3b9f93ddcf", "lemon half-a165dd867eef32d0", "food bin-2d940b254a258670", "sheet of foil2-1c47789129d95778", "red onion1-2baf306bd300c710", "red onion2-d163ea9e77bba3b5", "kitchen roll holder-16717aa38d9c6761", "kitchen towel sheet-4efc7069a149c8fb", "red onion1 half-d39fbc223c29f506", "red onion2 half-8cd4d1448a5540ff", "bowl-0ab274b6aefb8ab0", "paring knife-2491d9a022c695ee", "pepper mill-bfda19f3fe2dd99f", "pepper mill lid-4a2edd3b5c496f7a", "tablespoon2-a34ed0373b19c312", "minced garlic jar-85f46278462bdd06", "saucepan-4eb82fdc5e07dcea", "lid of saucepan-aae7fa5ab65189ce", "measuring jug-743b6f9e58fdab13", "sugar container-a1754f76c055c1f1", "sugar container lid-d653df1b6dd3628c", "tablespoon3-c92d1d368b93f83e", "can of something-94b4bbeb5fd47100", "worcestershire sauce bottle-8452f56b46feb4a0", "teaspoon2-c71c546ac4a2553f", "large plastic spoon-7987e29555606842", "worcestershire sauce bottle lid-26b8e908be753b5f", "lid of minced garlic jar-5b6f5163609dbfc1", "muffin tray1-6003a2b3b7d8ae6f", "muffin tray2-f7111da630b333e9", "plate1-357de90612e03533", "plate2-e35aff14357524e7", "plate3-19fd3f43fc135ffd", "teaspoon3-826e32548eff839a", "tablespoon4-f841e6911011217f", "bread knife-fbb4458baa763eb1", "butter knife-5c375f1ce479ea96", "spatula-45f1e2690b505215", "pan gripper1-05895977da0731ea", "pan gripper2-297e812676d64723" ], "reasoning": "1. I look for the first time the user grabs ingredients from the fridge, and I find the time range from 01:04-01:26.\n2. The first ingredient the user pulls from the fridge is a packet of mushrooms (packet of mushrooms1-dd1a59533ca0096a) at 01:07. The user pulls this ingredient from above the bottom fridge drawers.\n3. The second ingredient the user pulls is another packet of mushrooms (packet of mushrooms2-aa346d7d5e41892a) at 01:10. This ingredient comes from the bottom right fridge drawer.\n4. The third ingredient the user pulls is a packet of cherry tomatoes (tomatoes-e576112019072510) at 01:14. The user pulls this ingredient from the bottom left fridge drawer.\n5. The fourth and final ingredient the user pulls from the fridge is a bag of red onions (packet of red onions-f5cbe588f24087fb) from 01:16-01:18. The user pulls this ingredient from the bottom left fridge drawer as well.\n6. Only 3 of these ingredients come from the bottom fridge drawer, and, of these, the bag of red onions (packet of red onions-f5cbe588f24087fb) is deepest inside, because the user has to reach much further inside the drawer to grab the onions (packet of red onions-f5cbe588f24087fb) than they do for any other ingredient.\n7. The red onions (packet of red onions-f5cbe588f24087fb), therefore, is the answer.", "referenced_object_masks": [ "tomatoes-e576112019072510", "packet of mushrooms2-aa346d7d5e41892a", "packet of red onions-f5cbe588f24087fb", "packet of mushrooms1-dd1a59533ca0096a" ] }, { "key": "P02-20240211-175138:ed113326df92bb73a7e5fa2d84c9610105d4e2b7", "video_id": "P02-20240211-175138", "question": "Just after I squeeze my first lemon half, what is different about the way I throw away the first lemon seed that doesn't apply to the other 2 lemons seeds I throw away?", "answer_choice_0": "The first lemon seed is thrown away with a spoon", "answer_choice_1": "The first lemon seed is thrown into the trash, while the other 2 are thrown into the compost bin", "answer_choice_2": "The lemon seeds are forgotten, and left in the sauce", "answer_choice_3": "The first lemon seed is thrown away with the lemon", "answer_choice_4": "The first lemon seed is thrown into the sink, while the other 2 are thrown in the compost bin", "answer_id": 3, "answer": "The first lemon seed is thrown away with the lemon", "question_type": [ "Counting", "Object Recognition", "Situational Awareness", "Action Recognition" ], "objects_of_interest": [ "phone-e3c4c3cf4aacbafc", "recipe sheet-f1cad958e3f385f1", "plastic wrapper-1ada86e0171adb72", "pint glass-08273ca31b54ecea", "washing up liquid bottle-c581844420f23634", "cutting board-7d550d8e6d273150", "washing up bowl-45987bf991af86ad", "tupperware-c40f43f40f56f379", "packet of mushrooms1-dd1a59533ca0096a", "packet of mushrooms2-aa346d7d5e41892a", "tomatoes-e576112019072510", "packet of red onions-f5cbe588f24087fb", "baking tray1-4b716abaad27b7a7", "baking tray2-0b081f43b592c0f7", "box of foil roll-70796fde4537a552", "sheet of foil1-3d4b9dfa0a998a34", "plastic packaging of mushroom1-aa73afc1022b2f75", "mushroom1-b7de39c106df7218", "mushroom2-5b96a9ebf3b2e85d", "mushroom3-10c6533bb2a246bd", "mushroom4-ddd4918d43691f83", "mushroom5-e4cbf25c87731613", "mushroom6-ece92bfdf17cb502", "mushroom7-9f0cb68ade84de50", "mushroom8-24cc3c38ee4390b0", "mushroom9-5f3481321e53599d", "mushroom10-f320f5291c0fd1e2", "mushroom11-133563985d9a1d90", "mushroom12-b55dae0a1ec4c15b", "mushroom13-77387330a3e829bb", "mushroom14-4bc2a42075fa5052", "mushroom15-0aeb457828737a29", "mushroom16-a074b128f2857f3a", "mushroom17-3e6e3ebad690017a", "mushroom18-f4d8b8299a4b1fa7", "mushroom19-fcbd68479317b78a", "mushroom20-657591b4dfdff79f", "mushroom21-c5ee43cbfc384730", "towel-8d4b397557064586", "large mixing bowl-b2a18fcfa935ccc2", "cayennne pepper spice jar-ece5eeec002ee167", "cumin spice jar-53f87bf27776c074", "paprika spice jar-83c73927d4330f44", "garam masala spice jar-c7f15589b1219053", "ginger spice jar-4b58e369a7018d9d", "basket of spices-3ea1aac946265309", "ginger powder packet-f0c060929e848ec6", "coriander packet-2730f85fc6ce667a", "clip-fcbb34fcb84208b9", "teaspoon-b67daef958db7592", "cumin spice jar lid-9c893bf858830c9e", "paprika spice jar lid-a396e6d7f772d8f6", "cayenne pepper spice jar-a400e8a00b25d418", "cayenne pepper spice jar lid-69020437dd0b21fa", "salt bottle-6b7c4d0fcac0f02b", "olive oil bottle-4443d2de7a2f3d1d", "tablespoon-e10d8cc98abebd20", "olive oil bottle lid-d649041ef00753e3", "serrated knife-55dbeabd977bc7b1", "bag of lemons-e72f4acb16b83a07", "lemon-e59acc2de411a652", "chef's knife-73111e3b9f93ddcf", "lemon half-a165dd867eef32d0", "food bin-2d940b254a258670", "sheet of foil2-1c47789129d95778", "red onion1-2baf306bd300c710", "red onion2-d163ea9e77bba3b5", "kitchen roll holder-16717aa38d9c6761", "kitchen towel sheet-4efc7069a149c8fb", "red onion1 half-d39fbc223c29f506", "red onion2 half-8cd4d1448a5540ff", "bowl-0ab274b6aefb8ab0", "paring knife-2491d9a022c695ee", "pepper mill-bfda19f3fe2dd99f", "pepper mill lid-4a2edd3b5c496f7a", "tablespoon2-a34ed0373b19c312", "minced garlic jar-85f46278462bdd06", "saucepan-4eb82fdc5e07dcea", "lid of saucepan-aae7fa5ab65189ce", "measuring jug-743b6f9e58fdab13", "sugar container-a1754f76c055c1f1", "sugar container lid-d653df1b6dd3628c", "tablespoon3-c92d1d368b93f83e", "can of something-94b4bbeb5fd47100", "worcestershire sauce bottle-8452f56b46feb4a0", "teaspoon2-c71c546ac4a2553f", "large plastic spoon-7987e29555606842", "worcestershire sauce bottle lid-26b8e908be753b5f", "lid of minced garlic jar-5b6f5163609dbfc1", "muffin tray1-6003a2b3b7d8ae6f", "muffin tray2-f7111da630b333e9", "plate1-357de90612e03533", "plate2-e35aff14357524e7", "plate3-19fd3f43fc135ffd", "teaspoon3-826e32548eff839a", "tablespoon4-f841e6911011217f", "bread knife-fbb4458baa763eb1", "butter knife-5c375f1ce479ea96", "spatula-45f1e2690b505215", "pan gripper1-05895977da0731ea", "pan gripper2-297e812676d64723" ], "reasoning": "1. I look for the first lemon half (lemon half-a165dd867eef32d0) to be squeezed, and this happens from 09:49-10:11. Shortly after this, from 10:22-10:45, the user picks lemon seeds out of the sauce and throws them into the compost bin (food bin-2d940b254a258670).\n2. From 10:25-10:27, the user throws away the first lemon seed with the squeezed lemon half (lemon half-a165dd867eef32d0) in their hand.\n3. At 10:35, the user throws the second lemon seed into the bin (food bin-2d940b254a258670) after picking it out with a spoon (teaspoon-b67daef958db7592).\n4. From 10:44-10:45, the user throws the third lemon seed away after picking it out with a spoon (teaspoon-b67daef958db7592) and accidentally missing the bin (food bin-2d940b254a258670).\n5. Based on each of these events, I can determine that the first lemon seed was thrown away with the lemon (lemon half-a165dd867eef32d0), while the other two seeds were thrown away through different means.\n6. Therefore, being thrown away with the lemon (lemon half-a165dd867eef32d0) is the answer.", "referenced_object_masks": [ "teaspoon-b67daef958db7592", "food bin-2d940b254a258670", "lemon half-a165dd867eef32d0" ] }, { "key": "P02-20240211-175138:fe6fdb6474942b0981ac325497b57c6295b95bd0", "video_id": "P02-20240211-175138", "question": "When I pick up the printed recipe for the sixth time and flip its second page for the second time, what object on the counter is the printed page completely obscuring?", "answer_choice_0": "Baguette", "answer_choice_1": "Phone", "answer_choice_2": "Paper towels", "answer_choice_3": "Drinking glass", "answer_choice_4": "Microwave", "answer_id": 3, "answer": "Drinking glass", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Counting", "Situational Awareness", "Temporal Reasoning", "Action Recognition" ], "objects_of_interest": [ "phone-e3c4c3cf4aacbafc", "recipe sheet-f1cad958e3f385f1", "plastic wrapper-1ada86e0171adb72", "pint glass-08273ca31b54ecea", "washing up liquid bottle-c581844420f23634", "cutting board-7d550d8e6d273150", "washing up bowl-45987bf991af86ad", "tupperware-c40f43f40f56f379", "packet of mushrooms1-dd1a59533ca0096a", "packet of mushrooms2-aa346d7d5e41892a", "tomatoes-e576112019072510", "packet of red onions-f5cbe588f24087fb", "baking tray1-4b716abaad27b7a7", "baking tray2-0b081f43b592c0f7", "box of foil roll-70796fde4537a552", "sheet of foil1-3d4b9dfa0a998a34", "plastic packaging of mushroom1-aa73afc1022b2f75", "mushroom1-b7de39c106df7218", "mushroom2-5b96a9ebf3b2e85d", "mushroom3-10c6533bb2a246bd", "mushroom4-ddd4918d43691f83", "mushroom5-e4cbf25c87731613", "mushroom6-ece92bfdf17cb502", "mushroom7-9f0cb68ade84de50", "mushroom8-24cc3c38ee4390b0", "mushroom9-5f3481321e53599d", "mushroom10-f320f5291c0fd1e2", "mushroom11-133563985d9a1d90", "mushroom12-b55dae0a1ec4c15b", "mushroom13-77387330a3e829bb", "mushroom14-4bc2a42075fa5052", "mushroom15-0aeb457828737a29", "mushroom16-a074b128f2857f3a", "mushroom17-3e6e3ebad690017a", "mushroom18-f4d8b8299a4b1fa7", "mushroom19-fcbd68479317b78a", "mushroom20-657591b4dfdff79f", "mushroom21-c5ee43cbfc384730", "towel-8d4b397557064586", "large mixing bowl-b2a18fcfa935ccc2", "cayennne pepper spice jar-ece5eeec002ee167", "cumin spice jar-53f87bf27776c074", "paprika spice jar-83c73927d4330f44", "garam masala spice jar-c7f15589b1219053", "ginger spice jar-4b58e369a7018d9d", "basket of spices-3ea1aac946265309", "ginger powder packet-f0c060929e848ec6", "coriander packet-2730f85fc6ce667a", "clip-fcbb34fcb84208b9", "teaspoon-b67daef958db7592", "cumin spice jar lid-9c893bf858830c9e", "paprika spice jar lid-a396e6d7f772d8f6", "cayenne pepper spice jar-a400e8a00b25d418", "cayenne pepper spice jar lid-69020437dd0b21fa", "salt bottle-6b7c4d0fcac0f02b", "olive oil bottle-4443d2de7a2f3d1d", "tablespoon-e10d8cc98abebd20", "olive oil bottle lid-d649041ef00753e3", "serrated knife-55dbeabd977bc7b1", "bag of lemons-e72f4acb16b83a07", "lemon-e59acc2de411a652", "chef's knife-73111e3b9f93ddcf", "lemon half-a165dd867eef32d0", "food bin-2d940b254a258670", "sheet of foil2-1c47789129d95778", "red onion1-2baf306bd300c710", "red onion2-d163ea9e77bba3b5", "kitchen roll holder-16717aa38d9c6761", "kitchen towel sheet-4efc7069a149c8fb", "red onion1 half-d39fbc223c29f506", "red onion2 half-8cd4d1448a5540ff", "bowl-0ab274b6aefb8ab0", "paring knife-2491d9a022c695ee", "pepper mill-bfda19f3fe2dd99f", "pepper mill lid-4a2edd3b5c496f7a", "tablespoon2-a34ed0373b19c312", "minced garlic jar-85f46278462bdd06", "saucepan-4eb82fdc5e07dcea", "lid of saucepan-aae7fa5ab65189ce", "measuring jug-743b6f9e58fdab13", "sugar container-a1754f76c055c1f1", "sugar container lid-d653df1b6dd3628c", "tablespoon3-c92d1d368b93f83e", "can of something-94b4bbeb5fd47100", "worcestershire sauce bottle-8452f56b46feb4a0", "teaspoon2-c71c546ac4a2553f", "large plastic spoon-7987e29555606842", "worcestershire sauce bottle lid-26b8e908be753b5f", "lid of minced garlic jar-5b6f5163609dbfc1", "muffin tray1-6003a2b3b7d8ae6f", "muffin tray2-f7111da630b333e9", "plate1-357de90612e03533", "plate2-e35aff14357524e7", "plate3-19fd3f43fc135ffd", "teaspoon3-826e32548eff839a", "tablespoon4-f841e6911011217f", "bread knife-fbb4458baa763eb1", "butter knife-5c375f1ce479ea96", "spatula-45f1e2690b505215", "pan gripper1-05895977da0731ea", "pan gripper2-297e812676d64723" ], "reasoning": "1. First, I look for the sixth time the user picks up the printed recipe (recipe sheet-f1cad958e3f385f1).\n2. The user picks up the recipe (recipe sheet-f1cad958e3f385f1) once at 01:32, a second time at 06:02, a third time at 08:38, a fourth time at 15:10, a fifth time at 24:26, and a sixth time at 25:53.\n3. Second, I keep track of the time the user flips the second page of the recipe (recipe sheet-f1cad958e3f385f1) for the second time.\n4. I notice that the user flips the second page of the recipe (recipe sheet-f1cad958e3f385f1) the first time at 26:05, and the second page the second time at 26:16.\n5. I realize that, in between flips from 26:05-26:16, there is a drinking glass (pint glass-08273ca31b54ecea) on the counter that is obscured when the user flips the second page of the printed recipe.\n6. Therefore, the user completely obscures a drinking glass (pint glass-08273ca31b54ecea).", "referenced_object_masks": [ "recipe sheet-f1cad958e3f385f1", "pint glass-08273ca31b54ecea" ] }, { "key": "P02-20240211-182630:1d9d068ebb1aff2c619ab03fedccec6a994b631c", "video_id": "P02-20240211-182630", "question": "How many times did I tap my left hand when I opened the fridge for the first time?", "answer_choice_0": "3 times", "answer_choice_1": "5 times", "answer_choice_2": "8 times", "answer_choice_3": "10 times", "answer_choice_4": "2 times", "answer_id": 4, "answer": "2 times", "question_type": [ "Counting", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "phone-e24b1dc3f51a753b", "large plastic spoon-d7489765dea3624b", "oven gloves-c0199dca6a532eeb", "left oven glove-83654ece4db6ea9c", "recipe-7069ecc836ba7a3d", "baking trays-1aae1630d4988267", "top baking tray-a21ca94bf45e0915", "measuring cup-c1763bae382b4213", "foil-65bd2ce64cda14ae", "saucepan-07d71b882a62b00e", "spoons-1d752f7cd062870d", "spoon 1-e98e1cb117233a16", "spoon 2-98fbf154e513f782", "spoon 3-6f43bc91a1af0c1c", "plastic punnet-a177934077bcded3", "big black tub-4452b67202c862b3", "washing up liquid bottle-956e6d45a7e894df", "spoon 4-97206e85d3c85fe3", "spoon 5-20e81c97414518d3", "spoon 6-46ff6e311d909978", "spoon 7-e8985b60c52f9e1d", "spoon 8-ffcf04223c71053a", "paring knife-298bdfca1a72810e", "sponge-cf7ec46bf2b6522d", "translucent bowl-1dade0166898a8d5", "serrated knife-5cbc05bd56d18ca0", "second serrated knife-807a35ef0838e438", "food bin-3372d693c5d1b989", "cutting board-1521d57a27c118af", "towel-ed8f73e2902444d4", "bag of sweetcorn-225a5bb4114de27d", "food clip green-a3f383a27310e45a", "saucepan lid-1bb0d937558bc4ff", "first spoon-27433e01720bcfa3", "second spoon-6257ec0e7aff2dc0", "third spoon-d11f9d1240f3bb26", "fourth spoon-377845e870d89fe6", "fifth spoon-e49571e6a91dbebb", "kebab wraps-c14dc6fc7194bde4", "garlic jar-ab8d272990aea2b4", "jam jar-3d5c837e606e1607", "yoghurt bowl-6868cdeda0e3d157", "two kebab wraps-b645b182f6b09aed", "top kebab wrap-57ec8c86dfa3a16f", "oven gloves (pair)-e0e355fe30fdd6df", "cling film-33b697120d3f4662", "tablespoon-dd282967f28f0b6e", "tea towel-51ad33b4fe96333e", "cheese packet-80847502a28daa9a", "packet of shredded cheese-d5f3df8248b13233", "sieves-dc08c3b7e23140c3", "tongs-8c9244d8663d5f13", "weighing scale-03152e8b216c6fee", "cheese grater-ada1279a6eb5aa0b", "bottom kebab wrap-16f3ca33e0c1c779", "plate-43e47e257756ca56", "two plates-fc859072f962bba6", "spatula-2eb3b4ba769639f5", "bottom foil-11633579764d7786", "grey new baking tray-7e0589b0602bf50c", "bowl with onions-f66bb23589923bbf", "left plate-6ee11421a8e28342", "hot sauce bottle-a916f43d792d32a8", "right plate-4d4f6ba4095e77de", "bread knife-6e2d66d8dfd1f26e" ], "reasoning": "1. I watched the video to see when the user opened the fridge for the first time and saw this happen at 07:00.\n2. I observed that the user bends down and sets their left hand on the counter at 07:02 to balance themselves.\n3. After a few moments, the user taps their hand on the counter twice between 07:02-07:03.", "referenced_object_masks": [] }, { "key": "P02-20240211-182630:204c91e2122908799c08959fcd327b6049a7a5bb", "video_id": "P02-20240211-182630", "question": "What is the total duration when I combine the time I spent studying the back of the kebab wraps package for the fifth time and the time I spent staring at the oven glove I put on my left hand after I flipped the wraps over in the oven?", "answer_choice_0": "25 seconds", "answer_choice_1": "15 seconds", "answer_choice_2": "7 seconds", "answer_choice_3": "11 seconds", "answer_choice_4": "29 seconds", "answer_id": 1, "answer": "15 seconds", "question_type": [ "Action Recognition", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "phone-e24b1dc3f51a753b", "large plastic spoon-d7489765dea3624b", "oven gloves-c0199dca6a532eeb", "left oven glove-83654ece4db6ea9c", "recipe-7069ecc836ba7a3d", "baking trays-1aae1630d4988267", "top baking tray-a21ca94bf45e0915", "measuring cup-c1763bae382b4213", "foil-65bd2ce64cda14ae", "saucepan-07d71b882a62b00e", "spoons-1d752f7cd062870d", "spoon 1-e98e1cb117233a16", "spoon 2-98fbf154e513f782", "spoon 3-6f43bc91a1af0c1c", "plastic punnet-a177934077bcded3", "big black tub-4452b67202c862b3", "washing up liquid bottle-956e6d45a7e894df", "spoon 4-97206e85d3c85fe3", "spoon 5-20e81c97414518d3", "spoon 6-46ff6e311d909978", "spoon 7-e8985b60c52f9e1d", "spoon 8-ffcf04223c71053a", "paring knife-298bdfca1a72810e", "sponge-cf7ec46bf2b6522d", "translucent bowl-1dade0166898a8d5", "serrated knife-5cbc05bd56d18ca0", "second serrated knife-807a35ef0838e438", "food bin-3372d693c5d1b989", "cutting board-1521d57a27c118af", "towel-ed8f73e2902444d4", "bag of sweetcorn-225a5bb4114de27d", "food clip green-a3f383a27310e45a", "saucepan lid-1bb0d937558bc4ff", "first spoon-27433e01720bcfa3", "second spoon-6257ec0e7aff2dc0", "third spoon-d11f9d1240f3bb26", "fourth spoon-377845e870d89fe6", "fifth spoon-e49571e6a91dbebb", "kebab wraps-c14dc6fc7194bde4", "garlic jar-ab8d272990aea2b4", "jam jar-3d5c837e606e1607", "yoghurt bowl-6868cdeda0e3d157", "two kebab wraps-b645b182f6b09aed", "top kebab wrap-57ec8c86dfa3a16f", "oven gloves (pair)-e0e355fe30fdd6df", "cling film-33b697120d3f4662", "tablespoon-dd282967f28f0b6e", "tea towel-51ad33b4fe96333e", "cheese packet-80847502a28daa9a", "packet of shredded cheese-d5f3df8248b13233", "sieves-dc08c3b7e23140c3", "tongs-8c9244d8663d5f13", "weighing scale-03152e8b216c6fee", "cheese grater-ada1279a6eb5aa0b", "bottom kebab wrap-16f3ca33e0c1c779", "plate-43e47e257756ca56", "two plates-fc859072f962bba6", "spatula-2eb3b4ba769639f5", "bottom foil-11633579764d7786", "grey new baking tray-7e0589b0602bf50c", "bowl with onions-f66bb23589923bbf", "left plate-6ee11421a8e28342", "hot sauce bottle-a916f43d792d32a8", "right plate-4d4f6ba4095e77de", "bread knife-6e2d66d8dfd1f26e" ], "reasoning": "1. I observed that the user stared at the back of the kebab wraps package (kebab wraps-c14dc6fc7194bde4) for the first time from 07:19-07:22, for the second time from 08:32 to 08:46, for the third time at 08:51, a fourth time at 08:54 to 08:57, and a fifth time from 09:01 to 09:05. The total duration for the fifth time is 4 seconds.\n2. I observed the user at 16:55 flipping over the bottom wrap (bottom kebab wrap-16f3ca33e0c1c779) and the top wrap (top kebab wrap-57ec8c86dfa3a16f). \n3. I continued watching for the user to stare at the oven glove they put on their left hand (left oven glove-83654ece4db6ea9c).The user began staring at this glove (left oven glove-83654ece4db6ea9c) at 17:11 and stopped at 17:22 for a total of 11 seconds.\n4. The total duration of the two actions in question, when added together as 4 plus 11, totals 15 seconds.", "referenced_object_masks": [ "left oven glove-83654ece4db6ea9c", "bottom kebab wrap-16f3ca33e0c1c779", "top kebab wrap-57ec8c86dfa3a16f", "kebab wraps-c14dc6fc7194bde4" ] }, { "key": "P02-20240211-182630:36d467b241f5fd61653fa75988579ccb40709e6b", "video_id": "P02-20240211-182630", "question": "From standing at the kitchen sink, what directions do I need to take to reach my back door?", "answer_choice_0": "Turn right, walk foward, turn left", "answer_choice_1": "Walk forward, turn left, walk forward, turn right", "answer_choice_2": "Turn around, walk forward, turn right, walk forward, turn left, walk foward", "answer_choice_3": "Turn left, walk forward, turn right", "answer_choice_4": "Turn around, walk forward, turn right, walk forward, turn right, walk forward", "answer_id": 4, "answer": "Turn around, walk forward, turn right, walk forward, turn right, walk forward", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Counterfactual", "Situational Awareness" ], "objects_of_interest": [ "phone-e24b1dc3f51a753b", "large plastic spoon-d7489765dea3624b", "oven gloves-c0199dca6a532eeb", "left oven glove-83654ece4db6ea9c", "recipe-7069ecc836ba7a3d", "baking trays-1aae1630d4988267", "top baking tray-a21ca94bf45e0915", "measuring cup-c1763bae382b4213", "foil-65bd2ce64cda14ae", "saucepan-07d71b882a62b00e", "spoons-1d752f7cd062870d", "spoon 1-e98e1cb117233a16", "spoon 2-98fbf154e513f782", "spoon 3-6f43bc91a1af0c1c", "plastic punnet-a177934077bcded3", "big black tub-4452b67202c862b3", "washing up liquid bottle-956e6d45a7e894df", "spoon 4-97206e85d3c85fe3", "spoon 5-20e81c97414518d3", "spoon 6-46ff6e311d909978", "spoon 7-e8985b60c52f9e1d", "spoon 8-ffcf04223c71053a", "paring knife-298bdfca1a72810e", "sponge-cf7ec46bf2b6522d", "translucent bowl-1dade0166898a8d5", "serrated knife-5cbc05bd56d18ca0", "second serrated knife-807a35ef0838e438", "food bin-3372d693c5d1b989", "cutting board-1521d57a27c118af", "towel-ed8f73e2902444d4", "bag of sweetcorn-225a5bb4114de27d", "food clip green-a3f383a27310e45a", "saucepan lid-1bb0d937558bc4ff", "first spoon-27433e01720bcfa3", "second spoon-6257ec0e7aff2dc0", "third spoon-d11f9d1240f3bb26", "fourth spoon-377845e870d89fe6", "fifth spoon-e49571e6a91dbebb", "kebab wraps-c14dc6fc7194bde4", "garlic jar-ab8d272990aea2b4", "jam jar-3d5c837e606e1607", "yoghurt bowl-6868cdeda0e3d157", "two kebab wraps-b645b182f6b09aed", "top kebab wrap-57ec8c86dfa3a16f", "oven gloves (pair)-e0e355fe30fdd6df", "cling film-33b697120d3f4662", "tablespoon-dd282967f28f0b6e", "tea towel-51ad33b4fe96333e", "cheese packet-80847502a28daa9a", "packet of shredded cheese-d5f3df8248b13233", "sieves-dc08c3b7e23140c3", "tongs-8c9244d8663d5f13", "weighing scale-03152e8b216c6fee", "cheese grater-ada1279a6eb5aa0b", "bottom kebab wrap-16f3ca33e0c1c779", "plate-43e47e257756ca56", "two plates-fc859072f962bba6", "spatula-2eb3b4ba769639f5", "bottom foil-11633579764d7786", "grey new baking tray-7e0589b0602bf50c", "bowl with onions-f66bb23589923bbf", "left plate-6ee11421a8e28342", "hot sauce bottle-a916f43d792d32a8", "right plate-4d4f6ba4095e77de", "bread knife-6e2d66d8dfd1f26e" ], "reasoning": "1. I watched the video to establish where the user is when they are standing at the kitchen sink and saw this position at 02:21.\n2. I then watched the video to establish where the back door is and saw it at 05:16.\n3. As seen in the user's movements between 05:07-05:15, the kitchen sink and the back door appear to be right next to each other, but separated by a wall that has a doorway a few steps behind the user when they are at the sink.\n4. Therefore, the user must turn around, walk forward, turn right, walk forward (through the doorway), turn right again (now that they are on the other side of the wall), and walk forward to reach the back door .", "referenced_object_masks": [] }, { "key": "P02-20240211-182630:46c4bbeeaae287d44067691db3f2a93168f0e923", "video_id": "P02-20240211-182630", "question": "Which direction is the loaf of bread located from me when I stoop down to grab one of the oven gloves?", "answer_choice_0": "Behind and to the right", "answer_choice_1": "Directly behind", "answer_choice_2": "To the left", "answer_choice_3": "Directly above", "answer_choice_4": "To the right", "answer_id": 4, "answer": "To the right", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "phone-e24b1dc3f51a753b", "large plastic spoon-d7489765dea3624b", "oven gloves-c0199dca6a532eeb", "left oven glove-83654ece4db6ea9c", "recipe-7069ecc836ba7a3d", "baking trays-1aae1630d4988267", "top baking tray-a21ca94bf45e0915", "measuring cup-c1763bae382b4213", "foil-65bd2ce64cda14ae", "saucepan-07d71b882a62b00e", "spoons-1d752f7cd062870d", "spoon 1-e98e1cb117233a16", "spoon 2-98fbf154e513f782", "spoon 3-6f43bc91a1af0c1c", "plastic punnet-a177934077bcded3", "big black tub-4452b67202c862b3", "washing up liquid bottle-956e6d45a7e894df", "spoon 4-97206e85d3c85fe3", "spoon 5-20e81c97414518d3", "spoon 6-46ff6e311d909978", "spoon 7-e8985b60c52f9e1d", "spoon 8-ffcf04223c71053a", "paring knife-298bdfca1a72810e", "sponge-cf7ec46bf2b6522d", "translucent bowl-1dade0166898a8d5", "serrated knife-5cbc05bd56d18ca0", "second serrated knife-807a35ef0838e438", "food bin-3372d693c5d1b989", "cutting board-1521d57a27c118af", "towel-ed8f73e2902444d4", "bag of sweetcorn-225a5bb4114de27d", "food clip green-a3f383a27310e45a", "saucepan lid-1bb0d937558bc4ff", "first spoon-27433e01720bcfa3", "second spoon-6257ec0e7aff2dc0", "third spoon-d11f9d1240f3bb26", "fourth spoon-377845e870d89fe6", "fifth spoon-e49571e6a91dbebb", "kebab wraps-c14dc6fc7194bde4", "garlic jar-ab8d272990aea2b4", "jam jar-3d5c837e606e1607", "yoghurt bowl-6868cdeda0e3d157", "two kebab wraps-b645b182f6b09aed", "top kebab wrap-57ec8c86dfa3a16f", "oven gloves (pair)-e0e355fe30fdd6df", "cling film-33b697120d3f4662", "tablespoon-dd282967f28f0b6e", "tea towel-51ad33b4fe96333e", "cheese packet-80847502a28daa9a", "packet of shredded cheese-d5f3df8248b13233", "sieves-dc08c3b7e23140c3", "tongs-8c9244d8663d5f13", "weighing scale-03152e8b216c6fee", "cheese grater-ada1279a6eb5aa0b", "bottom kebab wrap-16f3ca33e0c1c779", "plate-43e47e257756ca56", "two plates-fc859072f962bba6", "spatula-2eb3b4ba769639f5", "bottom foil-11633579764d7786", "grey new baking tray-7e0589b0602bf50c", "bowl with onions-f66bb23589923bbf", "left plate-6ee11421a8e28342", "hot sauce bottle-a916f43d792d32a8", "right plate-4d4f6ba4095e77de", "bread knife-6e2d66d8dfd1f26e" ], "reasoning": "1. I identified the loaf of bread on the counter against the door of the microwave at 00:22.\n2. I continued watching and observed that the user stooped down at 00:34 to pick up the left oven glove (left oven glove-83654ece4db6ea9c) at 00:34. The user is facing the corner where the oven is located.\n3. I observed the setting at 00:03, when the area where the user dropped the left oven glove (left oven glove-83654ece4db6ea9c) is clearly visible. Because the user was facing the corner where the oven is located, the loaf of bread would be located to the user's right.", "referenced_object_masks": [ "left oven glove-83654ece4db6ea9c" ] }, { "key": "P02-20240211-182630:66b690ed553bcd990544da8ebe23ba70a7b50378", "video_id": "P02-20240211-182630", "question": "When I picked up the kebab wrap package for the third time, how many times did I move my hands away completely from it and put them back before putting it down again?", "answer_choice_0": "2 times", "answer_choice_1": "5 times", "answer_choice_2": "6 times", "answer_choice_3": "3 times", "answer_choice_4": "4 times", "answer_id": 1, "answer": "5 times", "question_type": [ "Counting", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "phone-e24b1dc3f51a753b", "large plastic spoon-d7489765dea3624b", "oven gloves-c0199dca6a532eeb", "left oven glove-83654ece4db6ea9c", "recipe-7069ecc836ba7a3d", "baking trays-1aae1630d4988267", "top baking tray-a21ca94bf45e0915", "measuring cup-c1763bae382b4213", "foil-65bd2ce64cda14ae", "saucepan-07d71b882a62b00e", "spoons-1d752f7cd062870d", "spoon 1-e98e1cb117233a16", "spoon 2-98fbf154e513f782", "spoon 3-6f43bc91a1af0c1c", "plastic punnet-a177934077bcded3", "big black tub-4452b67202c862b3", "washing up liquid bottle-956e6d45a7e894df", "spoon 4-97206e85d3c85fe3", "spoon 5-20e81c97414518d3", "spoon 6-46ff6e311d909978", "spoon 7-e8985b60c52f9e1d", "spoon 8-ffcf04223c71053a", "paring knife-298bdfca1a72810e", "sponge-cf7ec46bf2b6522d", "translucent bowl-1dade0166898a8d5", "serrated knife-5cbc05bd56d18ca0", "second serrated knife-807a35ef0838e438", "food bin-3372d693c5d1b989", "cutting board-1521d57a27c118af", "towel-ed8f73e2902444d4", "bag of sweetcorn-225a5bb4114de27d", "food clip green-a3f383a27310e45a", "saucepan lid-1bb0d937558bc4ff", "first spoon-27433e01720bcfa3", "second spoon-6257ec0e7aff2dc0", "third spoon-d11f9d1240f3bb26", "fourth spoon-377845e870d89fe6", "fifth spoon-e49571e6a91dbebb", "kebab wraps-c14dc6fc7194bde4", "garlic jar-ab8d272990aea2b4", "jam jar-3d5c837e606e1607", "yoghurt bowl-6868cdeda0e3d157", "two kebab wraps-b645b182f6b09aed", "top kebab wrap-57ec8c86dfa3a16f", "oven gloves (pair)-e0e355fe30fdd6df", "cling film-33b697120d3f4662", "tablespoon-dd282967f28f0b6e", "tea towel-51ad33b4fe96333e", "cheese packet-80847502a28daa9a", "packet of shredded cheese-d5f3df8248b13233", "sieves-dc08c3b7e23140c3", "tongs-8c9244d8663d5f13", "weighing scale-03152e8b216c6fee", "cheese grater-ada1279a6eb5aa0b", "bottom kebab wrap-16f3ca33e0c1c779", "plate-43e47e257756ca56", "two plates-fc859072f962bba6", "spatula-2eb3b4ba769639f5", "bottom foil-11633579764d7786", "grey new baking tray-7e0589b0602bf50c", "bowl with onions-f66bb23589923bbf", "left plate-6ee11421a8e28342", "hot sauce bottle-a916f43d792d32a8", "right plate-4d4f6ba4095e77de", "bread knife-6e2d66d8dfd1f26e" ], "reasoning": "1. I watched the video to find the first time the user picks up the kebab wrap package and saw this happen at 07:06.\n2. I then watched the video to find the second time the user picks up the kebab wrap package and saw this happen at 07:17.\n3. I then watched the video to find the third time the user picks up the kebab wrap package and saw this happen at 08:26.\n4. During the third time, while the user was handling the package , I observed the user move their left hand off of it to scratch their nose or their eye at 08:35, then put their hand back on the package at 08:37.\n5. I then observed that between 08:48-09:00, the user moved their hands on and off the package 3 different times to adjust the settings on the stove .\n6. Finally, I observed the user randomly move their hand from the package then put it back on at 09:02.\n7. Therefore, the user moved their hands away from the package 5 different times.", "referenced_object_masks": [] }, { "key": "P02-20240211-182630:adee0a57abde612f6ac2a6cceb2b225c78455848", "video_id": "P02-20240211-182630", "question": "Which direction are the red tongs located from me when I store away the black tray?", "answer_choice_0": "Behind, to the right", "answer_choice_1": "Directly behind", "answer_choice_2": "To the left, by the microwave", "answer_choice_3": "To the right, on the counter", "answer_choice_4": "Above, on the counter", "answer_id": 1, "answer": "Directly behind", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "phone-e24b1dc3f51a753b", "large plastic spoon-d7489765dea3624b", "oven gloves-c0199dca6a532eeb", "left oven glove-83654ece4db6ea9c", "recipe-7069ecc836ba7a3d", "baking trays-1aae1630d4988267", "top baking tray-a21ca94bf45e0915", "measuring cup-c1763bae382b4213", "foil-65bd2ce64cda14ae", "saucepan-07d71b882a62b00e", "spoons-1d752f7cd062870d", "spoon 1-e98e1cb117233a16", "spoon 2-98fbf154e513f782", "spoon 3-6f43bc91a1af0c1c", "plastic punnet-a177934077bcded3", "big black tub-4452b67202c862b3", "washing up liquid bottle-956e6d45a7e894df", "spoon 4-97206e85d3c85fe3", "spoon 5-20e81c97414518d3", "spoon 6-46ff6e311d909978", "spoon 7-e8985b60c52f9e1d", "spoon 8-ffcf04223c71053a", "paring knife-298bdfca1a72810e", "sponge-cf7ec46bf2b6522d", "translucent bowl-1dade0166898a8d5", "serrated knife-5cbc05bd56d18ca0", "second serrated knife-807a35ef0838e438", "food bin-3372d693c5d1b989", "cutting board-1521d57a27c118af", "towel-ed8f73e2902444d4", "bag of sweetcorn-225a5bb4114de27d", "food clip green-a3f383a27310e45a", "saucepan lid-1bb0d937558bc4ff", "first spoon-27433e01720bcfa3", "second spoon-6257ec0e7aff2dc0", "third spoon-d11f9d1240f3bb26", "fourth spoon-377845e870d89fe6", "fifth spoon-e49571e6a91dbebb", "kebab wraps-c14dc6fc7194bde4", "garlic jar-ab8d272990aea2b4", "jam jar-3d5c837e606e1607", "yoghurt bowl-6868cdeda0e3d157", "two kebab wraps-b645b182f6b09aed", "top kebab wrap-57ec8c86dfa3a16f", "oven gloves (pair)-e0e355fe30fdd6df", "cling film-33b697120d3f4662", "tablespoon-dd282967f28f0b6e", "tea towel-51ad33b4fe96333e", "cheese packet-80847502a28daa9a", "packet of shredded cheese-d5f3df8248b13233", "sieves-dc08c3b7e23140c3", "tongs-8c9244d8663d5f13", "weighing scale-03152e8b216c6fee", "cheese grater-ada1279a6eb5aa0b", "bottom kebab wrap-16f3ca33e0c1c779", "plate-43e47e257756ca56", "two plates-fc859072f962bba6", "spatula-2eb3b4ba769639f5", "bottom foil-11633579764d7786", "grey new baking tray-7e0589b0602bf50c", "bowl with onions-f66bb23589923bbf", "left plate-6ee11421a8e28342", "hot sauce bottle-a916f43d792d32a8", "right plate-4d4f6ba4095e77de", "bread knife-6e2d66d8dfd1f26e" ], "reasoning": "1. I observed the user grabbing the black tray (weighing scale-03152e8b216c6fee) at 16:06 and storing it away in a cabinet at 16:11.\n2. I then identified the red tongs (tongs-8c9244d8663d5f13) when the user grabbed it at 15:40. The user then places the red tongs (tongs-8c9244d8663d5f13) on the corner between the stove and the sink at 15:45, where it remains until 16:47.\n3. Because the cabinet where the user stores the black tray (weighing scale-03152e8b216c6fee) is in the corner directly across from the corner with the oven, the red tongs (tongs-8c9244d8663d5f13) are directly behind the user at 16:11.", "referenced_object_masks": [ "tongs-8c9244d8663d5f13", "weighing scale-03152e8b216c6fee" ] }, { "key": "P02-20240211-182630:ae04a3310958f63d7ae607ade2215e6490b1951d", "video_id": "P02-20240211-182630", "question": "How many times did I stir the first pot on the stove?", "answer_choice_0": "9 times", "answer_choice_1": "23 times", "answer_choice_2": "10 times", "answer_choice_3": "18 times", "answer_choice_4": "15 times", "answer_id": 4, "answer": "15 times", "question_type": [ "Counting", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "phone-e24b1dc3f51a753b", "large plastic spoon-d7489765dea3624b", "oven gloves-c0199dca6a532eeb", "left oven glove-83654ece4db6ea9c", "recipe-7069ecc836ba7a3d", "baking trays-1aae1630d4988267", "top baking tray-a21ca94bf45e0915", "measuring cup-c1763bae382b4213", "foil-65bd2ce64cda14ae", "saucepan-07d71b882a62b00e", "spoons-1d752f7cd062870d", "spoon 1-e98e1cb117233a16", "spoon 2-98fbf154e513f782", "spoon 3-6f43bc91a1af0c1c", "plastic punnet-a177934077bcded3", "big black tub-4452b67202c862b3", "washing up liquid bottle-956e6d45a7e894df", "spoon 4-97206e85d3c85fe3", "spoon 5-20e81c97414518d3", "spoon 6-46ff6e311d909978", "spoon 7-e8985b60c52f9e1d", "spoon 8-ffcf04223c71053a", "paring knife-298bdfca1a72810e", "sponge-cf7ec46bf2b6522d", "translucent bowl-1dade0166898a8d5", "serrated knife-5cbc05bd56d18ca0", "second serrated knife-807a35ef0838e438", "food bin-3372d693c5d1b989", "cutting board-1521d57a27c118af", "towel-ed8f73e2902444d4", "bag of sweetcorn-225a5bb4114de27d", "food clip green-a3f383a27310e45a", "saucepan lid-1bb0d937558bc4ff", "first spoon-27433e01720bcfa3", "second spoon-6257ec0e7aff2dc0", "third spoon-d11f9d1240f3bb26", "fourth spoon-377845e870d89fe6", "fifth spoon-e49571e6a91dbebb", "kebab wraps-c14dc6fc7194bde4", "garlic jar-ab8d272990aea2b4", "jam jar-3d5c837e606e1607", "yoghurt bowl-6868cdeda0e3d157", "two kebab wraps-b645b182f6b09aed", "top kebab wrap-57ec8c86dfa3a16f", "oven gloves (pair)-e0e355fe30fdd6df", "cling film-33b697120d3f4662", "tablespoon-dd282967f28f0b6e", "tea towel-51ad33b4fe96333e", "cheese packet-80847502a28daa9a", "packet of shredded cheese-d5f3df8248b13233", "sieves-dc08c3b7e23140c3", "tongs-8c9244d8663d5f13", "weighing scale-03152e8b216c6fee", "cheese grater-ada1279a6eb5aa0b", "bottom kebab wrap-16f3ca33e0c1c779", "plate-43e47e257756ca56", "two plates-fc859072f962bba6", "spatula-2eb3b4ba769639f5", "bottom foil-11633579764d7786", "grey new baking tray-7e0589b0602bf50c", "bowl with onions-f66bb23589923bbf", "left plate-6ee11421a8e28342", "hot sauce bottle-a916f43d792d32a8", "right plate-4d4f6ba4095e77de", "bread knife-6e2d66d8dfd1f26e" ], "reasoning": "1. I watched the video to establish what was the first pot on the stove and saw this pot (saucepan-07d71b882a62b00e) at 0:05.\n2. Using the starting position of the spoon (large plastic spoon-d7489765dea3624b) as the reference point for completing an entire stir, I observed the user stir the pot (saucepan-07d71b882a62b00e) 5 times between 00:05-00:08.\n3. I then observed the user stir the pot (saucepan-07d71b882a62b00e) 10 times between 01:02-01:07.\n4. Therefore, the user stirred the first pot on the stove 15 times.", "referenced_object_masks": [ "saucepan-07d71b882a62b00e", "large plastic spoon-d7489765dea3624b" ] }, { "key": "P02-20240211-182630:d6ad2afb00094d2404b978438e3a13e7e351b499", "video_id": "P02-20240211-182630", "question": "If I added the number of times I dropped the oven gloves to the number of cycles I completed while stirring the saucepan for the second time, what would be the total value?", "answer_choice_0": "12", "answer_choice_1": "13", "answer_choice_2": "14", "answer_choice_3": "11", "answer_choice_4": "10", "answer_id": 0, "answer": "12", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness", "Counting" ], "objects_of_interest": [ "phone-e24b1dc3f51a753b", "large plastic spoon-d7489765dea3624b", "oven gloves-c0199dca6a532eeb", "left oven glove-83654ece4db6ea9c", "recipe-7069ecc836ba7a3d", "baking trays-1aae1630d4988267", "top baking tray-a21ca94bf45e0915", "measuring cup-c1763bae382b4213", "foil-65bd2ce64cda14ae", "saucepan-07d71b882a62b00e", "spoons-1d752f7cd062870d", "spoon 1-e98e1cb117233a16", "spoon 2-98fbf154e513f782", "spoon 3-6f43bc91a1af0c1c", "plastic punnet-a177934077bcded3", "big black tub-4452b67202c862b3", "washing up liquid bottle-956e6d45a7e894df", "spoon 4-97206e85d3c85fe3", "spoon 5-20e81c97414518d3", "spoon 6-46ff6e311d909978", "spoon 7-e8985b60c52f9e1d", "spoon 8-ffcf04223c71053a", "paring knife-298bdfca1a72810e", "sponge-cf7ec46bf2b6522d", "translucent bowl-1dade0166898a8d5", "serrated knife-5cbc05bd56d18ca0", "second serrated knife-807a35ef0838e438", "food bin-3372d693c5d1b989", "cutting board-1521d57a27c118af", "towel-ed8f73e2902444d4", "bag of sweetcorn-225a5bb4114de27d", "food clip green-a3f383a27310e45a", "saucepan lid-1bb0d937558bc4ff", "first spoon-27433e01720bcfa3", "second spoon-6257ec0e7aff2dc0", "third spoon-d11f9d1240f3bb26", "fourth spoon-377845e870d89fe6", "fifth spoon-e49571e6a91dbebb", "kebab wraps-c14dc6fc7194bde4", "garlic jar-ab8d272990aea2b4", "jam jar-3d5c837e606e1607", "yoghurt bowl-6868cdeda0e3d157", "two kebab wraps-b645b182f6b09aed", "top kebab wrap-57ec8c86dfa3a16f", "oven gloves (pair)-e0e355fe30fdd6df", "cling film-33b697120d3f4662", "tablespoon-dd282967f28f0b6e", "tea towel-51ad33b4fe96333e", "cheese packet-80847502a28daa9a", "packet of shredded cheese-d5f3df8248b13233", "sieves-dc08c3b7e23140c3", "tongs-8c9244d8663d5f13", "weighing scale-03152e8b216c6fee", "cheese grater-ada1279a6eb5aa0b", "bottom kebab wrap-16f3ca33e0c1c779", "plate-43e47e257756ca56", "two plates-fc859072f962bba6", "spatula-2eb3b4ba769639f5", "bottom foil-11633579764d7786", "grey new baking tray-7e0589b0602bf50c", "bowl with onions-f66bb23589923bbf", "left plate-6ee11421a8e28342", "hot sauce bottle-a916f43d792d32a8", "right plate-4d4f6ba4095e77de", "bread knife-6e2d66d8dfd1f26e" ], "reasoning": "1. I observed the user stirring the saucepan (saucepan-07d71b882a62b00e) for the first time at 00:05. The user then stirred the saucepan (saucepan-07d71b882a62b00e) for the second time at 01:02.\n2. I counted each cycle the user made while stirring the saucepan (saucepan-07d71b882a62b00e) with the ladle (large plastic spoon-d7489765dea3624b) and concluded that they made 10 cycles.\n3. I watched the video to find when the user dropped one of the oven gloves (oven gloves-c0199dca6a532eeb). This happens at 00:34 and 08:11 for a total of 2 times.\n4. Adding together these values, 10 and 2, totals 12.", "referenced_object_masks": [ "oven gloves-c0199dca6a532eeb", "saucepan-07d71b882a62b00e", "large plastic spoon-d7489765dea3624b" ] }, { "key": "P02-20240211-182630:e53f5d1f3a47e6d4705080c538590d4902f2488a", "video_id": "P02-20240211-182630", "question": "What topping did I put on one wrap that I didn't put on the other wrap?", "answer_choice_0": "Hot Sauce", "answer_choice_1": "Cream", "answer_choice_2": "Cheese", "answer_choice_3": "Meat", "answer_choice_4": "Onions", "answer_id": 0, "answer": "Hot Sauce", "question_type": [ "Action Recognition", "Object Recognition", "State Changes" ], "objects_of_interest": [ "phone-e24b1dc3f51a753b", "large plastic spoon-d7489765dea3624b", "oven gloves-c0199dca6a532eeb", "left oven glove-83654ece4db6ea9c", "recipe-7069ecc836ba7a3d", "baking trays-1aae1630d4988267", "top baking tray-a21ca94bf45e0915", "measuring cup-c1763bae382b4213", "foil-65bd2ce64cda14ae", "saucepan-07d71b882a62b00e", "spoons-1d752f7cd062870d", "spoon 1-e98e1cb117233a16", "spoon 2-98fbf154e513f782", "spoon 3-6f43bc91a1af0c1c", "plastic punnet-a177934077bcded3", "big black tub-4452b67202c862b3", "washing up liquid bottle-956e6d45a7e894df", "spoon 4-97206e85d3c85fe3", "spoon 5-20e81c97414518d3", "spoon 6-46ff6e311d909978", "spoon 7-e8985b60c52f9e1d", "spoon 8-ffcf04223c71053a", "paring knife-298bdfca1a72810e", "sponge-cf7ec46bf2b6522d", "translucent bowl-1dade0166898a8d5", "serrated knife-5cbc05bd56d18ca0", "second serrated knife-807a35ef0838e438", "food bin-3372d693c5d1b989", "cutting board-1521d57a27c118af", "towel-ed8f73e2902444d4", "bag of sweetcorn-225a5bb4114de27d", "food clip green-a3f383a27310e45a", "saucepan lid-1bb0d937558bc4ff", "first spoon-27433e01720bcfa3", "second spoon-6257ec0e7aff2dc0", "third spoon-d11f9d1240f3bb26", "fourth spoon-377845e870d89fe6", "fifth spoon-e49571e6a91dbebb", "kebab wraps-c14dc6fc7194bde4", "garlic jar-ab8d272990aea2b4", "jam jar-3d5c837e606e1607", "yoghurt bowl-6868cdeda0e3d157", "two kebab wraps-b645b182f6b09aed", "top kebab wrap-57ec8c86dfa3a16f", "oven gloves (pair)-e0e355fe30fdd6df", "cling film-33b697120d3f4662", "tablespoon-dd282967f28f0b6e", "tea towel-51ad33b4fe96333e", "cheese packet-80847502a28daa9a", "packet of shredded cheese-d5f3df8248b13233", "sieves-dc08c3b7e23140c3", "tongs-8c9244d8663d5f13", "weighing scale-03152e8b216c6fee", "cheese grater-ada1279a6eb5aa0b", "bottom kebab wrap-16f3ca33e0c1c779", "plate-43e47e257756ca56", "two plates-fc859072f962bba6", "spatula-2eb3b4ba769639f5", "bottom foil-11633579764d7786", "grey new baking tray-7e0589b0602bf50c", "bowl with onions-f66bb23589923bbf", "left plate-6ee11421a8e28342", "hot sauce bottle-a916f43d792d32a8", "right plate-4d4f6ba4095e77de", "bread knife-6e2d66d8dfd1f26e" ], "reasoning": "1. I observed that the user began placing toppings on the two kebab wraps (two kebab wraps-b645b182f6b09aed) at 19:27.\n2. I watched the user as they place toppings on the left wrap (bottom kebab wrap-16f3ca33e0c1c779) and the right wrap (top kebab wrap-57ec8c86dfa3a16f) from 19:27 to 22:36, which is when the user folds up the left wrap (bottom kebab wrap-16f3ca33e0c1c779).\n3. The user then retrieves the hot sauce (hot sauce bottle-a916f43d792d32a8) at 22:41 and applies it to the right wrap (top kebab wrap-57ec8c86dfa3a16f) at 22:45. The user does not apply the hot sauce (hot sauce bottle-a916f43d792d32a8) to the left wrap (bottom kebab wrap-16f3ca33e0c1c779).", "referenced_object_masks": [ "two kebab wraps-b645b182f6b09aed", "bottom kebab wrap-16f3ca33e0c1c779", "top kebab wrap-57ec8c86dfa3a16f", "hot sauce bottle-a916f43d792d32a8" ] }, { "key": "P02-20240211-182630:f4f0554c86c9226a38956cd5a2a15c3c1627b269", "video_id": "P02-20240211-182630", "question": "After I take a bite from the bowl of onions, what is the fifth item I touch with my right hand?", "answer_choice_0": "Greek yogurt", "answer_choice_1": "Cheese package", "answer_choice_2": "Baking tray", "answer_choice_3": "Fridge door", "answer_choice_4": "Piece of plastic", "answer_id": 2, "answer": "Baking tray", "question_type": [ "Action Recognition", "Counterfactual", "Counting", "Object Recognition" ], "objects_of_interest": [ "phone-e24b1dc3f51a753b", "large plastic spoon-d7489765dea3624b", "oven gloves-c0199dca6a532eeb", "left oven glove-83654ece4db6ea9c", "recipe-7069ecc836ba7a3d", "baking trays-1aae1630d4988267", "top baking tray-a21ca94bf45e0915", "measuring cup-c1763bae382b4213", "foil-65bd2ce64cda14ae", "saucepan-07d71b882a62b00e", "spoons-1d752f7cd062870d", "spoon 1-e98e1cb117233a16", "spoon 2-98fbf154e513f782", "spoon 3-6f43bc91a1af0c1c", "plastic punnet-a177934077bcded3", "big black tub-4452b67202c862b3", "washing up liquid bottle-956e6d45a7e894df", "spoon 4-97206e85d3c85fe3", "spoon 5-20e81c97414518d3", "spoon 6-46ff6e311d909978", "spoon 7-e8985b60c52f9e1d", "spoon 8-ffcf04223c71053a", "paring knife-298bdfca1a72810e", "sponge-cf7ec46bf2b6522d", "translucent bowl-1dade0166898a8d5", "serrated knife-5cbc05bd56d18ca0", "second serrated knife-807a35ef0838e438", "food bin-3372d693c5d1b989", "cutting board-1521d57a27c118af", "towel-ed8f73e2902444d4", "bag of sweetcorn-225a5bb4114de27d", "food clip green-a3f383a27310e45a", "saucepan lid-1bb0d937558bc4ff", "first spoon-27433e01720bcfa3", "second spoon-6257ec0e7aff2dc0", "third spoon-d11f9d1240f3bb26", "fourth spoon-377845e870d89fe6", "fifth spoon-e49571e6a91dbebb", "kebab wraps-c14dc6fc7194bde4", "garlic jar-ab8d272990aea2b4", "jam jar-3d5c837e606e1607", "yoghurt bowl-6868cdeda0e3d157", "two kebab wraps-b645b182f6b09aed", "top kebab wrap-57ec8c86dfa3a16f", "oven gloves (pair)-e0e355fe30fdd6df", "cling film-33b697120d3f4662", "tablespoon-dd282967f28f0b6e", "tea towel-51ad33b4fe96333e", "cheese packet-80847502a28daa9a", "packet of shredded cheese-d5f3df8248b13233", "sieves-dc08c3b7e23140c3", "tongs-8c9244d8663d5f13", "weighing scale-03152e8b216c6fee", "cheese grater-ada1279a6eb5aa0b", "bottom kebab wrap-16f3ca33e0c1c779", "plate-43e47e257756ca56", "two plates-fc859072f962bba6", "spatula-2eb3b4ba769639f5", "bottom foil-11633579764d7786", "grey new baking tray-7e0589b0602bf50c", "bowl with onions-f66bb23589923bbf", "left plate-6ee11421a8e28342", "hot sauce bottle-a916f43d792d32a8", "right plate-4d4f6ba4095e77de", "bread knife-6e2d66d8dfd1f26e" ], "reasoning": "1. I watched the video to observe the user taking a bite from the bowl of onions (bowl with onions-f66bb23589923bbf) and saw this happen between 09:56-09:57.\n2. After this, the first object the user touched with their right hand was a piece of plastic , seen at 10:01.\n3. I then observed the user touch and open the fridge door at 10:12.\n4. I then observed the user touch the cheese package (packet of shredded cheese-d5f3df8248b13233) at 10:20.\n5. I then observed the user touching the piece of plastic again at 10:35 before throwing it in the garbage can .\n6. Finally, the user touched the baking tray (top baking tray-a21ca94bf45e0915) at 10:43.", "referenced_object_masks": [ "packet of shredded cheese-d5f3df8248b13233", "bowl with onions-f66bb23589923bbf", "top baking tray-a21ca94bf45e0915" ] }, { "key": "P03-20240216-084005:0275dc0686a6d7862139fcd0ba73c2fe92443ba7", "video_id": "P03-20240216-084005", "question": "If I had put the chopsticks into the dishwasher before the plastic lid, where would they be in order of things I put into the dishwasher?", "answer_choice_0": "Fourth", "answer_choice_1": "Third", "answer_choice_2": "Fifth", "answer_choice_3": "First", "answer_choice_4": "Second", "answer_id": 4, "answer": "Second", "question_type": [ "Action Recognition", "Counting", "Numerical Reasoning" ], "objects_of_interest": [ "small frying pan-bf53dbc70c79285b", "saucepan-b510651c0612bf5d", "padding-461cc7a60eaa5ec3", "colander-cb7b5ed0ed99d32d", "flask-d103e3fa34ce3a93", "flask lid-3d9a8e68644e336a", "v60 metal filter-0a542e30403deb38", "v60 stand-c29b7f30f12d7607", "ramekin-f0f442d3e88ed600", "plastic spatula-b71c30f541f155f0", "wooden spoon-f7e10de2cc215049", "another wooden chopstick-58e19a9fb337dee3", "knife-e206fb4b84a78638", "spoon-7c2ba9593d3934f7", "metal spoon-0ddab6a88d875136", "another wooden spoon-d5c0f5650ec3ee3c", "tea filter-6f2b023952779611", "teapot lid-b8783677e65c6eb3", "toaster-433988d29ba772e3", "two tea towels-d18eaee2b828cb8a", "tea towel-660fdb6a7b5d40ea", "tea towl2-b2cdf58fb5dfdaba", "bread-0ef9e8df5ca965a8", "food container-23e86bffc4bae3c2", "oat milk-5b94ca05eb43ae92", "plastic bag-3375571c995d2e85", "tote bag-516c2c84d68088c8", "black band-2ec4e7adb0d95ec4", "kitchen roll holder-da491c46559c7852", "kitchen roll-4b65e8391109478c", "fork-559a53e0e7ca0b18", "kitchen towel piece-11ac397aa297f511", "coffee beans bag-eee880237ca44692", "scissors-832aeef234cf8c91", "scissors safety cover-e7319384249bfb5e", "red coffee beans spoon-1d15269ce762ac43", "coffee grinder-b15572680339b915", "top lid of coffee grinder-cab01a3cd845f82a", "clip-4dd05db91a440279", "kettle-35103617f5ebd942", "bag top cut out-1c471bcc7afce213", "another piece of plastic-d7bee3b6c0e27c8c", "coffee grinder bottom compartment-ce6597076e840860", "v60 coffee filter-d076aad17dd70dcf", "coffee grinder container lid-d6dcbfe74626a8f2", "orange cloth-3b3567914224587a", "cloth-37f7fc8cbaf408ef", "black plastic spoon-084190aa32b92b00", "dirty plastic container-e8a8ea43e56872ee", "small grey plate-4681b0699e71e7b1", "red small plate-c8f20bbaabd1f040", "plastic lid-1d326b0817911b9e", "two dirty chopsticks-348593d7653c95e4", "dirty plastic box2-3d57079dd59f11cd", "metal colander-74a100a5fc6b628b", "saucepan2-a75f362ca5bcf045", "dirty bowl-be92755a88aca05a", "another dirty plastic container-b027d28aba2937bc", "large coffee cup-bf1fcaf6910bc1ec", "checkered cup-fbdce38314ed5872", "coffee jug-b0fb10b629299815", "another coffee cup-2b7d0b456a84765d", "another coffee cup2-900fdd129587e4cd", "sponge-4b91bca66ca4e912", "nutella jar-cb2c10a7e933c6e4", "slice of bread-07f9bcc879ea3ac9", "second slice of bread-bc17ebd3b6c15fe1", "two plates-eec89bc8ed72e4a4", "knife2-b8de907c658450a4", "glass-0a6223d2e12ce82e", "v60 coffee filter on coffee jug-d77cf189c8769ff7", "plate2-1067c8e5fe3e25cf", "nutella jar lid-c4d0dd5555f51046", "phone-7ccda200a2da18ea" ], "reasoning": "1. I watched for the user to begin loading items into the dishwasher, which occurs at 7:20.\n2. The first item the user puts into the dishwasher is the red coffee spoon (red coffee beans spoon-1d15269ce762ac43) at 7:20.\n3. The second item the user puts into the dishwasher is the plastic lid (plastic lid-1d326b0817911b9e) at 7:36.\n4. If the chopsticks (two dirty chopsticks-348593d7653c95e4) were put in the dishwasher before the plastic lid (plastic lid-1d326b0817911b9e), they would have been the second item.", "referenced_object_masks": [ "red coffee beans spoon-1d15269ce762ac43", "plastic lid-1d326b0817911b9e", "two dirty chopsticks-348593d7653c95e4" ] }, { "key": "P03-20240216-084005:2003628f8d10d09c971605a82aa529864a6a3d78", "video_id": "P03-20240216-084005", "question": "Rounded to the nearest tenth, if I take the amount of time it took me to spread Nutella on the first piece of toast and multipled it by the amount of time it took me to spread Nutella on the second piece of toast and raised it to the 1/2 power, what number would I get?", "answer_choice_0": "31.3", "answer_choice_1": "32.6", "answer_choice_2": "32.3", "answer_choice_3": "33.3", "answer_choice_4": "31.6", "answer_id": 2, "answer": "32.3", "question_type": [ "Action Recognition", "Event Occurence", "Numerical Reasoning", "Object Recognition", "State Changes", "Temporal Reasoning" ], "objects_of_interest": [ "small frying pan-bf53dbc70c79285b", "saucepan-b510651c0612bf5d", "padding-461cc7a60eaa5ec3", "colander-cb7b5ed0ed99d32d", "flask-d103e3fa34ce3a93", "flask lid-3d9a8e68644e336a", "v60 metal filter-0a542e30403deb38", "v60 stand-c29b7f30f12d7607", "ramekin-f0f442d3e88ed600", "plastic spatula-b71c30f541f155f0", "wooden spoon-f7e10de2cc215049", "another wooden chopstick-58e19a9fb337dee3", "knife-e206fb4b84a78638", "spoon-7c2ba9593d3934f7", "metal spoon-0ddab6a88d875136", "another wooden spoon-d5c0f5650ec3ee3c", "tea filter-6f2b023952779611", "teapot lid-b8783677e65c6eb3", "toaster-433988d29ba772e3", "two tea towels-d18eaee2b828cb8a", "tea towel-660fdb6a7b5d40ea", "tea towl2-b2cdf58fb5dfdaba", "bread-0ef9e8df5ca965a8", "food container-23e86bffc4bae3c2", "oat milk-5b94ca05eb43ae92", "plastic bag-3375571c995d2e85", "tote bag-516c2c84d68088c8", "black band-2ec4e7adb0d95ec4", "kitchen roll holder-da491c46559c7852", "kitchen roll-4b65e8391109478c", "fork-559a53e0e7ca0b18", "kitchen towel piece-11ac397aa297f511", "coffee beans bag-eee880237ca44692", "scissors-832aeef234cf8c91", "scissors safety cover-e7319384249bfb5e", "red coffee beans spoon-1d15269ce762ac43", "coffee grinder-b15572680339b915", "top lid of coffee grinder-cab01a3cd845f82a", "clip-4dd05db91a440279", "kettle-35103617f5ebd942", "bag top cut out-1c471bcc7afce213", "another piece of plastic-d7bee3b6c0e27c8c", "coffee grinder bottom compartment-ce6597076e840860", "v60 coffee filter-d076aad17dd70dcf", "coffee grinder container lid-d6dcbfe74626a8f2", "orange cloth-3b3567914224587a", "cloth-37f7fc8cbaf408ef", "black plastic spoon-084190aa32b92b00", "dirty plastic container-e8a8ea43e56872ee", "small grey plate-4681b0699e71e7b1", "red small plate-c8f20bbaabd1f040", "plastic lid-1d326b0817911b9e", "two dirty chopsticks-348593d7653c95e4", "dirty plastic box2-3d57079dd59f11cd", "metal colander-74a100a5fc6b628b", "saucepan2-a75f362ca5bcf045", "dirty bowl-be92755a88aca05a", "another dirty plastic container-b027d28aba2937bc", "large coffee cup-bf1fcaf6910bc1ec", "checkered cup-fbdce38314ed5872", "coffee jug-b0fb10b629299815", "another coffee cup-2b7d0b456a84765d", "another coffee cup2-900fdd129587e4cd", "sponge-4b91bca66ca4e912", "nutella jar-cb2c10a7e933c6e4", "slice of bread-07f9bcc879ea3ac9", "second slice of bread-bc17ebd3b6c15fe1", "two plates-eec89bc8ed72e4a4", "knife2-b8de907c658450a4", "glass-0a6223d2e12ce82e", "v60 coffee filter on coffee jug-d77cf189c8769ff7", "plate2-1067c8e5fe3e25cf", "nutella jar lid-c4d0dd5555f51046", "phone-7ccda200a2da18ea" ], "reasoning": "1. I watched the whole video and saw the user spreading Nutella (nutella jar-cb2c10a7e933c6e4) on the first piece of toast (slice of bread-07f9bcc879ea3ac9) at 16:00.\n2. The user stopped spreading Nutella (nutella jar-cb2c10a7e933c6e4) on the first piece of toast (slice of bread-07f9bcc879ea3ac9) at 16:29.\n3. The user spent 29 seconds spreading Nutella (nutella jar-cb2c10a7e933c6e4) on the first piece of toast (slice of bread-07f9bcc879ea3ac9).\n4. The user started spreading Nutella (nutella jar-cb2c10a7e933c6e4) on the second piece of toast (second slice of bread-bc17ebd3b6c15fe1) at 16:33 and finished at 17:09.\n5. The user spent 36 seconds spreading Nutella (nutella jar-cb2c10a7e933c6e4) on the second piece of toast (second slice of bread-bc17ebd3b6c15fe1.\n6. 29*36 is 1044.\n7. 1044^(1/2) is the same as taking the square root which is 32.31099.\n8. Rounded to the nearest tenth is 32.3.", "referenced_object_masks": [ "nutella jar-cb2c10a7e933c6e4", "slice of bread-07f9bcc879ea3ac9", "second slice of bread-bc17ebd3b6c15fe1" ] }, { "key": "P03-20240216-084005:45297ef17310274c74792f2bad1c138ea9618849", "video_id": "P03-20240216-084005", "question": "If I take the number of times I pick up the red tea kettle and raise it to the power of the number of items I take out of the refrigerator the first time I open it, what number will I get?", "answer_choice_0": "4096", "answer_choice_1": "49", "answer_choice_2": "64", "answer_choice_3": "343", "answer_choice_4": "512", "answer_id": 4, "answer": "512", "question_type": [ "Action Recognition", "Counting", "Event Occurence", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "small frying pan-bf53dbc70c79285b", "saucepan-b510651c0612bf5d", "padding-461cc7a60eaa5ec3", "colander-cb7b5ed0ed99d32d", "flask-d103e3fa34ce3a93", "flask lid-3d9a8e68644e336a", "v60 metal filter-0a542e30403deb38", "v60 stand-c29b7f30f12d7607", "ramekin-f0f442d3e88ed600", "plastic spatula-b71c30f541f155f0", "wooden spoon-f7e10de2cc215049", "another wooden chopstick-58e19a9fb337dee3", "knife-e206fb4b84a78638", "spoon-7c2ba9593d3934f7", "metal spoon-0ddab6a88d875136", "another wooden spoon-d5c0f5650ec3ee3c", "tea filter-6f2b023952779611", "teapot lid-b8783677e65c6eb3", "toaster-433988d29ba772e3", "two tea towels-d18eaee2b828cb8a", "tea towel-660fdb6a7b5d40ea", "tea towl2-b2cdf58fb5dfdaba", "bread-0ef9e8df5ca965a8", "food container-23e86bffc4bae3c2", "oat milk-5b94ca05eb43ae92", "plastic bag-3375571c995d2e85", "tote bag-516c2c84d68088c8", "black band-2ec4e7adb0d95ec4", "kitchen roll holder-da491c46559c7852", "kitchen roll-4b65e8391109478c", "fork-559a53e0e7ca0b18", "kitchen towel piece-11ac397aa297f511", "coffee beans bag-eee880237ca44692", "scissors-832aeef234cf8c91", "scissors safety cover-e7319384249bfb5e", "red coffee beans spoon-1d15269ce762ac43", "coffee grinder-b15572680339b915", "top lid of coffee grinder-cab01a3cd845f82a", "clip-4dd05db91a440279", "kettle-35103617f5ebd942", "bag top cut out-1c471bcc7afce213", "another piece of plastic-d7bee3b6c0e27c8c", "coffee grinder bottom compartment-ce6597076e840860", "v60 coffee filter-d076aad17dd70dcf", "coffee grinder container lid-d6dcbfe74626a8f2", "orange cloth-3b3567914224587a", "cloth-37f7fc8cbaf408ef", "black plastic spoon-084190aa32b92b00", "dirty plastic container-e8a8ea43e56872ee", "small grey plate-4681b0699e71e7b1", "red small plate-c8f20bbaabd1f040", "plastic lid-1d326b0817911b9e", "two dirty chopsticks-348593d7653c95e4", "dirty plastic box2-3d57079dd59f11cd", "metal colander-74a100a5fc6b628b", "saucepan2-a75f362ca5bcf045", "dirty bowl-be92755a88aca05a", "another dirty plastic container-b027d28aba2937bc", "large coffee cup-bf1fcaf6910bc1ec", "checkered cup-fbdce38314ed5872", "coffee jug-b0fb10b629299815", "another coffee cup-2b7d0b456a84765d", "another coffee cup2-900fdd129587e4cd", "sponge-4b91bca66ca4e912", "nutella jar-cb2c10a7e933c6e4", "slice of bread-07f9bcc879ea3ac9", "second slice of bread-bc17ebd3b6c15fe1", "two plates-eec89bc8ed72e4a4", "knife2-b8de907c658450a4", "glass-0a6223d2e12ce82e", "v60 coffee filter on coffee jug-d77cf189c8769ff7", "plate2-1067c8e5fe3e25cf", "nutella jar lid-c4d0dd5555f51046", "phone-7ccda200a2da18ea" ], "reasoning": "1. I watched the whole video and I saw the user pick up the red tea kettle (kettle-35103617f5ebd942) at 05:36, 09:45, 10:44, 11:21, 12:15, 12:57, 13:36, and 14:18. \n2. At 02:11, the user opened the refrigerator door and took out 3 items.\n3. 8^3 is 512.", "referenced_object_masks": [ "kettle-35103617f5ebd942" ] }, { "key": "P03-20240216-084005:4bc91d05bb2cd27568619de1f673c4014c61ed91", "video_id": "P03-20240216-084005", "question": "Why couldn't I have put the Nutella jar where the tea towel was?", "answer_choice_0": "Because the tea towel was in my pocket", "answer_choice_1": "Because the tea towel was soaking in water", "answer_choice_2": "Because the tea towel was in a drawer", "answer_choice_3": "Because the tea towel was on a hook", "answer_choice_4": "Because the tea towel was in the washing machine", "answer_id": 3, "answer": "Because the tea towel was on a hook", "question_type": [ "Egocentric & Spatial Perception", "Action Recognition", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "small frying pan-bf53dbc70c79285b", "saucepan-b510651c0612bf5d", "padding-461cc7a60eaa5ec3", "colander-cb7b5ed0ed99d32d", "flask-d103e3fa34ce3a93", "flask lid-3d9a8e68644e336a", "v60 metal filter-0a542e30403deb38", "v60 stand-c29b7f30f12d7607", "ramekin-f0f442d3e88ed600", "plastic spatula-b71c30f541f155f0", "wooden spoon-f7e10de2cc215049", "another wooden chopstick-58e19a9fb337dee3", "knife-e206fb4b84a78638", "spoon-7c2ba9593d3934f7", "metal spoon-0ddab6a88d875136", "another wooden spoon-d5c0f5650ec3ee3c", "tea filter-6f2b023952779611", "teapot lid-b8783677e65c6eb3", "toaster-433988d29ba772e3", "two tea towels-d18eaee2b828cb8a", "tea towel-660fdb6a7b5d40ea", "tea towl2-b2cdf58fb5dfdaba", "bread-0ef9e8df5ca965a8", "food container-23e86bffc4bae3c2", "oat milk-5b94ca05eb43ae92", "plastic bag-3375571c995d2e85", "tote bag-516c2c84d68088c8", "black band-2ec4e7adb0d95ec4", "kitchen roll holder-da491c46559c7852", "kitchen roll-4b65e8391109478c", "fork-559a53e0e7ca0b18", "kitchen towel piece-11ac397aa297f511", "coffee beans bag-eee880237ca44692", "scissors-832aeef234cf8c91", "scissors safety cover-e7319384249bfb5e", "red coffee beans spoon-1d15269ce762ac43", "coffee grinder-b15572680339b915", "top lid of coffee grinder-cab01a3cd845f82a", "clip-4dd05db91a440279", "kettle-35103617f5ebd942", "bag top cut out-1c471bcc7afce213", "another piece of plastic-d7bee3b6c0e27c8c", "coffee grinder bottom compartment-ce6597076e840860", "v60 coffee filter-d076aad17dd70dcf", "coffee grinder container lid-d6dcbfe74626a8f2", "orange cloth-3b3567914224587a", "cloth-37f7fc8cbaf408ef", "black plastic spoon-084190aa32b92b00", "dirty plastic container-e8a8ea43e56872ee", "small grey plate-4681b0699e71e7b1", "red small plate-c8f20bbaabd1f040", "plastic lid-1d326b0817911b9e", "two dirty chopsticks-348593d7653c95e4", "dirty plastic box2-3d57079dd59f11cd", "metal colander-74a100a5fc6b628b", "saucepan2-a75f362ca5bcf045", "dirty bowl-be92755a88aca05a", "another dirty plastic container-b027d28aba2937bc", "large coffee cup-bf1fcaf6910bc1ec", "checkered cup-fbdce38314ed5872", "coffee jug-b0fb10b629299815", "another coffee cup-2b7d0b456a84765d", "another coffee cup2-900fdd129587e4cd", "sponge-4b91bca66ca4e912", "nutella jar-cb2c10a7e933c6e4", "slice of bread-07f9bcc879ea3ac9", "second slice of bread-bc17ebd3b6c15fe1", "two plates-eec89bc8ed72e4a4", "knife2-b8de907c658450a4", "glass-0a6223d2e12ce82e", "v60 coffee filter on coffee jug-d77cf189c8769ff7", "plate2-1067c8e5fe3e25cf", "nutella jar lid-c4d0dd5555f51046", "phone-7ccda200a2da18ea" ], "reasoning": "1. I watched for the user to pick up the tea towel , which they do at 9:34.\n2. I looked to see where the user picks up the tea towel from, and it is from a hook on the cabinet below the sink.\n3. I looked for the Nutella jar (nutella jar-cb2c10a7e933c6e4), which the user takes from a cabinet at 11:16, and determined that its size and shape would not have been supported by the hook.", "referenced_object_masks": [ "nutella jar-cb2c10a7e933c6e4" ] }, { "key": "P03-20240216-084005:50df973ef584871761dc98759b3d9095202b968a", "video_id": "P03-20240216-084005", "question": "Where is the kitchen table in relation to me when I am putting bread in the toaster?", "answer_choice_0": "Behind me and to my left", "answer_choice_1": "Directly to my right", "answer_choice_2": "Directly behind me", "answer_choice_3": "Behind me and to my right", "answer_choice_4": "Directly to my left", "answer_id": 1, "answer": "Directly to my right", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Event Occurence", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "small frying pan-bf53dbc70c79285b", "saucepan-b510651c0612bf5d", "padding-461cc7a60eaa5ec3", "colander-cb7b5ed0ed99d32d", "flask-d103e3fa34ce3a93", "flask lid-3d9a8e68644e336a", "v60 metal filter-0a542e30403deb38", "v60 stand-c29b7f30f12d7607", "ramekin-f0f442d3e88ed600", "plastic spatula-b71c30f541f155f0", "wooden spoon-f7e10de2cc215049", "another wooden chopstick-58e19a9fb337dee3", "knife-e206fb4b84a78638", "spoon-7c2ba9593d3934f7", "metal spoon-0ddab6a88d875136", "another wooden spoon-d5c0f5650ec3ee3c", "tea filter-6f2b023952779611", "teapot lid-b8783677e65c6eb3", "toaster-433988d29ba772e3", "two tea towels-d18eaee2b828cb8a", "tea towel-660fdb6a7b5d40ea", "tea towl2-b2cdf58fb5dfdaba", "bread-0ef9e8df5ca965a8", "food container-23e86bffc4bae3c2", "oat milk-5b94ca05eb43ae92", "plastic bag-3375571c995d2e85", "tote bag-516c2c84d68088c8", "black band-2ec4e7adb0d95ec4", "kitchen roll holder-da491c46559c7852", "kitchen roll-4b65e8391109478c", "fork-559a53e0e7ca0b18", "kitchen towel piece-11ac397aa297f511", "coffee beans bag-eee880237ca44692", "scissors-832aeef234cf8c91", "scissors safety cover-e7319384249bfb5e", "red coffee beans spoon-1d15269ce762ac43", "coffee grinder-b15572680339b915", "top lid of coffee grinder-cab01a3cd845f82a", "clip-4dd05db91a440279", "kettle-35103617f5ebd942", "bag top cut out-1c471bcc7afce213", "another piece of plastic-d7bee3b6c0e27c8c", "coffee grinder bottom compartment-ce6597076e840860", "v60 coffee filter-d076aad17dd70dcf", "coffee grinder container lid-d6dcbfe74626a8f2", "orange cloth-3b3567914224587a", "cloth-37f7fc8cbaf408ef", "black plastic spoon-084190aa32b92b00", "dirty plastic container-e8a8ea43e56872ee", "small grey plate-4681b0699e71e7b1", "red small plate-c8f20bbaabd1f040", "plastic lid-1d326b0817911b9e", "two dirty chopsticks-348593d7653c95e4", "dirty plastic box2-3d57079dd59f11cd", "metal colander-74a100a5fc6b628b", "saucepan2-a75f362ca5bcf045", "dirty bowl-be92755a88aca05a", "another dirty plastic container-b027d28aba2937bc", "large coffee cup-bf1fcaf6910bc1ec", "checkered cup-fbdce38314ed5872", "coffee jug-b0fb10b629299815", "another coffee cup-2b7d0b456a84765d", "another coffee cup2-900fdd129587e4cd", "sponge-4b91bca66ca4e912", "nutella jar-cb2c10a7e933c6e4", "slice of bread-07f9bcc879ea3ac9", "second slice of bread-bc17ebd3b6c15fe1", "two plates-eec89bc8ed72e4a4", "knife2-b8de907c658450a4", "glass-0a6223d2e12ce82e", "v60 coffee filter on coffee jug-d77cf189c8769ff7", "plate2-1067c8e5fe3e25cf", "nutella jar lid-c4d0dd5555f51046", "phone-7ccda200a2da18ea" ], "reasoning": "1. I watched the whole video and starting at 11:59, I saw the user put bread (slice of bread-07f9bcc879ea3ac9, second slice of bread-bc17ebd3b6c15fe1) in the toaster (toaster-433988d29ba772e3). \n2. At 12:31, the user picks up the bag of bread next to the toaster (toaster-433988d29ba772e3), turns right, walks forward past the kitchen table on their left and the refrigerator on their right to put the bread back in the refrigerator . \n3. So standing at the toaster (toaster-433988d29ba772e3) putting in bread means the kitchen table is directly to the user's right.", "referenced_object_masks": [ "toaster-433988d29ba772e3", "slice of bread-07f9bcc879ea3ac9", "second slice of bread-bc17ebd3b6c15fe1" ] }, { "key": "P03-20240216-084005:60925b22e79c4c4808ab09372c5e4cbc41998f8b", "video_id": "P03-20240216-084005", "question": "What is the sum of the number of coffee mugs that I handwash times the number of clockwise circles I make the first 2 times that I pour water from the kettle into the French press?", "answer_choice_0": "15", "answer_choice_1": "16", "answer_choice_2": "24", "answer_choice_3": "14", "answer_choice_4": "22", "answer_id": 4, "answer": "22", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Object Recognition", "Counting", "Numerical Reasoning" ], "objects_of_interest": [ "small frying pan-bf53dbc70c79285b", "saucepan-b510651c0612bf5d", "padding-461cc7a60eaa5ec3", "colander-cb7b5ed0ed99d32d", "flask-d103e3fa34ce3a93", "flask lid-3d9a8e68644e336a", "v60 metal filter-0a542e30403deb38", "v60 stand-c29b7f30f12d7607", "ramekin-f0f442d3e88ed600", "plastic spatula-b71c30f541f155f0", "wooden spoon-f7e10de2cc215049", "another wooden chopstick-58e19a9fb337dee3", "knife-e206fb4b84a78638", "spoon-7c2ba9593d3934f7", "metal spoon-0ddab6a88d875136", "another wooden spoon-d5c0f5650ec3ee3c", "tea filter-6f2b023952779611", "teapot lid-b8783677e65c6eb3", "toaster-433988d29ba772e3", "two tea towels-d18eaee2b828cb8a", "tea towel-660fdb6a7b5d40ea", "tea towl2-b2cdf58fb5dfdaba", "bread-0ef9e8df5ca965a8", "food container-23e86bffc4bae3c2", "oat milk-5b94ca05eb43ae92", "plastic bag-3375571c995d2e85", "tote bag-516c2c84d68088c8", "black band-2ec4e7adb0d95ec4", "kitchen roll holder-da491c46559c7852", "kitchen roll-4b65e8391109478c", "fork-559a53e0e7ca0b18", "kitchen towel piece-11ac397aa297f511", "coffee beans bag-eee880237ca44692", "scissors-832aeef234cf8c91", "scissors safety cover-e7319384249bfb5e", "red coffee beans spoon-1d15269ce762ac43", "coffee grinder-b15572680339b915", "top lid of coffee grinder-cab01a3cd845f82a", "clip-4dd05db91a440279", "kettle-35103617f5ebd942", "bag top cut out-1c471bcc7afce213", "another piece of plastic-d7bee3b6c0e27c8c", "coffee grinder bottom compartment-ce6597076e840860", "v60 coffee filter-d076aad17dd70dcf", "coffee grinder container lid-d6dcbfe74626a8f2", "orange cloth-3b3567914224587a", "cloth-37f7fc8cbaf408ef", "black plastic spoon-084190aa32b92b00", "dirty plastic container-e8a8ea43e56872ee", "small grey plate-4681b0699e71e7b1", "red small plate-c8f20bbaabd1f040", "plastic lid-1d326b0817911b9e", "two dirty chopsticks-348593d7653c95e4", "dirty plastic box2-3d57079dd59f11cd", "metal colander-74a100a5fc6b628b", "saucepan2-a75f362ca5bcf045", "dirty bowl-be92755a88aca05a", "another dirty plastic container-b027d28aba2937bc", "large coffee cup-bf1fcaf6910bc1ec", "checkered cup-fbdce38314ed5872", "coffee jug-b0fb10b629299815", "another coffee cup-2b7d0b456a84765d", "another coffee cup2-900fdd129587e4cd", "sponge-4b91bca66ca4e912", "nutella jar-cb2c10a7e933c6e4", "slice of bread-07f9bcc879ea3ac9", "second slice of bread-bc17ebd3b6c15fe1", "two plates-eec89bc8ed72e4a4", "knife2-b8de907c658450a4", "glass-0a6223d2e12ce82e", "v60 coffee filter on coffee jug-d77cf189c8769ff7", "plate2-1067c8e5fe3e25cf", "nutella jar lid-c4d0dd5555f51046", "phone-7ccda200a2da18ea" ], "reasoning": "1. I watched to see how many coffee mugs the user washes by hand.\n2. The user washes a coffee mug (another coffee cup-2b7d0b456a84765d) by hand at 10:06 and a second coffee mug (another coffee cup2-900fdd129587e4cd) at 10:24.\n3. I watched for the user to pour water from the kettle (kettle-35103617f5ebd942) into the French press (coffee jug-b0fb10b629299815) beginning at 9:49, and again at 11:22.\n4. The first time, the user moves the kettle (kettle-35103617f5ebd942) in 5 clockwise circles, and 6 clockwise circles the second time.\n5. The kettle (kettle-35103617f5ebd942) is moved in a total of 11 clockwise circles.\n6. 2x11=22", "referenced_object_masks": [ "another coffee cup2-900fdd129587e4cd", "coffee jug-b0fb10b629299815", "another coffee cup-2b7d0b456a84765d", "kettle-35103617f5ebd942" ] }, { "key": "P03-20240216-084005:62fd48d883a645b10f39e997f8ebcf7517ae62d2", "video_id": "P03-20240216-084005", "question": "While the coffee is being ground, how many individual items do I touch, minus the number of times I turn on the coffee grinder?", "answer_choice_0": "3", "answer_choice_1": "4", "answer_choice_2": "5", "answer_choice_3": "6", "answer_choice_4": "2", "answer_id": 3, "answer": "6", "question_type": [ "Action Recognition", "Counting", "Object Recognition", "Numerical Reasoning" ], "objects_of_interest": [ "small frying pan-bf53dbc70c79285b", "saucepan-b510651c0612bf5d", "padding-461cc7a60eaa5ec3", "colander-cb7b5ed0ed99d32d", "flask-d103e3fa34ce3a93", "flask lid-3d9a8e68644e336a", "v60 metal filter-0a542e30403deb38", "v60 stand-c29b7f30f12d7607", "ramekin-f0f442d3e88ed600", "plastic spatula-b71c30f541f155f0", "wooden spoon-f7e10de2cc215049", "another wooden chopstick-58e19a9fb337dee3", "knife-e206fb4b84a78638", "spoon-7c2ba9593d3934f7", "metal spoon-0ddab6a88d875136", "another wooden spoon-d5c0f5650ec3ee3c", "tea filter-6f2b023952779611", "teapot lid-b8783677e65c6eb3", "toaster-433988d29ba772e3", "two tea towels-d18eaee2b828cb8a", "tea towel-660fdb6a7b5d40ea", "tea towl2-b2cdf58fb5dfdaba", "bread-0ef9e8df5ca965a8", "food container-23e86bffc4bae3c2", "oat milk-5b94ca05eb43ae92", "plastic bag-3375571c995d2e85", "tote bag-516c2c84d68088c8", "black band-2ec4e7adb0d95ec4", "kitchen roll holder-da491c46559c7852", "kitchen roll-4b65e8391109478c", "fork-559a53e0e7ca0b18", "kitchen towel piece-11ac397aa297f511", "coffee beans bag-eee880237ca44692", "scissors-832aeef234cf8c91", "scissors safety cover-e7319384249bfb5e", "red coffee beans spoon-1d15269ce762ac43", "coffee grinder-b15572680339b915", "top lid of coffee grinder-cab01a3cd845f82a", "clip-4dd05db91a440279", "kettle-35103617f5ebd942", "bag top cut out-1c471bcc7afce213", "another piece of plastic-d7bee3b6c0e27c8c", "coffee grinder bottom compartment-ce6597076e840860", "v60 coffee filter-d076aad17dd70dcf", "coffee grinder container lid-d6dcbfe74626a8f2", "orange cloth-3b3567914224587a", "cloth-37f7fc8cbaf408ef", "black plastic spoon-084190aa32b92b00", "dirty plastic container-e8a8ea43e56872ee", "small grey plate-4681b0699e71e7b1", "red small plate-c8f20bbaabd1f040", "plastic lid-1d326b0817911b9e", "two dirty chopsticks-348593d7653c95e4", "dirty plastic box2-3d57079dd59f11cd", "metal colander-74a100a5fc6b628b", "saucepan2-a75f362ca5bcf045", "dirty bowl-be92755a88aca05a", "another dirty plastic container-b027d28aba2937bc", "large coffee cup-bf1fcaf6910bc1ec", "checkered cup-fbdce38314ed5872", "coffee jug-b0fb10b629299815", "another coffee cup-2b7d0b456a84765d", "another coffee cup2-900fdd129587e4cd", "sponge-4b91bca66ca4e912", "nutella jar-cb2c10a7e933c6e4", "slice of bread-07f9bcc879ea3ac9", "second slice of bread-bc17ebd3b6c15fe1", "two plates-eec89bc8ed72e4a4", "knife2-b8de907c658450a4", "glass-0a6223d2e12ce82e", "v60 coffee filter on coffee jug-d77cf189c8769ff7", "plate2-1067c8e5fe3e25cf", "nutella jar lid-c4d0dd5555f51046", "phone-7ccda200a2da18ea" ], "reasoning": "1. I watched for the user to grind the coffee.\n2. The user turns the coffee grinder (coffee grinder-b15572680339b915) on 3 times between 4:38 and 6:09. This occurs at 04:38, 04:51, and 05:48.\n3. Between 4:38 and 6:09, the user touches the coffee grinder (coffee grinder-b15572680339b915) at 4:38, the bag of coffee beans (coffee beans bag-eee880237ca44692) at 4:38, the lid of the grinder (top lid of coffee grinder-cab01a3cd845f82a) at 4:43, the clip (clip-4dd05db91a440279) at 5:15, the kettle (kettle-35103617f5ebd942) at 5:34, the top portion of the coffee bag (bag top cut out-1c471bcc7afce213) at 5:50, the trash can and sheet of paper at 05:56, and another piece of plastic (another piece of plastic-d7bee3b6c0e27c8c) at 5:59.\n4. In total, the user touches 9 different items.\n5. The number of items touched (9) minus times the user turns the grinder on (3) is 6.", "referenced_object_masks": [ "top lid of coffee grinder-cab01a3cd845f82a", "clip-4dd05db91a440279", "bag top cut out-1c471bcc7afce213", "kettle-35103617f5ebd942", "another piece of plastic-d7bee3b6c0e27c8c", "coffee beans bag-eee880237ca44692", "coffee grinder-b15572680339b915" ] }, { "key": "P03-20240216-084005:8197229d8508bd1c1e8da3d448adb7fe17341e21", "video_id": "P03-20240216-084005", "question": "In what order are the colors of the saucepans in the stack in the cabinet, from top to bottom, in the first cabinet to be opened after a paper towel is torn off of the roll?", "answer_choice_0": "Gray, white, gray", "answer_choice_1": "White, gray, white", "answer_choice_2": "Gray, gray, white", "answer_choice_3": "White, gray, gray", "answer_choice_4": "White, white, gray", "answer_id": 0, "answer": "Gray, white, gray", "question_type": [ "Action Recognition", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "small frying pan-bf53dbc70c79285b", "saucepan-b510651c0612bf5d", "padding-461cc7a60eaa5ec3", "colander-cb7b5ed0ed99d32d", "flask-d103e3fa34ce3a93", "flask lid-3d9a8e68644e336a", "v60 metal filter-0a542e30403deb38", "v60 stand-c29b7f30f12d7607", "ramekin-f0f442d3e88ed600", "plastic spatula-b71c30f541f155f0", "wooden spoon-f7e10de2cc215049", "another wooden chopstick-58e19a9fb337dee3", "knife-e206fb4b84a78638", "spoon-7c2ba9593d3934f7", "metal spoon-0ddab6a88d875136", "another wooden spoon-d5c0f5650ec3ee3c", "tea filter-6f2b023952779611", "teapot lid-b8783677e65c6eb3", "toaster-433988d29ba772e3", "two tea towels-d18eaee2b828cb8a", "tea towel-660fdb6a7b5d40ea", "tea towl2-b2cdf58fb5dfdaba", "bread-0ef9e8df5ca965a8", "food container-23e86bffc4bae3c2", "oat milk-5b94ca05eb43ae92", "plastic bag-3375571c995d2e85", "tote bag-516c2c84d68088c8", "black band-2ec4e7adb0d95ec4", "kitchen roll holder-da491c46559c7852", "kitchen roll-4b65e8391109478c", "fork-559a53e0e7ca0b18", "kitchen towel piece-11ac397aa297f511", "coffee beans bag-eee880237ca44692", "scissors-832aeef234cf8c91", "scissors safety cover-e7319384249bfb5e", "red coffee beans spoon-1d15269ce762ac43", "coffee grinder-b15572680339b915", "top lid of coffee grinder-cab01a3cd845f82a", "clip-4dd05db91a440279", "kettle-35103617f5ebd942", "bag top cut out-1c471bcc7afce213", "another piece of plastic-d7bee3b6c0e27c8c", "coffee grinder bottom compartment-ce6597076e840860", "v60 coffee filter-d076aad17dd70dcf", "coffee grinder container lid-d6dcbfe74626a8f2", "orange cloth-3b3567914224587a", "cloth-37f7fc8cbaf408ef", "black plastic spoon-084190aa32b92b00", "dirty plastic container-e8a8ea43e56872ee", "small grey plate-4681b0699e71e7b1", "red small plate-c8f20bbaabd1f040", "plastic lid-1d326b0817911b9e", "two dirty chopsticks-348593d7653c95e4", "dirty plastic box2-3d57079dd59f11cd", "metal colander-74a100a5fc6b628b", "saucepan2-a75f362ca5bcf045", "dirty bowl-be92755a88aca05a", "another dirty plastic container-b027d28aba2937bc", "large coffee cup-bf1fcaf6910bc1ec", "checkered cup-fbdce38314ed5872", "coffee jug-b0fb10b629299815", "another coffee cup-2b7d0b456a84765d", "another coffee cup2-900fdd129587e4cd", "sponge-4b91bca66ca4e912", "nutella jar-cb2c10a7e933c6e4", "slice of bread-07f9bcc879ea3ac9", "second slice of bread-bc17ebd3b6c15fe1", "two plates-eec89bc8ed72e4a4", "knife2-b8de907c658450a4", "glass-0a6223d2e12ce82e", "v60 coffee filter on coffee jug-d77cf189c8769ff7", "plate2-1067c8e5fe3e25cf", "nutella jar lid-c4d0dd5555f51046", "phone-7ccda200a2da18ea" ], "reasoning": "1. I watched for a piece of paper towel (kitchen towel piece-11ac397aa297f511) to be torn off of the roll (kitchen roll-4b65e8391109478c), which the user does at 2:54.\n2. I looked for a cabinet to be opened. There are several open already, but the next cabinet to get opened by the user is at 2:57.\n3. Inside the cabinet is a stack of three saucepans, which are gray, white, and gray from top to bottom.", "referenced_object_masks": [ "kitchen towel piece-11ac397aa297f511", "kitchen roll-4b65e8391109478c" ] }, { "key": "P03-20240216-084005:8e36e18a7d8fa5a5ad560e6f0fce9a0747834fef", "video_id": "P03-20240216-084005", "question": "What number do I get if I take the number of seconds to fill up the coffee bean grinder with whole beans, multiply the number by the number of scoops of whole coffee beans I put in the coffee grinder and then divide by the number of circles I make with the tea kettle when pouring hot water into the grounded coffee the first time?", "answer_choice_0": "4.8", "answer_choice_1": "5.0", "answer_choice_2": "4.6", "answer_choice_3": "5.2", "answer_choice_4": "5.4", "answer_id": 3, "answer": "5.2", "question_type": [ "Action Recognition", "Counting", "Event Occurence", "Numerical Reasoning", "State Changes", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "small frying pan-bf53dbc70c79285b", "saucepan-b510651c0612bf5d", "padding-461cc7a60eaa5ec3", "colander-cb7b5ed0ed99d32d", "flask-d103e3fa34ce3a93", "flask lid-3d9a8e68644e336a", "v60 metal filter-0a542e30403deb38", "v60 stand-c29b7f30f12d7607", "ramekin-f0f442d3e88ed600", "plastic spatula-b71c30f541f155f0", "wooden spoon-f7e10de2cc215049", "another wooden chopstick-58e19a9fb337dee3", "knife-e206fb4b84a78638", "spoon-7c2ba9593d3934f7", "metal spoon-0ddab6a88d875136", "another wooden spoon-d5c0f5650ec3ee3c", "tea filter-6f2b023952779611", "teapot lid-b8783677e65c6eb3", "toaster-433988d29ba772e3", "two tea towels-d18eaee2b828cb8a", "tea towel-660fdb6a7b5d40ea", "tea towl2-b2cdf58fb5dfdaba", "bread-0ef9e8df5ca965a8", "food container-23e86bffc4bae3c2", "oat milk-5b94ca05eb43ae92", "plastic bag-3375571c995d2e85", "tote bag-516c2c84d68088c8", "black band-2ec4e7adb0d95ec4", "kitchen roll holder-da491c46559c7852", "kitchen roll-4b65e8391109478c", "fork-559a53e0e7ca0b18", "kitchen towel piece-11ac397aa297f511", "coffee beans bag-eee880237ca44692", "scissors-832aeef234cf8c91", "scissors safety cover-e7319384249bfb5e", "red coffee beans spoon-1d15269ce762ac43", "coffee grinder-b15572680339b915", "top lid of coffee grinder-cab01a3cd845f82a", "clip-4dd05db91a440279", "kettle-35103617f5ebd942", "bag top cut out-1c471bcc7afce213", "another piece of plastic-d7bee3b6c0e27c8c", "coffee grinder bottom compartment-ce6597076e840860", "v60 coffee filter-d076aad17dd70dcf", "coffee grinder container lid-d6dcbfe74626a8f2", "orange cloth-3b3567914224587a", "cloth-37f7fc8cbaf408ef", "black plastic spoon-084190aa32b92b00", "dirty plastic container-e8a8ea43e56872ee", "small grey plate-4681b0699e71e7b1", "red small plate-c8f20bbaabd1f040", "plastic lid-1d326b0817911b9e", "two dirty chopsticks-348593d7653c95e4", "dirty plastic box2-3d57079dd59f11cd", "metal colander-74a100a5fc6b628b", "saucepan2-a75f362ca5bcf045", "dirty bowl-be92755a88aca05a", "another dirty plastic container-b027d28aba2937bc", "large coffee cup-bf1fcaf6910bc1ec", "checkered cup-fbdce38314ed5872", "coffee jug-b0fb10b629299815", "another coffee cup-2b7d0b456a84765d", "another coffee cup2-900fdd129587e4cd", "sponge-4b91bca66ca4e912", "nutella jar-cb2c10a7e933c6e4", "slice of bread-07f9bcc879ea3ac9", "second slice of bread-bc17ebd3b6c15fe1", "two plates-eec89bc8ed72e4a4", "knife2-b8de907c658450a4", "glass-0a6223d2e12ce82e", "v60 coffee filter on coffee jug-d77cf189c8769ff7", "plate2-1067c8e5fe3e25cf", "nutella jar lid-c4d0dd5555f51046", "phone-7ccda200a2da18ea" ], "reasoning": "1. I watched the whole video and saw the user start scooping whole coffee beans (coffee beans bag-eee880237ca44692) into the coffee grinder (coffee grinder-b15572680339b915) at 04:23. \n2. The user stopped scooping whole coffee beans into the coffee grinder at 04:36.\n3. The user scooped whole coffee beans (coffee beans bag-eee880237ca44692) into the grinder (coffee grinder-b15572680339b915) for 13 seconds. \n4. The user pours 2 scoops of the whole coffee beans (coffee beans bag-eee880237ca44692) into the coffee grinder (coffee grinder-b15572680339b915) at 04:26.\n5. At 09:50, the user pours 5 circles of hot water into the ground coffee .\n6. 13*2 divided by 5 is 5.2.", "referenced_object_masks": [ "coffee beans bag-eee880237ca44692", "coffee grinder-b15572680339b915" ] }, { "key": "P03-20240216-084005:cad1f9356dea908ccd4f9205acf214f6e88eac2a", "video_id": "P03-20240216-084005", "question": "After I put the Nutella away in the cabinet, how many degrees to my right do I have to turn to face the refrigerator?", "answer_choice_0": "90 degrees", "answer_choice_1": "135 degrees", "answer_choice_2": "45 degrees", "answer_choice_3": "100 degrees", "answer_choice_4": "30 degrees", "answer_id": 1, "answer": "135 degrees", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Event Occurence", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "small frying pan-bf53dbc70c79285b", "saucepan-b510651c0612bf5d", "padding-461cc7a60eaa5ec3", "colander-cb7b5ed0ed99d32d", "flask-d103e3fa34ce3a93", "flask lid-3d9a8e68644e336a", "v60 metal filter-0a542e30403deb38", "v60 stand-c29b7f30f12d7607", "ramekin-f0f442d3e88ed600", "plastic spatula-b71c30f541f155f0", "wooden spoon-f7e10de2cc215049", "another wooden chopstick-58e19a9fb337dee3", "knife-e206fb4b84a78638", "spoon-7c2ba9593d3934f7", "metal spoon-0ddab6a88d875136", "another wooden spoon-d5c0f5650ec3ee3c", "tea filter-6f2b023952779611", "teapot lid-b8783677e65c6eb3", "toaster-433988d29ba772e3", "two tea towels-d18eaee2b828cb8a", "tea towel-660fdb6a7b5d40ea", "tea towl2-b2cdf58fb5dfdaba", "bread-0ef9e8df5ca965a8", "food container-23e86bffc4bae3c2", "oat milk-5b94ca05eb43ae92", "plastic bag-3375571c995d2e85", "tote bag-516c2c84d68088c8", "black band-2ec4e7adb0d95ec4", "kitchen roll holder-da491c46559c7852", "kitchen roll-4b65e8391109478c", "fork-559a53e0e7ca0b18", "kitchen towel piece-11ac397aa297f511", "coffee beans bag-eee880237ca44692", "scissors-832aeef234cf8c91", "scissors safety cover-e7319384249bfb5e", "red coffee beans spoon-1d15269ce762ac43", "coffee grinder-b15572680339b915", "top lid of coffee grinder-cab01a3cd845f82a", "clip-4dd05db91a440279", "kettle-35103617f5ebd942", "bag top cut out-1c471bcc7afce213", "another piece of plastic-d7bee3b6c0e27c8c", "coffee grinder bottom compartment-ce6597076e840860", "v60 coffee filter-d076aad17dd70dcf", "coffee grinder container lid-d6dcbfe74626a8f2", "orange cloth-3b3567914224587a", "cloth-37f7fc8cbaf408ef", "black plastic spoon-084190aa32b92b00", "dirty plastic container-e8a8ea43e56872ee", "small grey plate-4681b0699e71e7b1", "red small plate-c8f20bbaabd1f040", "plastic lid-1d326b0817911b9e", "two dirty chopsticks-348593d7653c95e4", "dirty plastic box2-3d57079dd59f11cd", "metal colander-74a100a5fc6b628b", "saucepan2-a75f362ca5bcf045", "dirty bowl-be92755a88aca05a", "another dirty plastic container-b027d28aba2937bc", "large coffee cup-bf1fcaf6910bc1ec", "checkered cup-fbdce38314ed5872", "coffee jug-b0fb10b629299815", "another coffee cup-2b7d0b456a84765d", "another coffee cup2-900fdd129587e4cd", "sponge-4b91bca66ca4e912", "nutella jar-cb2c10a7e933c6e4", "slice of bread-07f9bcc879ea3ac9", "second slice of bread-bc17ebd3b6c15fe1", "two plates-eec89bc8ed72e4a4", "knife2-b8de907c658450a4", "glass-0a6223d2e12ce82e", "v60 coffee filter on coffee jug-d77cf189c8769ff7", "plate2-1067c8e5fe3e25cf", "nutella jar lid-c4d0dd5555f51046", "phone-7ccda200a2da18ea" ], "reasoning": "1. At 17:33, the user is facing the wall with the cabinet they just put Nutella (nutella jar-cb2c10a7e933c6e4) in.\n2. At 17:33, the user picks up the 2 plates (two plates-eec89bc8ed72e4a4) of toast with Nutella (nutella jar-cb2c10a7e933c6e4) and turns right. \n3. Turning to the right is a 90 degree turn.\n4. At 17:34, the user walks straight down the kitchen to put the plates (two plates-eec89bc8ed72e4a4) on the kitchen table .\n5. The refrigerator is directly across from the kitchen table , so the user has about another 45 degrees to move right to face the refrigerator . \n6. 90 plus 45 is 135 degrees.", "referenced_object_masks": [ "nutella jar-cb2c10a7e933c6e4", "two plates-eec89bc8ed72e4a4" ] }, { "key": "P03-20240216-090759:660fbfa1fd557aa63ec6b452061d30da520c9455", "video_id": "P03-20240216-090759", "question": "What is the total when adding the duration of time the dishwasher was open and the duration of time that I stood on the cardboard tray?", "answer_choice_0": "03:15 minutes", "answer_choice_1": "03:11 minutes", "answer_choice_2": "03:17 minutes", "answer_choice_3": "03:19 minutes", "answer_choice_4": "03:13 minutes", "answer_id": 0, "answer": "03:15 minutes", "question_type": [ "Object Recognition", "State Changes", "Action Recognition", "Numerical Reasoning", "Temporal Reasoning" ], "objects_of_interest": [ "dirty plates-814508c0edb209cb", "dirty plate2-24a18ac963cfcfa3", "dish washer brush-af169df6762f87db", "knife-590805b7d185cf90", "jug-5bf7cdefa0b0837a", "cup-da5eb03e14d49590", "oatmeal carton-85be4ce91cd91cae", "glass-8add30b95c9fcda8", "used tissue-7fc0ea62c36961c7", "tea towels-295d5cb0aa923460", "tea towel2-fddc7af70823c0df", "chair-13b498db1ae74fd8", "chair2-225f2aa2d56eb3fb", "food waste bin-011dcfe375419738", "food waste bin bag-1c284224d6fa2c55", "coffee filter-1de906680787cbcd", "coffee filter stand-f004c50039a6030e", "dishwasher tablet-ab7db3309cc1d625", "cloth-56e5d43106c2fdeb", "sink drain plug-406a3a31215f2c85", "soap tray-61d52cda6f5bf452", "brush-e9274111e8c8993b", "toaster-ad0a57d8b4d43644", "recycling bin-d730944e681e9179", "cardboard tray-a3f73539d84d5435", "blue cupboard tray 1-9a118aa7161ad9c2" ], "reasoning": "1. I observed that the user opened the dishwasher at 00:14. I watched the video until I observed the user closing the dishwasher at 03:13. This means the dishwasher was open for 02:59 minutes.\n2. I observed the user standing on the cardboard tray (cardboard tray-a3f73539d84d5435) beginning at 05:26. The user removed their foot from the cardboard tray (cardboard tray-a3f73539d84d5435) at 05:42. This totals 16 seconds.\n3. The total duration of these two periods is 03:15 minutes.", "referenced_object_masks": [ "cardboard tray-a3f73539d84d5435" ] }, { "key": "P03-20240216-090759:79561f3fd15377e4432e91f5676422f332af1d08", "video_id": "P03-20240216-090759", "question": "How many times in total did I brush the first three items over the sink?", "answer_choice_0": "24", "answer_choice_1": "21", "answer_choice_2": "22", "answer_choice_3": "23", "answer_choice_4": "20", "answer_id": 2, "answer": "22", "question_type": [ "Action Recognition", "Object Recognition", "Counting", "Numerical Reasoning" ], "objects_of_interest": [ "dirty plates-814508c0edb209cb", "dirty plate2-24a18ac963cfcfa3", "dish washer brush-af169df6762f87db", "knife-590805b7d185cf90", "jug-5bf7cdefa0b0837a", "cup-da5eb03e14d49590", "oatmeal carton-85be4ce91cd91cae", "glass-8add30b95c9fcda8", "used tissue-7fc0ea62c36961c7", "tea towels-295d5cb0aa923460", "tea towel2-fddc7af70823c0df", "chair-13b498db1ae74fd8", "chair2-225f2aa2d56eb3fb", "food waste bin-011dcfe375419738", "food waste bin bag-1c284224d6fa2c55", "coffee filter-1de906680787cbcd", "coffee filter stand-f004c50039a6030e", "dishwasher tablet-ab7db3309cc1d625", "cloth-56e5d43106c2fdeb", "sink drain plug-406a3a31215f2c85", "soap tray-61d52cda6f5bf452", "brush-e9274111e8c8993b", "toaster-ad0a57d8b4d43644", "recycling bin-d730944e681e9179", "cardboard tray-a3f73539d84d5435", "blue cupboard tray 1-9a118aa7161ad9c2" ], "reasoning": "1. I observed the user brushing the first item over the sink, the left plate (dirty plate2-24a18ac963cfcfa3), with a dish washer brush (dish washer brush-af169df6762f87db) from 00:10 to 00:12. During this segment, the user brushed 6 times.\n2. The user brushed the second item, the right plate (dirty plates-814508c0edb209cb), from 00:20 to 00:23. During this segment, the user brushed 7 times.\n3. The user brushed the third item, the butter knife (knife-590805b7d185cf90), from 00:27 to 00:30. During this segment, the user brushed 9 times.\n4. When added together, 6+7+9, the user brushed the first 3 items 22 times.", "referenced_object_masks": [ "knife-590805b7d185cf90", "dirty plates-814508c0edb209cb", "dirty plate2-24a18ac963cfcfa3", "dish washer brush-af169df6762f87db" ] }, { "key": "P03-20240216-090759:b6c8fb448ce4bca004848b6d9b2b34b06a33907b", "video_id": "P03-20240216-090759", "question": "How many times did I open the cabinet where the food waste bin is stored?", "answer_choice_0": "2", "answer_choice_1": "1", "answer_choice_2": "4", "answer_choice_3": "5", "answer_choice_4": "3", "answer_id": 4, "answer": "3", "question_type": [ "Action Recognition", "Object Recognition", "Situational Awareness", "Counting" ], "objects_of_interest": [ "dirty plates-814508c0edb209cb", "dirty plate2-24a18ac963cfcfa3", "dish washer brush-af169df6762f87db", "knife-590805b7d185cf90", "jug-5bf7cdefa0b0837a", "cup-da5eb03e14d49590", "oatmeal carton-85be4ce91cd91cae", "glass-8add30b95c9fcda8", "used tissue-7fc0ea62c36961c7", "tea towels-295d5cb0aa923460", "tea towel2-fddc7af70823c0df", "chair-13b498db1ae74fd8", "chair2-225f2aa2d56eb3fb", "food waste bin-011dcfe375419738", "food waste bin bag-1c284224d6fa2c55", "coffee filter-1de906680787cbcd", "coffee filter stand-f004c50039a6030e", "dishwasher tablet-ab7db3309cc1d625", "cloth-56e5d43106c2fdeb", "sink drain plug-406a3a31215f2c85", "soap tray-61d52cda6f5bf452", "brush-e9274111e8c8993b", "toaster-ad0a57d8b4d43644", "recycling bin-d730944e681e9179", "cardboard tray-a3f73539d84d5435", "blue cupboard tray 1-9a118aa7161ad9c2" ], "reasoning": "1. I identified the food waste bin (food waste bin-011dcfe375419738) at 01:50. The food waste bin (food waste bin-011dcfe375419738) is located in the cabinet under the sink.\n2. I then counted each time the user opened up the cabinet under the sink. The user opens this cabinet at 01:46, 02:48, and 04:02 for a total of 3 times.", "referenced_object_masks": [ "food waste bin-011dcfe375419738" ] }, { "key": "P03-20240216-090759:cd482e6c837a77bd3c9ee9e3b685587bf0f7be51", "video_id": "P03-20240216-090759", "question": "What is the difference in duration that I spent rinsing the coffee filter and the coffee filter stand?", "answer_choice_0": "6 seconds", "answer_choice_1": "7 seconds", "answer_choice_2": "5 seconds", "answer_choice_3": "4 seconds", "answer_choice_4": "3 seconds", "answer_id": 0, "answer": "6 seconds", "question_type": [ "Action Recognition", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "dirty plates-814508c0edb209cb", "dirty plate2-24a18ac963cfcfa3", "dish washer brush-af169df6762f87db", "knife-590805b7d185cf90", "jug-5bf7cdefa0b0837a", "cup-da5eb03e14d49590", "oatmeal carton-85be4ce91cd91cae", "glass-8add30b95c9fcda8", "used tissue-7fc0ea62c36961c7", "tea towels-295d5cb0aa923460", "tea towel2-fddc7af70823c0df", "chair-13b498db1ae74fd8", "chair2-225f2aa2d56eb3fb", "food waste bin-011dcfe375419738", "food waste bin bag-1c284224d6fa2c55", "coffee filter-1de906680787cbcd", "coffee filter stand-f004c50039a6030e", "dishwasher tablet-ab7db3309cc1d625", "cloth-56e5d43106c2fdeb", "sink drain plug-406a3a31215f2c85", "soap tray-61d52cda6f5bf452", "brush-e9274111e8c8993b", "toaster-ad0a57d8b4d43644", "recycling bin-d730944e681e9179", "cardboard tray-a3f73539d84d5435", "blue cupboard tray 1-9a118aa7161ad9c2" ], "reasoning": "1. I observed that the user rinsed the coffee filter (coffee filter-1de906680787cbcd) from 02:21 to 02:32, when the user turned off the faucet. This totals 11 seconds.\n2. The user rinsed the coffee filter stand (coffee filter stand-f004c50039a6030e) from 02:36 to 02:41. This totals 5 seconds.\n3. The difference between 11 seconds and 5 seconds is 6 seconds.", "referenced_object_masks": [ "coffee filter stand-f004c50039a6030e", "coffee filter-1de906680787cbcd" ] }, { "key": "P03-20240216-090759:e6b86e585bb007e646d0ce34a9280baa9dca0132", "video_id": "P03-20240216-090759", "question": "If I were standing and facing 180 degrees in the opposite direction from where I was standing when I put the fourth item into the dishwasher, what directions would I have to follow to reach the dining table?", "answer_choice_0": "Turn right, walk forward, turn left", "answer_choice_1": "Turn left, walk forward, turn left", "answer_choice_2": "Turn left, walk forward, stop", "answer_choice_3": "Turn right, walk forward, turn right", "answer_choice_4": "Turn directly around, walk forward, turn right", "answer_id": 0, "answer": "Turn right, walk forward, turn left", "question_type": [ "Temporal Reasoning", "Situational Awareness", "Counterfactual", "Egocentric & Spatial Perception", "Object Recognition" ], "objects_of_interest": [ "dirty plates-814508c0edb209cb", "dirty plate2-24a18ac963cfcfa3", "dish washer brush-af169df6762f87db", "knife-590805b7d185cf90", "jug-5bf7cdefa0b0837a", "cup-da5eb03e14d49590", "oatmeal carton-85be4ce91cd91cae", "glass-8add30b95c9fcda8", "used tissue-7fc0ea62c36961c7", "tea towels-295d5cb0aa923460", "tea towel2-fddc7af70823c0df", "chair-13b498db1ae74fd8", "chair2-225f2aa2d56eb3fb", "food waste bin-011dcfe375419738", "food waste bin bag-1c284224d6fa2c55", "coffee filter-1de906680787cbcd", "coffee filter stand-f004c50039a6030e", "dishwasher tablet-ab7db3309cc1d625", "cloth-56e5d43106c2fdeb", "sink drain plug-406a3a31215f2c85", "soap tray-61d52cda6f5bf452", "brush-e9274111e8c8993b", "toaster-ad0a57d8b4d43644", "recycling bin-d730944e681e9179", "cardboard tray-a3f73539d84d5435", "blue cupboard tray 1-9a118aa7161ad9c2" ], "reasoning": "1. From 00:17 to 00:31, I observed the user placing 3 items into the dishwasher. At 00:43, the user places the fourth item, a cup (cup-da5eb03e14d49590), into the dishwasher.\n2. At 00:43, I observe that the user is facing the dishwasher at a slight angle to the left. I then counterfactually imagine that the user was facing 180 degrees in the opposite direction, which would make them face toward the stove. \n3. From here, I imagine that I need to reach the dining table , which is rightward from where one faces at the stove, based on my view of the hallway and the table at 00:03. Therefore, to reach the dining table, I would have to turn right, walk forward down the hall, and turn left to face the table , which is on the wall that would be on the left when facing from the kitchen.", "referenced_object_masks": [ "cup-da5eb03e14d49590" ] }, { "key": "P03-20240216-185832:1e75283e3e0c07469f2d711a2c3816e343d6767f", "video_id": "P03-20240216-185832", "question": "After putting the unpeeled potatoes in the refrigerator, what is the fourth unique item I touch with my right hand?", "answer_choice_0": "Cabinet door", "answer_choice_1": "Tea towel", "answer_choice_2": "Dish soap", "answer_choice_3": "Kitchen faucet", "answer_choice_4": "Kitchen sponge", "answer_id": 3, "answer": "Kitchen faucet", "question_type": [ "Action Recognition", "Object Recognition", "Situational Awareness", "Temporal Reasoning" ], "objects_of_interest": [ "bag of spring greens-d94464beb250fb38", "bag of kale-9cd0eb0af15afe72", "vegetable-65a2e3a47c6302d0", "box of mushrooms-5480891ea71366cd", "spring onion-fd3fe499e30ff53c", "box of food-3b09d3574748315b", "cauliflower-705371010f925839", "large shopping bag-ab44271b31236a56", "bag of salad-a96d954fabf438d9", "bag of mozzarella-8deaddf9cb5a1c14", "single cream pot-4ffbd866d873e1e6", "spaghetti bag-f8193f01def0ea7e", "box of grapes-99d179636e4e26b3", "bananas bag-afa30e91f9a2b2ea", "bag of potatoes-8664af8e0c4512fa", "bag of mandarins-57e9d0df0122ca8b", "bag of mandarin2-05b21532ace5fc01", "garlic-124c0fee31ec5bc3", "bag of pears-2691488edebe6c10", "bag of kiwis-270bfcee080ea7af", "squash-bf5700a01518efad", "pack of pasta-107b336f60723036", "box of yeast-e96b26f5f93c789f", "bag of lentils-fc82371146cac8f8", "paper food waste bag-5f3983f5e379ca63", "superglue box-63c2e054d6869957", "bread-6039bb22e14eacbd", "tomato concentrated puree-8b68051c77a6200d", "coconut milk can-06a9fc40ecc82644", "tomato sauce bottle-ea967e3e7ff63770", "tomato sauce bottle2-ea67d9e08b7b48aa", "bottle with green lid-f75030cbed5ecaad", "tomato sauce bottle3-ca3fa9d369bd7a9b", "paper receipt-ea5837ad2dd012b3", "large shopping bag2-c1a67a7d60bed8f5", "black shoe left-b9c8a09619f10cc7", "black shoe right-fb84ffe2c21f414c", "dirty plastic spatula-39bf8dc418d3c568", "plate-ccac08aa8e50801a", "plate2-3bfd83d35c5b7ce4", "saucepan-da8b913a944ea3a1", "smaller saucepan-5b6f8054c1661133", "small bowl-f354cc2d8e7f353f", "white bowl-297b16122e17d1ba", "small bowl2-bf5691912176ed31", "coffee cup-1856f70dbc366538", "cup2-34d665c224823126", "plastic box container-16c45dc5640b85eb", "plastic container2-66b0bd40422b43a6", "cloth-e905582ea3e0d5cb", "chopstick1-384d046924eb0850", "chopstick2-740e52f3bd7b0104", "chopstick3-83bfc567ece98531", "chopstick4-00a31c1bcd4d68b3", "chopstick5-af86dc66d58998df", "chopstick6-7e52c88efe127a22", "fork1-c48edec98d98bae4", "fork2-955ca204be163621", "knife1-30debca82d73d304", "spoon1-3c0a243af93fba2e", "spoon2-a3da42ad4dd94bd9", "red spoon-546cfa010b78f974", "metal colander-f9f1b975f21e8568", "large bowl-e297ef66a083c612", "plastic colander-7d62c20f4ec67122", "bowl3-bfd73628becf1677", "bowl4-28b8b3314216e252", "bowl5-1309234381bf1d70", "white bowl2-08d9cb477606e8a6", "bowl6-c94a84c2091f7a90", "cup3-0bacf6cf9105f10d", "glass-3cefa6050e618946", "glass2-71459ef5d1e087a8", "glass3-13dffc2ecc98b459", "glass4-125b49dacdb84198", "glass5-5552f629a26efb08", "white medium sized bowl-173a8b87bb07cb50", "glass ramekin-27976f43e25f4162", "small grey plate-018ce8550f0704b7", "two identical plastic box lids-317227f0df46f5b1", "red plate-3e5c9ed8e2252755", "one small grey plate-252423245bc68222", "v60 glass filter-bb96da8cfbd03752", "v60 metal filter-7c833fe84fc512ee", "plastic food bag-7dca5a67acbec046", "plastic food box-a6ec4ab01b9766d3", "plastic food box lid-0da741d3f28b4d44", "fork3-dc4e1cc6003d165c", "dishwasher brush-8d37bfb2df0fa22c", "fork4-866bca4acda794b3", "white plate-ff6e78f4c64444d5", "grater-7fa171dbaf625b84", "small dirty bowl-a19b985558c5e8c8", "green coffee cup1-c9c6724b8274ff67", "wooden spoon-69be1eab24a0cc61", "metal teaspoon-5846ee31ef4a5033", "checkered cup-bdb131456dfe461a", "coffee jug-7feb82765554d728", "chopstick-cb987a664aeeba91", "one food waste paper bag-d21c1a1f6f84d7fb", "eggshell-c08839df3778e8ff", "blue plastic bag-3422821684ce83ab", "food waste bin-4bad12162c13724a", "sponge-0dd841b9997653d1", "washing up liquid bottle-a3e90fc19a4317dd", "pan-1dd5d7741c7265ed", "large bowl2-75fe73e9a85a889a", "plastic calendar-63c80e3f9cdf75b5", "potato1-a2e73c6ae3d8264d", "potato2-1e44164bfbaa9cad", "potato3-a4ace2eabb1c6a34", "potato4-aa365386e5442e73", "potato5-140dcd14519daf63", "potato6-1ade5ad88a5fb7c8", "potato7-a8956f52788169a1", "potato8-7ce9f636df79b299", "chopping board-0fa725b89e5753bd", "knife2-6e02fef99df274b3", "knife3-5aa57c591cb06a1d", "cloth2-08bd78e9df7e00fd", "potato9-f9df4d0f9b83b31c", "potato10-a2ed32ce73f688fc", "potato11-310f57a9a9cf7339", "potato12-e57379c4dc03dc67", "can of olive oil-3ded77db4988444c", "salt-32d76cb271cd9d21", "olive oil can cap-ac705c30088b78fd", "glass lid-924ba845005deac0", "hand wash dispenser-e454f6197c20d6d4", "plate3-dc739385c55942c8", "plate4-3277214c484e2b40", "orange cloth-a316ea7c38bb7e34", "bag of flour-55b8a2e0f7b61d16", "scale-71b4f54bf2930f23", "scissors-1cc56ec76660d927", "scissor safety cover-d689b523849381f6", "bag of flour top of bag-1582a3609c7495e6", "clip-cb17faf107a72a23", "large plastic box-5adeb358c8c9ea55", "yeast box2-a8228c1e3ffccb72", "folded sachet-fec3b7c95948d0b2", "wooden spoon2-9e1e2402cdbdda71", "wooden pizza peel-dec25775f59bf7f8", "wooden rolling pin-bc989fd347e588a1", "square plastic container-680dc3051eb0c50f", "cake mould-db4ac615fc94220f", "dough scraper-db658d2e4487aab7", "egg carton-0e0dcb4ecdf92e37", "egg-688e78a44cefb142", "egg2-205ae9b80988d4b1", "egg3-da39fb088198419b", "egg4-fc8097f80e4b2cbe", "egg5-7a615aef1c3a47b5", "white bowl3-11c6cc86c3de1854", "egg6-a3582ca144add83a", "fork5-32a1e3a313eaa6b7", "smaller baking tray-2c9c033205f0a1b5", "white baking tray-a81532f9e3d87905", "apron-f40a30089727ad06", "spoon3-f0ea632888bf9f67", "oven mitts-1f178a7dad4a1f3a", "container lid-b094d8e1f9b9a3cc", "bag of onions-14e044d9f9c02dd4", "one onion-a20caf9e35684670", "onion half-3dc3bb9a9acac0b3", "tea towel-2f785cd10ad2da5c", "tea towel2-fdb00402c2d6473b", "fork6-2ed7aa38d9a7aa5c", "knife4-c47a15c6ca400d00", "knife5-10bb43e6546b75c9", "pizza dough-d086c99d3d0752c8", "tissue box-01cec62e4005940b", "yellow glass-b0036c30465b1dd5", "cork pot stand-6aa9a0ad87fcd995", "fork7-efea21a6c8ba72ef", "spoon4-48bc0864caeb5f85" ], "reasoning": "1. I watched the video to observe the user wrap the potato bag (bag of potatoes-8664af8e0c4512fa) and put it in the refrigerator from 28:13-28:23.\n2. After doing this, the user closes the refrigerator door with their right hand, making it the first item touched with the right hand after putting the potatoes (bag of potatoes-8664af8e0c4512fa) away.\n3. After doing this, I then saw the user walk to the kitchen sink and bend down to open the cabinet door at 28:29 with their right hand. This is the second unique item grabbed with the right hand.\n4. Next, the user touches the lid of the compost bin (food waste bin-4bad12162c13724a) with both hands, at 28:31, making this the third unique item touched with the right hand.\n5. After this, the user grabs the cabinet door with the right hand from 28:32-28:33, but since this was already touched, it doesn't count for the total.\n6. Then, the user touches the kitchen faucet from 38:24-28:35 with the right hand, which is the fourth unique item.", "referenced_object_masks": [ "food waste bin-4bad12162c13724a", "bag of potatoes-8664af8e0c4512fa" ] }, { "key": "P03-20240216-185832:25c8a8263d679b0975b82a21bf815fa1395b13e0", "video_id": "P03-20240216-185832", "question": "How long would the video have been if I cut the part of me kneading dough?", "answer_choice_0": "(1) 01:09:41", "answer_choice_1": "01:08:11", "answer_choice_2": "01:10:56", "answer_choice_3": "01:14:01", "answer_choice_4": "01:12:23", "answer_id": 0, "answer": "(1) 01:09:41", "question_type": [ "Counterfactual", "Action Recognition", "Numerical Reasoning", "Temporal Reasoning" ], "objects_of_interest": [ "bag of spring greens-d94464beb250fb38", "bag of kale-9cd0eb0af15afe72", "vegetable-65a2e3a47c6302d0", "box of mushrooms-5480891ea71366cd", "spring onion-fd3fe499e30ff53c", "box of food-3b09d3574748315b", "cauliflower-705371010f925839", "large shopping bag-ab44271b31236a56", "bag of salad-a96d954fabf438d9", "bag of mozzarella-8deaddf9cb5a1c14", "single cream pot-4ffbd866d873e1e6", "spaghetti bag-f8193f01def0ea7e", "box of grapes-99d179636e4e26b3", "bananas bag-afa30e91f9a2b2ea", "bag of potatoes-8664af8e0c4512fa", "bag of mandarins-57e9d0df0122ca8b", "bag of mandarin2-05b21532ace5fc01", "garlic-124c0fee31ec5bc3", "bag of pears-2691488edebe6c10", "bag of kiwis-270bfcee080ea7af", "squash-bf5700a01518efad", "pack of pasta-107b336f60723036", "box of yeast-e96b26f5f93c789f", "bag of lentils-fc82371146cac8f8", "paper food waste bag-5f3983f5e379ca63", "superglue box-63c2e054d6869957", "bread-6039bb22e14eacbd", "tomato concentrated puree-8b68051c77a6200d", "coconut milk can-06a9fc40ecc82644", "tomato sauce bottle-ea967e3e7ff63770", "tomato sauce bottle2-ea67d9e08b7b48aa", "bottle with green lid-f75030cbed5ecaad", "tomato sauce bottle3-ca3fa9d369bd7a9b", "paper receipt-ea5837ad2dd012b3", "large shopping bag2-c1a67a7d60bed8f5", "black shoe left-b9c8a09619f10cc7", "black shoe right-fb84ffe2c21f414c", "dirty plastic spatula-39bf8dc418d3c568", "plate-ccac08aa8e50801a", "plate2-3bfd83d35c5b7ce4", "saucepan-da8b913a944ea3a1", "smaller saucepan-5b6f8054c1661133", "small bowl-f354cc2d8e7f353f", "white bowl-297b16122e17d1ba", "small bowl2-bf5691912176ed31", "coffee cup-1856f70dbc366538", "cup2-34d665c224823126", "plastic box container-16c45dc5640b85eb", "plastic container2-66b0bd40422b43a6", "cloth-e905582ea3e0d5cb", "chopstick1-384d046924eb0850", "chopstick2-740e52f3bd7b0104", "chopstick3-83bfc567ece98531", "chopstick4-00a31c1bcd4d68b3", "chopstick5-af86dc66d58998df", "chopstick6-7e52c88efe127a22", "fork1-c48edec98d98bae4", "fork2-955ca204be163621", "knife1-30debca82d73d304", "spoon1-3c0a243af93fba2e", "spoon2-a3da42ad4dd94bd9", "red spoon-546cfa010b78f974", "metal colander-f9f1b975f21e8568", "large bowl-e297ef66a083c612", "plastic colander-7d62c20f4ec67122", "bowl3-bfd73628becf1677", "bowl4-28b8b3314216e252", "bowl5-1309234381bf1d70", "white bowl2-08d9cb477606e8a6", "bowl6-c94a84c2091f7a90", "cup3-0bacf6cf9105f10d", "glass-3cefa6050e618946", "glass2-71459ef5d1e087a8", "glass3-13dffc2ecc98b459", "glass4-125b49dacdb84198", "glass5-5552f629a26efb08", "white medium sized bowl-173a8b87bb07cb50", "glass ramekin-27976f43e25f4162", "small grey plate-018ce8550f0704b7", "two identical plastic box lids-317227f0df46f5b1", "red plate-3e5c9ed8e2252755", "one small grey plate-252423245bc68222", "v60 glass filter-bb96da8cfbd03752", "v60 metal filter-7c833fe84fc512ee", "plastic food bag-7dca5a67acbec046", "plastic food box-a6ec4ab01b9766d3", "plastic food box lid-0da741d3f28b4d44", "fork3-dc4e1cc6003d165c", "dishwasher brush-8d37bfb2df0fa22c", "fork4-866bca4acda794b3", "white plate-ff6e78f4c64444d5", "grater-7fa171dbaf625b84", "small dirty bowl-a19b985558c5e8c8", "green coffee cup1-c9c6724b8274ff67", "wooden spoon-69be1eab24a0cc61", "metal teaspoon-5846ee31ef4a5033", "checkered cup-bdb131456dfe461a", "coffee jug-7feb82765554d728", "chopstick-cb987a664aeeba91", "one food waste paper bag-d21c1a1f6f84d7fb", "eggshell-c08839df3778e8ff", "blue plastic bag-3422821684ce83ab", "food waste bin-4bad12162c13724a", "sponge-0dd841b9997653d1", "washing up liquid bottle-a3e90fc19a4317dd", "pan-1dd5d7741c7265ed", "large bowl2-75fe73e9a85a889a", "plastic calendar-63c80e3f9cdf75b5", "potato1-a2e73c6ae3d8264d", "potato2-1e44164bfbaa9cad", "potato3-a4ace2eabb1c6a34", "potato4-aa365386e5442e73", "potato5-140dcd14519daf63", "potato6-1ade5ad88a5fb7c8", "potato7-a8956f52788169a1", "potato8-7ce9f636df79b299", "chopping board-0fa725b89e5753bd", "knife2-6e02fef99df274b3", "knife3-5aa57c591cb06a1d", "cloth2-08bd78e9df7e00fd", "potato9-f9df4d0f9b83b31c", "potato10-a2ed32ce73f688fc", "potato11-310f57a9a9cf7339", "potato12-e57379c4dc03dc67", "can of olive oil-3ded77db4988444c", "salt-32d76cb271cd9d21", "olive oil can cap-ac705c30088b78fd", "glass lid-924ba845005deac0", "hand wash dispenser-e454f6197c20d6d4", "plate3-dc739385c55942c8", "plate4-3277214c484e2b40", "orange cloth-a316ea7c38bb7e34", "bag of flour-55b8a2e0f7b61d16", "scale-71b4f54bf2930f23", "scissors-1cc56ec76660d927", "scissor safety cover-d689b523849381f6", "bag of flour top of bag-1582a3609c7495e6", "clip-cb17faf107a72a23", "large plastic box-5adeb358c8c9ea55", "yeast box2-a8228c1e3ffccb72", "folded sachet-fec3b7c95948d0b2", "wooden spoon2-9e1e2402cdbdda71", "wooden pizza peel-dec25775f59bf7f8", "wooden rolling pin-bc989fd347e588a1", "square plastic container-680dc3051eb0c50f", "cake mould-db4ac615fc94220f", "dough scraper-db658d2e4487aab7", "egg carton-0e0dcb4ecdf92e37", "egg-688e78a44cefb142", "egg2-205ae9b80988d4b1", "egg3-da39fb088198419b", "egg4-fc8097f80e4b2cbe", "egg5-7a615aef1c3a47b5", "white bowl3-11c6cc86c3de1854", "egg6-a3582ca144add83a", "fork5-32a1e3a313eaa6b7", "smaller baking tray-2c9c033205f0a1b5", "white baking tray-a81532f9e3d87905", "apron-f40a30089727ad06", "spoon3-f0ea632888bf9f67", "oven mitts-1f178a7dad4a1f3a", "container lid-b094d8e1f9b9a3cc", "bag of onions-14e044d9f9c02dd4", "one onion-a20caf9e35684670", "onion half-3dc3bb9a9acac0b3", "tea towel-2f785cd10ad2da5c", "tea towel2-fdb00402c2d6473b", "fork6-2ed7aa38d9a7aa5c", "knife4-c47a15c6ca400d00", "knife5-10bb43e6546b75c9", "pizza dough-d086c99d3d0752c8", "tissue box-01cec62e4005940b", "yellow glass-b0036c30465b1dd5", "cork pot stand-6aa9a0ad87fcd995", "fork7-efea21a6c8ba72ef", "spoon4-48bc0864caeb5f85" ], "reasoning": "1. I watched the video to find that the user starts to knead the dough (pizza dough-d086c99d3d0752c8) at 38:45.\n2. Initially, the user kneaded the dough (pizza dough-d086c99d3d0752c8) from 38:45-39:15, for a total of 00:30.\n3. The user takes a brief break before resuming the kneading process from 39:16-40:33, which is 01:17.\n4. After another pause, the user kneaded the dough (pizza dough-d086c99d3d0752c8) from 40:34-42:54, totaling 02:20. \n5. The user stops briefly and then kneads again from 42:58-43:55, lasting 00:57.\n6. Then, from 43:55-44:00 the user walks to the sink to wash their hands, ending the kneading process.\n7. I then observed that the video is a total of 01:14:45.\n8. The math to get the answer is 01:14:45 - 05:04 = 1:09:41.", "referenced_object_masks": [ "pizza dough-d086c99d3d0752c8" ] }, { "key": "P03-20240216-185832:25cd32a7661eca45a1d8eacd15649a85cb1fa06f", "video_id": "P03-20240216-185832", "question": "While peeling the third potato, in what direction, and by approximately how many degrees would I need to turn to face towards the sink?", "answer_choice_0": "Right 90 degrees", "answer_choice_1": "Left 90 degrees", "answer_choice_2": "Right 30 degrees", "answer_choice_3": "Left 30 degrees", "answer_choice_4": "Right 180 degrees", "answer_id": 0, "answer": "Right 90 degrees", "question_type": [ "Egocentric & Spatial Perception", "Situational Awareness", "Action Recognition", "State Changes", "Object Recognition" ], "objects_of_interest": [ "bag of spring greens-d94464beb250fb38", "bag of kale-9cd0eb0af15afe72", "vegetable-65a2e3a47c6302d0", "box of mushrooms-5480891ea71366cd", "spring onion-fd3fe499e30ff53c", "box of food-3b09d3574748315b", "cauliflower-705371010f925839", "large shopping bag-ab44271b31236a56", "bag of salad-a96d954fabf438d9", "bag of mozzarella-8deaddf9cb5a1c14", "single cream pot-4ffbd866d873e1e6", "spaghetti bag-f8193f01def0ea7e", "box of grapes-99d179636e4e26b3", "bananas bag-afa30e91f9a2b2ea", "bag of potatoes-8664af8e0c4512fa", "bag of mandarins-57e9d0df0122ca8b", "bag of mandarin2-05b21532ace5fc01", "garlic-124c0fee31ec5bc3", "bag of pears-2691488edebe6c10", "bag of kiwis-270bfcee080ea7af", "squash-bf5700a01518efad", "pack of pasta-107b336f60723036", "box of yeast-e96b26f5f93c789f", "bag of lentils-fc82371146cac8f8", "paper food waste bag-5f3983f5e379ca63", "superglue box-63c2e054d6869957", "bread-6039bb22e14eacbd", "tomato concentrated puree-8b68051c77a6200d", "coconut milk can-06a9fc40ecc82644", "tomato sauce bottle-ea967e3e7ff63770", "tomato sauce bottle2-ea67d9e08b7b48aa", "bottle with green lid-f75030cbed5ecaad", "tomato sauce bottle3-ca3fa9d369bd7a9b", "paper receipt-ea5837ad2dd012b3", "large shopping bag2-c1a67a7d60bed8f5", "black shoe left-b9c8a09619f10cc7", "black shoe right-fb84ffe2c21f414c", "dirty plastic spatula-39bf8dc418d3c568", "plate-ccac08aa8e50801a", "plate2-3bfd83d35c5b7ce4", "saucepan-da8b913a944ea3a1", "smaller saucepan-5b6f8054c1661133", "small bowl-f354cc2d8e7f353f", "white bowl-297b16122e17d1ba", "small bowl2-bf5691912176ed31", "coffee cup-1856f70dbc366538", "cup2-34d665c224823126", "plastic box container-16c45dc5640b85eb", "plastic container2-66b0bd40422b43a6", "cloth-e905582ea3e0d5cb", "chopstick1-384d046924eb0850", "chopstick2-740e52f3bd7b0104", "chopstick3-83bfc567ece98531", "chopstick4-00a31c1bcd4d68b3", "chopstick5-af86dc66d58998df", "chopstick6-7e52c88efe127a22", "fork1-c48edec98d98bae4", "fork2-955ca204be163621", "knife1-30debca82d73d304", "spoon1-3c0a243af93fba2e", "spoon2-a3da42ad4dd94bd9", "red spoon-546cfa010b78f974", "metal colander-f9f1b975f21e8568", "large bowl-e297ef66a083c612", "plastic colander-7d62c20f4ec67122", "bowl3-bfd73628becf1677", "bowl4-28b8b3314216e252", "bowl5-1309234381bf1d70", "white bowl2-08d9cb477606e8a6", "bowl6-c94a84c2091f7a90", "cup3-0bacf6cf9105f10d", "glass-3cefa6050e618946", "glass2-71459ef5d1e087a8", "glass3-13dffc2ecc98b459", "glass4-125b49dacdb84198", "glass5-5552f629a26efb08", "white medium sized bowl-173a8b87bb07cb50", "glass ramekin-27976f43e25f4162", "small grey plate-018ce8550f0704b7", "two identical plastic box lids-317227f0df46f5b1", "red plate-3e5c9ed8e2252755", "one small grey plate-252423245bc68222", "v60 glass filter-bb96da8cfbd03752", "v60 metal filter-7c833fe84fc512ee", "plastic food bag-7dca5a67acbec046", "plastic food box-a6ec4ab01b9766d3", "plastic food box lid-0da741d3f28b4d44", "fork3-dc4e1cc6003d165c", "dishwasher brush-8d37bfb2df0fa22c", "fork4-866bca4acda794b3", "white plate-ff6e78f4c64444d5", "grater-7fa171dbaf625b84", "small dirty bowl-a19b985558c5e8c8", "green coffee cup1-c9c6724b8274ff67", "wooden spoon-69be1eab24a0cc61", "metal teaspoon-5846ee31ef4a5033", "checkered cup-bdb131456dfe461a", "coffee jug-7feb82765554d728", "chopstick-cb987a664aeeba91", "one food waste paper bag-d21c1a1f6f84d7fb", "eggshell-c08839df3778e8ff", "blue plastic bag-3422821684ce83ab", "food waste bin-4bad12162c13724a", "sponge-0dd841b9997653d1", "washing up liquid bottle-a3e90fc19a4317dd", "pan-1dd5d7741c7265ed", "large bowl2-75fe73e9a85a889a", "plastic calendar-63c80e3f9cdf75b5", "potato1-a2e73c6ae3d8264d", "potato2-1e44164bfbaa9cad", "potato3-a4ace2eabb1c6a34", "potato4-aa365386e5442e73", "potato5-140dcd14519daf63", "potato6-1ade5ad88a5fb7c8", "potato7-a8956f52788169a1", "potato8-7ce9f636df79b299", "chopping board-0fa725b89e5753bd", "knife2-6e02fef99df274b3", "knife3-5aa57c591cb06a1d", "cloth2-08bd78e9df7e00fd", "potato9-f9df4d0f9b83b31c", "potato10-a2ed32ce73f688fc", "potato11-310f57a9a9cf7339", "potato12-e57379c4dc03dc67", "can of olive oil-3ded77db4988444c", "salt-32d76cb271cd9d21", "olive oil can cap-ac705c30088b78fd", "glass lid-924ba845005deac0", "hand wash dispenser-e454f6197c20d6d4", "plate3-dc739385c55942c8", "plate4-3277214c484e2b40", "orange cloth-a316ea7c38bb7e34", "bag of flour-55b8a2e0f7b61d16", "scale-71b4f54bf2930f23", "scissors-1cc56ec76660d927", "scissor safety cover-d689b523849381f6", "bag of flour top of bag-1582a3609c7495e6", "clip-cb17faf107a72a23", "large plastic box-5adeb358c8c9ea55", "yeast box2-a8228c1e3ffccb72", "folded sachet-fec3b7c95948d0b2", "wooden spoon2-9e1e2402cdbdda71", "wooden pizza peel-dec25775f59bf7f8", "wooden rolling pin-bc989fd347e588a1", "square plastic container-680dc3051eb0c50f", "cake mould-db4ac615fc94220f", "dough scraper-db658d2e4487aab7", "egg carton-0e0dcb4ecdf92e37", "egg-688e78a44cefb142", "egg2-205ae9b80988d4b1", "egg3-da39fb088198419b", "egg4-fc8097f80e4b2cbe", "egg5-7a615aef1c3a47b5", "white bowl3-11c6cc86c3de1854", "egg6-a3582ca144add83a", "fork5-32a1e3a313eaa6b7", "smaller baking tray-2c9c033205f0a1b5", "white baking tray-a81532f9e3d87905", "apron-f40a30089727ad06", "spoon3-f0ea632888bf9f67", "oven mitts-1f178a7dad4a1f3a", "container lid-b094d8e1f9b9a3cc", "bag of onions-14e044d9f9c02dd4", "one onion-a20caf9e35684670", "onion half-3dc3bb9a9acac0b3", "tea towel-2f785cd10ad2da5c", "tea towel2-fdb00402c2d6473b", "fork6-2ed7aa38d9a7aa5c", "knife4-c47a15c6ca400d00", "knife5-10bb43e6546b75c9", "pizza dough-d086c99d3d0752c8", "tissue box-01cec62e4005940b", "yellow glass-b0036c30465b1dd5", "cork pot stand-6aa9a0ad87fcd995", "fork7-efea21a6c8ba72ef", "spoon4-48bc0864caeb5f85" ], "reasoning": "1. I watched the video to see the user begin peeling potatoes (bag of potatoes-8664af8e0c4512fa) at 14:03.\n2. They then peel the second potato (potato2-1e44164bfbaa9cad) from 14:41-15:04.\n3. The user peels the third potato (potato3-a4ace2eabb1c6a34) from 15:06-15:40.\n4. At this time, I observed that the user is standing at the counter, with a rice cooker and multiple other containers .\n5. Prior to the user peeling the third potato (potato3-a4ace2eabb1c6a34), I saw the user peeling the first potato (potato1-a2e73c6ae3d8264d) from 14:03-14:13 while facing the sink .\n6. Then, I watched the user turn 90 degrees to the left from 14:13-14:15 to get to their position when they peel the third potato (potato3-a4ace2eabb1c6a34). This means that the user would have to turn 90 degrees to the right to turn back to face the sink .", "referenced_object_masks": [ "bag of potatoes-8664af8e0c4512fa", "potato2-1e44164bfbaa9cad", "potato1-a2e73c6ae3d8264d", "potato3-a4ace2eabb1c6a34" ] }, { "key": "P03-20240216-185832:4d2415a0bf8c99622e1c34c178fa87fdad6fec7a", "video_id": "P03-20240216-185832", "question": "If I doubled the number of potatoes I cooked, how many more potatoes would I have cooked than eggs?", "answer_choice_0": "30", "answer_choice_1": "6", "answer_choice_2": "18", "answer_choice_3": "12", "answer_choice_4": "24", "answer_id": 2, "answer": "18", "question_type": [ "Counterfactual", "Counting", "Numerical Reasoning", "Object Recognition" ], "objects_of_interest": [ "bag of spring greens-d94464beb250fb38", "bag of kale-9cd0eb0af15afe72", "vegetable-65a2e3a47c6302d0", "box of mushrooms-5480891ea71366cd", "spring onion-fd3fe499e30ff53c", "box of food-3b09d3574748315b", "cauliflower-705371010f925839", "large shopping bag-ab44271b31236a56", "bag of salad-a96d954fabf438d9", "bag of mozzarella-8deaddf9cb5a1c14", "single cream pot-4ffbd866d873e1e6", "spaghetti bag-f8193f01def0ea7e", "box of grapes-99d179636e4e26b3", "bananas bag-afa30e91f9a2b2ea", "bag of potatoes-8664af8e0c4512fa", "bag of mandarins-57e9d0df0122ca8b", "bag of mandarin2-05b21532ace5fc01", "garlic-124c0fee31ec5bc3", "bag of pears-2691488edebe6c10", "bag of kiwis-270bfcee080ea7af", "squash-bf5700a01518efad", "pack of pasta-107b336f60723036", "box of yeast-e96b26f5f93c789f", "bag of lentils-fc82371146cac8f8", "paper food waste bag-5f3983f5e379ca63", "superglue box-63c2e054d6869957", "bread-6039bb22e14eacbd", "tomato concentrated puree-8b68051c77a6200d", "coconut milk can-06a9fc40ecc82644", "tomato sauce bottle-ea967e3e7ff63770", "tomato sauce bottle2-ea67d9e08b7b48aa", "bottle with green lid-f75030cbed5ecaad", "tomato sauce bottle3-ca3fa9d369bd7a9b", "paper receipt-ea5837ad2dd012b3", "large shopping bag2-c1a67a7d60bed8f5", "black shoe left-b9c8a09619f10cc7", "black shoe right-fb84ffe2c21f414c", "dirty plastic spatula-39bf8dc418d3c568", "plate-ccac08aa8e50801a", "plate2-3bfd83d35c5b7ce4", "saucepan-da8b913a944ea3a1", "smaller saucepan-5b6f8054c1661133", "small bowl-f354cc2d8e7f353f", "white bowl-297b16122e17d1ba", "small bowl2-bf5691912176ed31", "coffee cup-1856f70dbc366538", "cup2-34d665c224823126", "plastic box container-16c45dc5640b85eb", "plastic container2-66b0bd40422b43a6", "cloth-e905582ea3e0d5cb", "chopstick1-384d046924eb0850", "chopstick2-740e52f3bd7b0104", "chopstick3-83bfc567ece98531", "chopstick4-00a31c1bcd4d68b3", "chopstick5-af86dc66d58998df", "chopstick6-7e52c88efe127a22", "fork1-c48edec98d98bae4", "fork2-955ca204be163621", "knife1-30debca82d73d304", "spoon1-3c0a243af93fba2e", "spoon2-a3da42ad4dd94bd9", "red spoon-546cfa010b78f974", "metal colander-f9f1b975f21e8568", "large bowl-e297ef66a083c612", "plastic colander-7d62c20f4ec67122", "bowl3-bfd73628becf1677", "bowl4-28b8b3314216e252", "bowl5-1309234381bf1d70", "white bowl2-08d9cb477606e8a6", "bowl6-c94a84c2091f7a90", "cup3-0bacf6cf9105f10d", "glass-3cefa6050e618946", "glass2-71459ef5d1e087a8", "glass3-13dffc2ecc98b459", "glass4-125b49dacdb84198", "glass5-5552f629a26efb08", "white medium sized bowl-173a8b87bb07cb50", "glass ramekin-27976f43e25f4162", "small grey plate-018ce8550f0704b7", "two identical plastic box lids-317227f0df46f5b1", "red plate-3e5c9ed8e2252755", "one small grey plate-252423245bc68222", "v60 glass filter-bb96da8cfbd03752", "v60 metal filter-7c833fe84fc512ee", "plastic food bag-7dca5a67acbec046", "plastic food box-a6ec4ab01b9766d3", "plastic food box lid-0da741d3f28b4d44", "fork3-dc4e1cc6003d165c", "dishwasher brush-8d37bfb2df0fa22c", "fork4-866bca4acda794b3", "white plate-ff6e78f4c64444d5", "grater-7fa171dbaf625b84", "small dirty bowl-a19b985558c5e8c8", "green coffee cup1-c9c6724b8274ff67", "wooden spoon-69be1eab24a0cc61", "metal teaspoon-5846ee31ef4a5033", "checkered cup-bdb131456dfe461a", "coffee jug-7feb82765554d728", "chopstick-cb987a664aeeba91", "one food waste paper bag-d21c1a1f6f84d7fb", "eggshell-c08839df3778e8ff", "blue plastic bag-3422821684ce83ab", "food waste bin-4bad12162c13724a", "sponge-0dd841b9997653d1", "washing up liquid bottle-a3e90fc19a4317dd", "pan-1dd5d7741c7265ed", "large bowl2-75fe73e9a85a889a", "plastic calendar-63c80e3f9cdf75b5", "potato1-a2e73c6ae3d8264d", "potato2-1e44164bfbaa9cad", "potato3-a4ace2eabb1c6a34", "potato4-aa365386e5442e73", "potato5-140dcd14519daf63", "potato6-1ade5ad88a5fb7c8", "potato7-a8956f52788169a1", "potato8-7ce9f636df79b299", "chopping board-0fa725b89e5753bd", "knife2-6e02fef99df274b3", "knife3-5aa57c591cb06a1d", "cloth2-08bd78e9df7e00fd", "potato9-f9df4d0f9b83b31c", "potato10-a2ed32ce73f688fc", "potato11-310f57a9a9cf7339", "potato12-e57379c4dc03dc67", "can of olive oil-3ded77db4988444c", "salt-32d76cb271cd9d21", "olive oil can cap-ac705c30088b78fd", "glass lid-924ba845005deac0", "hand wash dispenser-e454f6197c20d6d4", "plate3-dc739385c55942c8", "plate4-3277214c484e2b40", "orange cloth-a316ea7c38bb7e34", "bag of flour-55b8a2e0f7b61d16", "scale-71b4f54bf2930f23", "scissors-1cc56ec76660d927", "scissor safety cover-d689b523849381f6", "bag of flour top of bag-1582a3609c7495e6", "clip-cb17faf107a72a23", "large plastic box-5adeb358c8c9ea55", "yeast box2-a8228c1e3ffccb72", "folded sachet-fec3b7c95948d0b2", "wooden spoon2-9e1e2402cdbdda71", "wooden pizza peel-dec25775f59bf7f8", "wooden rolling pin-bc989fd347e588a1", "square plastic container-680dc3051eb0c50f", "cake mould-db4ac615fc94220f", "dough scraper-db658d2e4487aab7", "egg carton-0e0dcb4ecdf92e37", "egg-688e78a44cefb142", "egg2-205ae9b80988d4b1", "egg3-da39fb088198419b", "egg4-fc8097f80e4b2cbe", "egg5-7a615aef1c3a47b5", "white bowl3-11c6cc86c3de1854", "egg6-a3582ca144add83a", "fork5-32a1e3a313eaa6b7", "smaller baking tray-2c9c033205f0a1b5", "white baking tray-a81532f9e3d87905", "apron-f40a30089727ad06", "spoon3-f0ea632888bf9f67", "oven mitts-1f178a7dad4a1f3a", "container lid-b094d8e1f9b9a3cc", "bag of onions-14e044d9f9c02dd4", "one onion-a20caf9e35684670", "onion half-3dc3bb9a9acac0b3", "tea towel-2f785cd10ad2da5c", "tea towel2-fdb00402c2d6473b", "fork6-2ed7aa38d9a7aa5c", "knife4-c47a15c6ca400d00", "knife5-10bb43e6546b75c9", "pizza dough-d086c99d3d0752c8", "tissue box-01cec62e4005940b", "yellow glass-b0036c30465b1dd5", "cork pot stand-6aa9a0ad87fcd995", "fork7-efea21a6c8ba72ef", "spoon4-48bc0864caeb5f85" ], "reasoning": "1. I watched to see how many potatoes and eggs the user cooked with. From 19:16-20:44, I can see the user chop 8 potatoes and add them to a pan (pan-1dd5d7741c7265ed) to cook.\n2. From 24:03-25:00 I can see the user chop 4 more potatoes and add them to the pan (pan-1dd5d7741c7265ed) to cook. I did not see the user add any more potatoes to the pan.\n3. 2*(8+4)=24.\n4. From 48:01-48:47 I can see the user crack 6 eggs into a bowl (white bowl3-11c6cc86c3de1854).\n5. At 64:32 the user pours the eggs from the bowl (white bowl3-11c6cc86c3de1854) over the potatoes in the pan (pan-1dd5d7741c7265ed). They do not add any more eggs.\n6. 24-6=18.", "referenced_object_masks": [ "pan-1dd5d7741c7265ed", "white bowl3-11c6cc86c3de1854" ] }, { "key": "P03-20240216-185832:528712f8425ab996599e489238a39451a2379201", "video_id": "P03-20240216-185832", "question": "If I had stored the new tube of tomato paste 2 feet higher, where would I have stored it?", "answer_choice_0": "The bottom drawer to the right of the stove", "answer_choice_1": "The top shelf of the fridge", "answer_choice_2": "The bottom shelf of the cabinet above the stove to the left", "answer_choice_3": "The top shelf of the cabinet to the right of the stove", "answer_choice_4": "The top drawer of the cabinet to the right of the stove.", "answer_id": 4, "answer": "The top drawer of the cabinet to the right of the stove.", "question_type": [ "Counterfactual", "Egocentric & Spatial Perception", "Object Recognition" ], "objects_of_interest": [ "bag of spring greens-d94464beb250fb38", "bag of kale-9cd0eb0af15afe72", "vegetable-65a2e3a47c6302d0", "box of mushrooms-5480891ea71366cd", "spring onion-fd3fe499e30ff53c", "box of food-3b09d3574748315b", "cauliflower-705371010f925839", "large shopping bag-ab44271b31236a56", "bag of salad-a96d954fabf438d9", "bag of mozzarella-8deaddf9cb5a1c14", "single cream pot-4ffbd866d873e1e6", "spaghetti bag-f8193f01def0ea7e", "box of grapes-99d179636e4e26b3", "bananas bag-afa30e91f9a2b2ea", "bag of potatoes-8664af8e0c4512fa", "bag of mandarins-57e9d0df0122ca8b", "bag of mandarin2-05b21532ace5fc01", "garlic-124c0fee31ec5bc3", "bag of pears-2691488edebe6c10", "bag of kiwis-270bfcee080ea7af", "squash-bf5700a01518efad", "pack of pasta-107b336f60723036", "box of yeast-e96b26f5f93c789f", "bag of lentils-fc82371146cac8f8", "paper food waste bag-5f3983f5e379ca63", "superglue box-63c2e054d6869957", "bread-6039bb22e14eacbd", "tomato concentrated puree-8b68051c77a6200d", "coconut milk can-06a9fc40ecc82644", "tomato sauce bottle-ea967e3e7ff63770", "tomato sauce bottle2-ea67d9e08b7b48aa", "bottle with green lid-f75030cbed5ecaad", "tomato sauce bottle3-ca3fa9d369bd7a9b", "paper receipt-ea5837ad2dd012b3", "large shopping bag2-c1a67a7d60bed8f5", "black shoe left-b9c8a09619f10cc7", "black shoe right-fb84ffe2c21f414c", "dirty plastic spatula-39bf8dc418d3c568", "plate-ccac08aa8e50801a", "plate2-3bfd83d35c5b7ce4", "saucepan-da8b913a944ea3a1", "smaller saucepan-5b6f8054c1661133", "small bowl-f354cc2d8e7f353f", "white bowl-297b16122e17d1ba", "small bowl2-bf5691912176ed31", "coffee cup-1856f70dbc366538", "cup2-34d665c224823126", "plastic box container-16c45dc5640b85eb", "plastic container2-66b0bd40422b43a6", "cloth-e905582ea3e0d5cb", "chopstick1-384d046924eb0850", "chopstick2-740e52f3bd7b0104", "chopstick3-83bfc567ece98531", "chopstick4-00a31c1bcd4d68b3", "chopstick5-af86dc66d58998df", "chopstick6-7e52c88efe127a22", "fork1-c48edec98d98bae4", "fork2-955ca204be163621", "knife1-30debca82d73d304", "spoon1-3c0a243af93fba2e", "spoon2-a3da42ad4dd94bd9", "red spoon-546cfa010b78f974", "metal colander-f9f1b975f21e8568", "large bowl-e297ef66a083c612", "plastic colander-7d62c20f4ec67122", "bowl3-bfd73628becf1677", "bowl4-28b8b3314216e252", "bowl5-1309234381bf1d70", "white bowl2-08d9cb477606e8a6", "bowl6-c94a84c2091f7a90", "cup3-0bacf6cf9105f10d", "glass-3cefa6050e618946", "glass2-71459ef5d1e087a8", "glass3-13dffc2ecc98b459", "glass4-125b49dacdb84198", "glass5-5552f629a26efb08", "white medium sized bowl-173a8b87bb07cb50", "glass ramekin-27976f43e25f4162", "small grey plate-018ce8550f0704b7", "two identical plastic box lids-317227f0df46f5b1", "red plate-3e5c9ed8e2252755", "one small grey plate-252423245bc68222", "v60 glass filter-bb96da8cfbd03752", "v60 metal filter-7c833fe84fc512ee", "plastic food bag-7dca5a67acbec046", "plastic food box-a6ec4ab01b9766d3", "plastic food box lid-0da741d3f28b4d44", "fork3-dc4e1cc6003d165c", "dishwasher brush-8d37bfb2df0fa22c", "fork4-866bca4acda794b3", "white plate-ff6e78f4c64444d5", "grater-7fa171dbaf625b84", "small dirty bowl-a19b985558c5e8c8", "green coffee cup1-c9c6724b8274ff67", "wooden spoon-69be1eab24a0cc61", "metal teaspoon-5846ee31ef4a5033", "checkered cup-bdb131456dfe461a", "coffee jug-7feb82765554d728", "chopstick-cb987a664aeeba91", "one food waste paper bag-d21c1a1f6f84d7fb", "eggshell-c08839df3778e8ff", "blue plastic bag-3422821684ce83ab", "food waste bin-4bad12162c13724a", "sponge-0dd841b9997653d1", "washing up liquid bottle-a3e90fc19a4317dd", "pan-1dd5d7741c7265ed", "large bowl2-75fe73e9a85a889a", "plastic calendar-63c80e3f9cdf75b5", "potato1-a2e73c6ae3d8264d", "potato2-1e44164bfbaa9cad", "potato3-a4ace2eabb1c6a34", "potato4-aa365386e5442e73", "potato5-140dcd14519daf63", "potato6-1ade5ad88a5fb7c8", "potato7-a8956f52788169a1", "potato8-7ce9f636df79b299", "chopping board-0fa725b89e5753bd", "knife2-6e02fef99df274b3", "knife3-5aa57c591cb06a1d", "cloth2-08bd78e9df7e00fd", "potato9-f9df4d0f9b83b31c", "potato10-a2ed32ce73f688fc", "potato11-310f57a9a9cf7339", "potato12-e57379c4dc03dc67", "can of olive oil-3ded77db4988444c", "salt-32d76cb271cd9d21", "olive oil can cap-ac705c30088b78fd", "glass lid-924ba845005deac0", "hand wash dispenser-e454f6197c20d6d4", "plate3-dc739385c55942c8", "plate4-3277214c484e2b40", "orange cloth-a316ea7c38bb7e34", "bag of flour-55b8a2e0f7b61d16", "scale-71b4f54bf2930f23", "scissors-1cc56ec76660d927", "scissor safety cover-d689b523849381f6", "bag of flour top of bag-1582a3609c7495e6", "clip-cb17faf107a72a23", "large plastic box-5adeb358c8c9ea55", "yeast box2-a8228c1e3ffccb72", "folded sachet-fec3b7c95948d0b2", "wooden spoon2-9e1e2402cdbdda71", "wooden pizza peel-dec25775f59bf7f8", "wooden rolling pin-bc989fd347e588a1", "square plastic container-680dc3051eb0c50f", "cake mould-db4ac615fc94220f", "dough scraper-db658d2e4487aab7", "egg carton-0e0dcb4ecdf92e37", "egg-688e78a44cefb142", "egg2-205ae9b80988d4b1", "egg3-da39fb088198419b", "egg4-fc8097f80e4b2cbe", "egg5-7a615aef1c3a47b5", "white bowl3-11c6cc86c3de1854", "egg6-a3582ca144add83a", "fork5-32a1e3a313eaa6b7", "smaller baking tray-2c9c033205f0a1b5", "white baking tray-a81532f9e3d87905", "apron-f40a30089727ad06", "spoon3-f0ea632888bf9f67", "oven mitts-1f178a7dad4a1f3a", "container lid-b094d8e1f9b9a3cc", "bag of onions-14e044d9f9c02dd4", "one onion-a20caf9e35684670", "onion half-3dc3bb9a9acac0b3", "tea towel-2f785cd10ad2da5c", "tea towel2-fdb00402c2d6473b", "fork6-2ed7aa38d9a7aa5c", "knife4-c47a15c6ca400d00", "knife5-10bb43e6546b75c9", "pizza dough-d086c99d3d0752c8", "tissue box-01cec62e4005940b", "yellow glass-b0036c30465b1dd5", "cork pot stand-6aa9a0ad87fcd995", "fork7-efea21a6c8ba72ef", "spoon4-48bc0864caeb5f85" ], "reasoning": "1. I watched for the user to interact with a tube of tomato paste. I saw them pull a tube of tomato paste (tomato concentrated puree-8b68051c77a6200d) from a shopping bag (large shopping bag2-c1a67a7d60bed8f5) at 02:10.\n2. I then watched to see where the user stored the tomato paste (tomato concentrated puree-8b68051c77a6200d). They first placed it down on the counter at 02:18, but then placed it in the bottom drawer of the cabinet below, to the right of the stove, at 02:23.\n3. I saw that there were two drawers above the one where the tomato paste (tomato concentrated puree-8b68051c77a6200d) was stored. If the paste (tomato concentrated puree-8b68051c77a6200d) were stored 2 feet higher, it would be in the top drawer to the right of the stove.", "referenced_object_masks": [ "large shopping bag2-c1a67a7d60bed8f5", "tomato concentrated puree-8b68051c77a6200d" ] }, { "key": "P03-20240216-185832:6d2b61b4a51a0d9ef37d7823cd8e5a21f6ab4d10", "video_id": "P03-20240216-185832", "question": "How much time passes between the first and last time I add oil to the potatoes?", "answer_choice_0": "35:03", "answer_choice_1": "33:49", "answer_choice_2": "35:12", "answer_choice_3": "34:54", "answer_choice_4": "34:14", "answer_id": 4, "answer": "34:14", "question_type": [ "Temporal Reasoning", "Action Recognition", "Numerical Reasoning" ], "objects_of_interest": [ "bag of spring greens-d94464beb250fb38", "bag of kale-9cd0eb0af15afe72", "vegetable-65a2e3a47c6302d0", "box of mushrooms-5480891ea71366cd", "spring onion-fd3fe499e30ff53c", "box of food-3b09d3574748315b", "cauliflower-705371010f925839", "large shopping bag-ab44271b31236a56", "bag of salad-a96d954fabf438d9", "bag of mozzarella-8deaddf9cb5a1c14", "single cream pot-4ffbd866d873e1e6", "spaghetti bag-f8193f01def0ea7e", "box of grapes-99d179636e4e26b3", "bananas bag-afa30e91f9a2b2ea", "bag of potatoes-8664af8e0c4512fa", "bag of mandarins-57e9d0df0122ca8b", "bag of mandarin2-05b21532ace5fc01", "garlic-124c0fee31ec5bc3", "bag of pears-2691488edebe6c10", "bag of kiwis-270bfcee080ea7af", "squash-bf5700a01518efad", "pack of pasta-107b336f60723036", "box of yeast-e96b26f5f93c789f", "bag of lentils-fc82371146cac8f8", "paper food waste bag-5f3983f5e379ca63", "superglue box-63c2e054d6869957", "bread-6039bb22e14eacbd", "tomato concentrated puree-8b68051c77a6200d", "coconut milk can-06a9fc40ecc82644", "tomato sauce bottle-ea967e3e7ff63770", "tomato sauce bottle2-ea67d9e08b7b48aa", "bottle with green lid-f75030cbed5ecaad", "tomato sauce bottle3-ca3fa9d369bd7a9b", "paper receipt-ea5837ad2dd012b3", "large shopping bag2-c1a67a7d60bed8f5", "black shoe left-b9c8a09619f10cc7", "black shoe right-fb84ffe2c21f414c", "dirty plastic spatula-39bf8dc418d3c568", "plate-ccac08aa8e50801a", "plate2-3bfd83d35c5b7ce4", "saucepan-da8b913a944ea3a1", "smaller saucepan-5b6f8054c1661133", "small bowl-f354cc2d8e7f353f", "white bowl-297b16122e17d1ba", "small bowl2-bf5691912176ed31", "coffee cup-1856f70dbc366538", "cup2-34d665c224823126", "plastic box container-16c45dc5640b85eb", "plastic container2-66b0bd40422b43a6", "cloth-e905582ea3e0d5cb", "chopstick1-384d046924eb0850", "chopstick2-740e52f3bd7b0104", "chopstick3-83bfc567ece98531", "chopstick4-00a31c1bcd4d68b3", "chopstick5-af86dc66d58998df", "chopstick6-7e52c88efe127a22", "fork1-c48edec98d98bae4", "fork2-955ca204be163621", "knife1-30debca82d73d304", "spoon1-3c0a243af93fba2e", "spoon2-a3da42ad4dd94bd9", "red spoon-546cfa010b78f974", "metal colander-f9f1b975f21e8568", "large bowl-e297ef66a083c612", "plastic colander-7d62c20f4ec67122", "bowl3-bfd73628becf1677", "bowl4-28b8b3314216e252", "bowl5-1309234381bf1d70", "white bowl2-08d9cb477606e8a6", "bowl6-c94a84c2091f7a90", "cup3-0bacf6cf9105f10d", "glass-3cefa6050e618946", "glass2-71459ef5d1e087a8", "glass3-13dffc2ecc98b459", "glass4-125b49dacdb84198", "glass5-5552f629a26efb08", "white medium sized bowl-173a8b87bb07cb50", "glass ramekin-27976f43e25f4162", "small grey plate-018ce8550f0704b7", "two identical plastic box lids-317227f0df46f5b1", "red plate-3e5c9ed8e2252755", "one small grey plate-252423245bc68222", "v60 glass filter-bb96da8cfbd03752", "v60 metal filter-7c833fe84fc512ee", "plastic food bag-7dca5a67acbec046", "plastic food box-a6ec4ab01b9766d3", "plastic food box lid-0da741d3f28b4d44", "fork3-dc4e1cc6003d165c", "dishwasher brush-8d37bfb2df0fa22c", "fork4-866bca4acda794b3", "white plate-ff6e78f4c64444d5", "grater-7fa171dbaf625b84", "small dirty bowl-a19b985558c5e8c8", "green coffee cup1-c9c6724b8274ff67", "wooden spoon-69be1eab24a0cc61", "metal teaspoon-5846ee31ef4a5033", "checkered cup-bdb131456dfe461a", "coffee jug-7feb82765554d728", "chopstick-cb987a664aeeba91", "one food waste paper bag-d21c1a1f6f84d7fb", "eggshell-c08839df3778e8ff", "blue plastic bag-3422821684ce83ab", "food waste bin-4bad12162c13724a", "sponge-0dd841b9997653d1", "washing up liquid bottle-a3e90fc19a4317dd", "pan-1dd5d7741c7265ed", "large bowl2-75fe73e9a85a889a", "plastic calendar-63c80e3f9cdf75b5", "potato1-a2e73c6ae3d8264d", "potato2-1e44164bfbaa9cad", "potato3-a4ace2eabb1c6a34", "potato4-aa365386e5442e73", "potato5-140dcd14519daf63", "potato6-1ade5ad88a5fb7c8", "potato7-a8956f52788169a1", "potato8-7ce9f636df79b299", "chopping board-0fa725b89e5753bd", "knife2-6e02fef99df274b3", "knife3-5aa57c591cb06a1d", "cloth2-08bd78e9df7e00fd", "potato9-f9df4d0f9b83b31c", "potato10-a2ed32ce73f688fc", "potato11-310f57a9a9cf7339", "potato12-e57379c4dc03dc67", "can of olive oil-3ded77db4988444c", "salt-32d76cb271cd9d21", "olive oil can cap-ac705c30088b78fd", "glass lid-924ba845005deac0", "hand wash dispenser-e454f6197c20d6d4", "plate3-dc739385c55942c8", "plate4-3277214c484e2b40", "orange cloth-a316ea7c38bb7e34", "bag of flour-55b8a2e0f7b61d16", "scale-71b4f54bf2930f23", "scissors-1cc56ec76660d927", "scissor safety cover-d689b523849381f6", "bag of flour top of bag-1582a3609c7495e6", "clip-cb17faf107a72a23", "large plastic box-5adeb358c8c9ea55", "yeast box2-a8228c1e3ffccb72", "folded sachet-fec3b7c95948d0b2", "wooden spoon2-9e1e2402cdbdda71", "wooden pizza peel-dec25775f59bf7f8", "wooden rolling pin-bc989fd347e588a1", "square plastic container-680dc3051eb0c50f", "cake mould-db4ac615fc94220f", "dough scraper-db658d2e4487aab7", "egg carton-0e0dcb4ecdf92e37", "egg-688e78a44cefb142", "egg2-205ae9b80988d4b1", "egg3-da39fb088198419b", "egg4-fc8097f80e4b2cbe", "egg5-7a615aef1c3a47b5", "white bowl3-11c6cc86c3de1854", "egg6-a3582ca144add83a", "fork5-32a1e3a313eaa6b7", "smaller baking tray-2c9c033205f0a1b5", "white baking tray-a81532f9e3d87905", "apron-f40a30089727ad06", "spoon3-f0ea632888bf9f67", "oven mitts-1f178a7dad4a1f3a", "container lid-b094d8e1f9b9a3cc", "bag of onions-14e044d9f9c02dd4", "one onion-a20caf9e35684670", "onion half-3dc3bb9a9acac0b3", "tea towel-2f785cd10ad2da5c", "tea towel2-fdb00402c2d6473b", "fork6-2ed7aa38d9a7aa5c", "knife4-c47a15c6ca400d00", "knife5-10bb43e6546b75c9", "pizza dough-d086c99d3d0752c8", "tissue box-01cec62e4005940b", "yellow glass-b0036c30465b1dd5", "cork pot stand-6aa9a0ad87fcd995", "fork7-efea21a6c8ba72ef", "spoon4-48bc0864caeb5f85" ], "reasoning": "1. I watched for the user to add oil to potatoes.\n2. I saw the user add pour oil from a can (can of olive oil-3ded77db4988444c) over potatoes in a pan (pan-1dd5d7741c7265ed) at 25:30.\n3. I continued watching until I saw the user add oil from the can (can of olive oil-3ded77db4988444c) to the potatoes in the pan (pan-1dd5d7741c7265ed) for a last time at 59:44. I did not see the user add oil again after this point.\n4. 59:44-25:30=34:14.", "referenced_object_masks": [ "can of olive oil-3ded77db4988444c", "pan-1dd5d7741c7265ed" ] }, { "key": "P03-20240216-185832:7375c33420663b2b65aa7007e55a38555781892a", "video_id": "P03-20240216-185832", "question": "When I cut the potatoes, on how many potatoes do I make more than 4 cuts?", "answer_choice_0": "2", "answer_choice_1": "5", "answer_choice_2": "0", "answer_choice_3": "1", "answer_choice_4": "3", "answer_id": 4, "answer": "3", "question_type": [ "Counting", "Action Recognition", "Object Recognition" ], "objects_of_interest": [ "bag of spring greens-d94464beb250fb38", "bag of kale-9cd0eb0af15afe72", "vegetable-65a2e3a47c6302d0", "box of mushrooms-5480891ea71366cd", "spring onion-fd3fe499e30ff53c", "box of food-3b09d3574748315b", "cauliflower-705371010f925839", "large shopping bag-ab44271b31236a56", "bag of salad-a96d954fabf438d9", "bag of mozzarella-8deaddf9cb5a1c14", "single cream pot-4ffbd866d873e1e6", "spaghetti bag-f8193f01def0ea7e", "box of grapes-99d179636e4e26b3", "bananas bag-afa30e91f9a2b2ea", "bag of potatoes-8664af8e0c4512fa", "bag of mandarins-57e9d0df0122ca8b", "bag of mandarin2-05b21532ace5fc01", "garlic-124c0fee31ec5bc3", "bag of pears-2691488edebe6c10", "bag of kiwis-270bfcee080ea7af", "squash-bf5700a01518efad", "pack of pasta-107b336f60723036", "box of yeast-e96b26f5f93c789f", "bag of lentils-fc82371146cac8f8", "paper food waste bag-5f3983f5e379ca63", "superglue box-63c2e054d6869957", "bread-6039bb22e14eacbd", "tomato concentrated puree-8b68051c77a6200d", "coconut milk can-06a9fc40ecc82644", "tomato sauce bottle-ea967e3e7ff63770", "tomato sauce bottle2-ea67d9e08b7b48aa", "bottle with green lid-f75030cbed5ecaad", "tomato sauce bottle3-ca3fa9d369bd7a9b", "paper receipt-ea5837ad2dd012b3", "large shopping bag2-c1a67a7d60bed8f5", "black shoe left-b9c8a09619f10cc7", "black shoe right-fb84ffe2c21f414c", "dirty plastic spatula-39bf8dc418d3c568", "plate-ccac08aa8e50801a", "plate2-3bfd83d35c5b7ce4", "saucepan-da8b913a944ea3a1", "smaller saucepan-5b6f8054c1661133", "small bowl-f354cc2d8e7f353f", "white bowl-297b16122e17d1ba", "small bowl2-bf5691912176ed31", "coffee cup-1856f70dbc366538", "cup2-34d665c224823126", "plastic box container-16c45dc5640b85eb", "plastic container2-66b0bd40422b43a6", "cloth-e905582ea3e0d5cb", "chopstick1-384d046924eb0850", "chopstick2-740e52f3bd7b0104", "chopstick3-83bfc567ece98531", "chopstick4-00a31c1bcd4d68b3", "chopstick5-af86dc66d58998df", "chopstick6-7e52c88efe127a22", "fork1-c48edec98d98bae4", "fork2-955ca204be163621", "knife1-30debca82d73d304", "spoon1-3c0a243af93fba2e", "spoon2-a3da42ad4dd94bd9", "red spoon-546cfa010b78f974", "metal colander-f9f1b975f21e8568", "large bowl-e297ef66a083c612", "plastic colander-7d62c20f4ec67122", "bowl3-bfd73628becf1677", "bowl4-28b8b3314216e252", "bowl5-1309234381bf1d70", "white bowl2-08d9cb477606e8a6", "bowl6-c94a84c2091f7a90", "cup3-0bacf6cf9105f10d", "glass-3cefa6050e618946", "glass2-71459ef5d1e087a8", "glass3-13dffc2ecc98b459", "glass4-125b49dacdb84198", "glass5-5552f629a26efb08", "white medium sized bowl-173a8b87bb07cb50", "glass ramekin-27976f43e25f4162", "small grey plate-018ce8550f0704b7", "two identical plastic box lids-317227f0df46f5b1", "red plate-3e5c9ed8e2252755", "one small grey plate-252423245bc68222", "v60 glass filter-bb96da8cfbd03752", "v60 metal filter-7c833fe84fc512ee", "plastic food bag-7dca5a67acbec046", "plastic food box-a6ec4ab01b9766d3", "plastic food box lid-0da741d3f28b4d44", "fork3-dc4e1cc6003d165c", "dishwasher brush-8d37bfb2df0fa22c", "fork4-866bca4acda794b3", "white plate-ff6e78f4c64444d5", "grater-7fa171dbaf625b84", "small dirty bowl-a19b985558c5e8c8", "green coffee cup1-c9c6724b8274ff67", "wooden spoon-69be1eab24a0cc61", "metal teaspoon-5846ee31ef4a5033", "checkered cup-bdb131456dfe461a", "coffee jug-7feb82765554d728", "chopstick-cb987a664aeeba91", "one food waste paper bag-d21c1a1f6f84d7fb", "eggshell-c08839df3778e8ff", "blue plastic bag-3422821684ce83ab", "food waste bin-4bad12162c13724a", "sponge-0dd841b9997653d1", "washing up liquid bottle-a3e90fc19a4317dd", "pan-1dd5d7741c7265ed", "large bowl2-75fe73e9a85a889a", "plastic calendar-63c80e3f9cdf75b5", "potato1-a2e73c6ae3d8264d", "potato2-1e44164bfbaa9cad", "potato3-a4ace2eabb1c6a34", "potato4-aa365386e5442e73", "potato5-140dcd14519daf63", "potato6-1ade5ad88a5fb7c8", "potato7-a8956f52788169a1", "potato8-7ce9f636df79b299", "chopping board-0fa725b89e5753bd", "knife2-6e02fef99df274b3", "knife3-5aa57c591cb06a1d", "cloth2-08bd78e9df7e00fd", "potato9-f9df4d0f9b83b31c", "potato10-a2ed32ce73f688fc", "potato11-310f57a9a9cf7339", "potato12-e57379c4dc03dc67", "can of olive oil-3ded77db4988444c", "salt-32d76cb271cd9d21", "olive oil can cap-ac705c30088b78fd", "glass lid-924ba845005deac0", "hand wash dispenser-e454f6197c20d6d4", "plate3-dc739385c55942c8", "plate4-3277214c484e2b40", "orange cloth-a316ea7c38bb7e34", "bag of flour-55b8a2e0f7b61d16", "scale-71b4f54bf2930f23", "scissors-1cc56ec76660d927", "scissor safety cover-d689b523849381f6", "bag of flour top of bag-1582a3609c7495e6", "clip-cb17faf107a72a23", "large plastic box-5adeb358c8c9ea55", "yeast box2-a8228c1e3ffccb72", "folded sachet-fec3b7c95948d0b2", "wooden spoon2-9e1e2402cdbdda71", "wooden pizza peel-dec25775f59bf7f8", "wooden rolling pin-bc989fd347e588a1", "square plastic container-680dc3051eb0c50f", "cake mould-db4ac615fc94220f", "dough scraper-db658d2e4487aab7", "egg carton-0e0dcb4ecdf92e37", "egg-688e78a44cefb142", "egg2-205ae9b80988d4b1", "egg3-da39fb088198419b", "egg4-fc8097f80e4b2cbe", "egg5-7a615aef1c3a47b5", "white bowl3-11c6cc86c3de1854", "egg6-a3582ca144add83a", "fork5-32a1e3a313eaa6b7", "smaller baking tray-2c9c033205f0a1b5", "white baking tray-a81532f9e3d87905", "apron-f40a30089727ad06", "spoon3-f0ea632888bf9f67", "oven mitts-1f178a7dad4a1f3a", "container lid-b094d8e1f9b9a3cc", "bag of onions-14e044d9f9c02dd4", "one onion-a20caf9e35684670", "onion half-3dc3bb9a9acac0b3", "tea towel-2f785cd10ad2da5c", "tea towel2-fdb00402c2d6473b", "fork6-2ed7aa38d9a7aa5c", "knife4-c47a15c6ca400d00", "knife5-10bb43e6546b75c9", "pizza dough-d086c99d3d0752c8", "tissue box-01cec62e4005940b", "yellow glass-b0036c30465b1dd5", "cork pot stand-6aa9a0ad87fcd995", "fork7-efea21a6c8ba72ef", "spoon4-48bc0864caeb5f85" ], "reasoning": "1. I watched the video to find that the user began peeling the potatoes (bag of potatoes-8664af8e0c4512fa).\n2. I then watched the user cut the first potato (potato1-a2e73c6ae3d8264d) from 19:19-19:25, using 5 cuts.\n3. Next, they cut the second potato (potato2-1e44164bfbaa9cad) from 19:30-19:42, using 5 cuts.\n4. Then, the user cut the third potato (potato3-a4ace2eabb1c6a34) from 19:45-19:53, using 5 cuts.\n5. After this, this user cuts the fourth potato (potato4-aa365386e5442e73) from 19:57-20:02, using 4 cuts.\n6. For the fifth potato (potato5-140dcd14519daf63), the user uses 4 cuts to chop it from 20:07-20:13.\n7. Next, the user chopped the sixth potato (potato6-1ade5ad88a5fb7c8) from 20:18-20:25, using 4 cuts.\n8. Then, the user takes the seventh potato (potato7-a8956f52788169a1) from 20:29-20:33, using 3 cuts.\n9. Lastly, the user cuts the eighth potato (potato8-7ce9f636df79b299) from 20:35-20:42, with 4 cuts.\n10. I concluded that the user cuts a total of 3 potatoes using more than 4 cuts.", "referenced_object_masks": [ "potato5-140dcd14519daf63", "potato8-7ce9f636df79b299", "potato4-aa365386e5442e73", "bag of potatoes-8664af8e0c4512fa", "potato6-1ade5ad88a5fb7c8", "potato7-a8956f52788169a1", "potato2-1e44164bfbaa9cad", "potato1-a2e73c6ae3d8264d", "potato3-a4ace2eabb1c6a34" ] }, { "key": "P03-20240216-185832:8cc6b63450495f5439d54d587e019ac06b7cfc1d", "video_id": "P03-20240216-185832", "question": "How many times do I slide the yellow grocery bag on the ground before folding it up?", "answer_choice_0": "3", "answer_choice_1": "4", "answer_choice_2": "2", "answer_choice_3": "1", "answer_choice_4": "0", "answer_id": 0, "answer": "3", "question_type": [ "Action Recognition", "State Changes", "Egocentric & Spatial Perception", "Object Recognition" ], "objects_of_interest": [ "bag of spring greens-d94464beb250fb38", "bag of kale-9cd0eb0af15afe72", "vegetable-65a2e3a47c6302d0", "box of mushrooms-5480891ea71366cd", "spring onion-fd3fe499e30ff53c", "box of food-3b09d3574748315b", "cauliflower-705371010f925839", "large shopping bag-ab44271b31236a56", "bag of salad-a96d954fabf438d9", "bag of mozzarella-8deaddf9cb5a1c14", "single cream pot-4ffbd866d873e1e6", "spaghetti bag-f8193f01def0ea7e", "box of grapes-99d179636e4e26b3", "bananas bag-afa30e91f9a2b2ea", "bag of potatoes-8664af8e0c4512fa", "bag of mandarins-57e9d0df0122ca8b", "bag of mandarin2-05b21532ace5fc01", "garlic-124c0fee31ec5bc3", "bag of pears-2691488edebe6c10", "bag of kiwis-270bfcee080ea7af", "squash-bf5700a01518efad", "pack of pasta-107b336f60723036", "box of yeast-e96b26f5f93c789f", "bag of lentils-fc82371146cac8f8", "paper food waste bag-5f3983f5e379ca63", "superglue box-63c2e054d6869957", "bread-6039bb22e14eacbd", "tomato concentrated puree-8b68051c77a6200d", "coconut milk can-06a9fc40ecc82644", "tomato sauce bottle-ea967e3e7ff63770", "tomato sauce bottle2-ea67d9e08b7b48aa", "bottle with green lid-f75030cbed5ecaad", "tomato sauce bottle3-ca3fa9d369bd7a9b", "paper receipt-ea5837ad2dd012b3", "large shopping bag2-c1a67a7d60bed8f5", "black shoe left-b9c8a09619f10cc7", "black shoe right-fb84ffe2c21f414c", "dirty plastic spatula-39bf8dc418d3c568", "plate-ccac08aa8e50801a", "plate2-3bfd83d35c5b7ce4", "saucepan-da8b913a944ea3a1", "smaller saucepan-5b6f8054c1661133", "small bowl-f354cc2d8e7f353f", "white bowl-297b16122e17d1ba", "small bowl2-bf5691912176ed31", "coffee cup-1856f70dbc366538", "cup2-34d665c224823126", "plastic box container-16c45dc5640b85eb", "plastic container2-66b0bd40422b43a6", "cloth-e905582ea3e0d5cb", "chopstick1-384d046924eb0850", "chopstick2-740e52f3bd7b0104", "chopstick3-83bfc567ece98531", "chopstick4-00a31c1bcd4d68b3", "chopstick5-af86dc66d58998df", "chopstick6-7e52c88efe127a22", "fork1-c48edec98d98bae4", "fork2-955ca204be163621", "knife1-30debca82d73d304", "spoon1-3c0a243af93fba2e", "spoon2-a3da42ad4dd94bd9", "red spoon-546cfa010b78f974", "metal colander-f9f1b975f21e8568", "large bowl-e297ef66a083c612", "plastic colander-7d62c20f4ec67122", "bowl3-bfd73628becf1677", "bowl4-28b8b3314216e252", "bowl5-1309234381bf1d70", "white bowl2-08d9cb477606e8a6", "bowl6-c94a84c2091f7a90", "cup3-0bacf6cf9105f10d", "glass-3cefa6050e618946", "glass2-71459ef5d1e087a8", "glass3-13dffc2ecc98b459", "glass4-125b49dacdb84198", "glass5-5552f629a26efb08", "white medium sized bowl-173a8b87bb07cb50", "glass ramekin-27976f43e25f4162", "small grey plate-018ce8550f0704b7", "two identical plastic box lids-317227f0df46f5b1", "red plate-3e5c9ed8e2252755", "one small grey plate-252423245bc68222", "v60 glass filter-bb96da8cfbd03752", "v60 metal filter-7c833fe84fc512ee", "plastic food bag-7dca5a67acbec046", "plastic food box-a6ec4ab01b9766d3", "plastic food box lid-0da741d3f28b4d44", "fork3-dc4e1cc6003d165c", "dishwasher brush-8d37bfb2df0fa22c", "fork4-866bca4acda794b3", "white plate-ff6e78f4c64444d5", "grater-7fa171dbaf625b84", "small dirty bowl-a19b985558c5e8c8", "green coffee cup1-c9c6724b8274ff67", "wooden spoon-69be1eab24a0cc61", "metal teaspoon-5846ee31ef4a5033", "checkered cup-bdb131456dfe461a", "coffee jug-7feb82765554d728", "chopstick-cb987a664aeeba91", "one food waste paper bag-d21c1a1f6f84d7fb", "eggshell-c08839df3778e8ff", "blue plastic bag-3422821684ce83ab", "food waste bin-4bad12162c13724a", "sponge-0dd841b9997653d1", "washing up liquid bottle-a3e90fc19a4317dd", "pan-1dd5d7741c7265ed", "large bowl2-75fe73e9a85a889a", "plastic calendar-63c80e3f9cdf75b5", "potato1-a2e73c6ae3d8264d", "potato2-1e44164bfbaa9cad", "potato3-a4ace2eabb1c6a34", "potato4-aa365386e5442e73", "potato5-140dcd14519daf63", "potato6-1ade5ad88a5fb7c8", "potato7-a8956f52788169a1", "potato8-7ce9f636df79b299", "chopping board-0fa725b89e5753bd", "knife2-6e02fef99df274b3", "knife3-5aa57c591cb06a1d", "cloth2-08bd78e9df7e00fd", "potato9-f9df4d0f9b83b31c", "potato10-a2ed32ce73f688fc", "potato11-310f57a9a9cf7339", "potato12-e57379c4dc03dc67", "can of olive oil-3ded77db4988444c", "salt-32d76cb271cd9d21", "olive oil can cap-ac705c30088b78fd", "glass lid-924ba845005deac0", "hand wash dispenser-e454f6197c20d6d4", "plate3-dc739385c55942c8", "plate4-3277214c484e2b40", "orange cloth-a316ea7c38bb7e34", "bag of flour-55b8a2e0f7b61d16", "scale-71b4f54bf2930f23", "scissors-1cc56ec76660d927", "scissor safety cover-d689b523849381f6", "bag of flour top of bag-1582a3609c7495e6", "clip-cb17faf107a72a23", "large plastic box-5adeb358c8c9ea55", "yeast box2-a8228c1e3ffccb72", "folded sachet-fec3b7c95948d0b2", "wooden spoon2-9e1e2402cdbdda71", "wooden pizza peel-dec25775f59bf7f8", "wooden rolling pin-bc989fd347e588a1", "square plastic container-680dc3051eb0c50f", "cake mould-db4ac615fc94220f", "dough scraper-db658d2e4487aab7", "egg carton-0e0dcb4ecdf92e37", "egg-688e78a44cefb142", "egg2-205ae9b80988d4b1", "egg3-da39fb088198419b", "egg4-fc8097f80e4b2cbe", "egg5-7a615aef1c3a47b5", "white bowl3-11c6cc86c3de1854", "egg6-a3582ca144add83a", "fork5-32a1e3a313eaa6b7", "smaller baking tray-2c9c033205f0a1b5", "white baking tray-a81532f9e3d87905", "apron-f40a30089727ad06", "spoon3-f0ea632888bf9f67", "oven mitts-1f178a7dad4a1f3a", "container lid-b094d8e1f9b9a3cc", "bag of onions-14e044d9f9c02dd4", "one onion-a20caf9e35684670", "onion half-3dc3bb9a9acac0b3", "tea towel-2f785cd10ad2da5c", "tea towel2-fdb00402c2d6473b", "fork6-2ed7aa38d9a7aa5c", "knife4-c47a15c6ca400d00", "knife5-10bb43e6546b75c9", "pizza dough-d086c99d3d0752c8", "tissue box-01cec62e4005940b", "yellow glass-b0036c30465b1dd5", "cork pot stand-6aa9a0ad87fcd995", "fork7-efea21a6c8ba72ef", "spoon4-48bc0864caeb5f85" ], "reasoning": "1. I watched the video to identify the yellow grocery bag (large shopping bag-ab44271b31236a56), which is first visible at 00:05. \n2. I continued to watch as the user unloads the bag (large shopping bag-ab44271b31236a56), observing the first time the user slides the bag (large shopping bag-ab44271b31236a56) towards the refrigerator from 00:42-00:43.\n3. The user continues, sliding the bag (large shopping bag-ab44271b31236a56) for the second time from 01:25-01:27.\n4. Next, I saw the user slightly slide the bag (large shopping bag-ab44271b31236a56) a third time at 01:38.\n5. I then continued to watch as the user empties a different bag (large shopping bag2-c1a67a7d60bed8f5) before folding the yellow bag (large shopping bag-ab44271b31236a56) from 02:57-03:16.", "referenced_object_masks": [ "large shopping bag2-c1a67a7d60bed8f5", "large shopping bag-ab44271b31236a56" ] }, { "key": "P03-20240216-185832:9531370d221be472ddb44211c20da7ac26e99d21", "video_id": "P03-20240216-185832", "question": "If I peeled potatoes at the average speed it took me to peel the last four potatoes, how long would it have taken me to peel all the potatoes I use in the video at that rate?", "answer_choice_0": "6:30", "answer_choice_1": "5:00", "answer_choice_2": "5:30", "answer_choice_3": "6:00", "answer_choice_4": "7:00", "answer_id": 4, "answer": "7:00", "question_type": [ "Counterfactual", "Counting", "Numerical Reasoning" ], "objects_of_interest": [ "bag of spring greens-d94464beb250fb38", "bag of kale-9cd0eb0af15afe72", "vegetable-65a2e3a47c6302d0", "box of mushrooms-5480891ea71366cd", "spring onion-fd3fe499e30ff53c", "box of food-3b09d3574748315b", "cauliflower-705371010f925839", "large shopping bag-ab44271b31236a56", "bag of salad-a96d954fabf438d9", "bag of mozzarella-8deaddf9cb5a1c14", "single cream pot-4ffbd866d873e1e6", "spaghetti bag-f8193f01def0ea7e", "box of grapes-99d179636e4e26b3", "bananas bag-afa30e91f9a2b2ea", "bag of potatoes-8664af8e0c4512fa", "bag of mandarins-57e9d0df0122ca8b", "bag of mandarin2-05b21532ace5fc01", "garlic-124c0fee31ec5bc3", "bag of pears-2691488edebe6c10", "bag of kiwis-270bfcee080ea7af", "squash-bf5700a01518efad", "pack of pasta-107b336f60723036", "box of yeast-e96b26f5f93c789f", "bag of lentils-fc82371146cac8f8", "paper food waste bag-5f3983f5e379ca63", "superglue box-63c2e054d6869957", "bread-6039bb22e14eacbd", "tomato concentrated puree-8b68051c77a6200d", "coconut milk can-06a9fc40ecc82644", "tomato sauce bottle-ea967e3e7ff63770", "tomato sauce bottle2-ea67d9e08b7b48aa", "bottle with green lid-f75030cbed5ecaad", "tomato sauce bottle3-ca3fa9d369bd7a9b", "paper receipt-ea5837ad2dd012b3", "large shopping bag2-c1a67a7d60bed8f5", "black shoe left-b9c8a09619f10cc7", "black shoe right-fb84ffe2c21f414c", "dirty plastic spatula-39bf8dc418d3c568", "plate-ccac08aa8e50801a", "plate2-3bfd83d35c5b7ce4", "saucepan-da8b913a944ea3a1", "smaller saucepan-5b6f8054c1661133", "small bowl-f354cc2d8e7f353f", "white bowl-297b16122e17d1ba", "small bowl2-bf5691912176ed31", "coffee cup-1856f70dbc366538", "cup2-34d665c224823126", "plastic box container-16c45dc5640b85eb", "plastic container2-66b0bd40422b43a6", "cloth-e905582ea3e0d5cb", "chopstick1-384d046924eb0850", "chopstick2-740e52f3bd7b0104", "chopstick3-83bfc567ece98531", "chopstick4-00a31c1bcd4d68b3", "chopstick5-af86dc66d58998df", "chopstick6-7e52c88efe127a22", "fork1-c48edec98d98bae4", "fork2-955ca204be163621", "knife1-30debca82d73d304", "spoon1-3c0a243af93fba2e", "spoon2-a3da42ad4dd94bd9", "red spoon-546cfa010b78f974", "metal colander-f9f1b975f21e8568", "large bowl-e297ef66a083c612", "plastic colander-7d62c20f4ec67122", "bowl3-bfd73628becf1677", "bowl4-28b8b3314216e252", "bowl5-1309234381bf1d70", "white bowl2-08d9cb477606e8a6", "bowl6-c94a84c2091f7a90", "cup3-0bacf6cf9105f10d", "glass-3cefa6050e618946", "glass2-71459ef5d1e087a8", "glass3-13dffc2ecc98b459", "glass4-125b49dacdb84198", "glass5-5552f629a26efb08", "white medium sized bowl-173a8b87bb07cb50", "glass ramekin-27976f43e25f4162", "small grey plate-018ce8550f0704b7", "two identical plastic box lids-317227f0df46f5b1", "red plate-3e5c9ed8e2252755", "one small grey plate-252423245bc68222", "v60 glass filter-bb96da8cfbd03752", "v60 metal filter-7c833fe84fc512ee", "plastic food bag-7dca5a67acbec046", "plastic food box-a6ec4ab01b9766d3", "plastic food box lid-0da741d3f28b4d44", "fork3-dc4e1cc6003d165c", "dishwasher brush-8d37bfb2df0fa22c", "fork4-866bca4acda794b3", "white plate-ff6e78f4c64444d5", "grater-7fa171dbaf625b84", "small dirty bowl-a19b985558c5e8c8", "green coffee cup1-c9c6724b8274ff67", "wooden spoon-69be1eab24a0cc61", "metal teaspoon-5846ee31ef4a5033", "checkered cup-bdb131456dfe461a", "coffee jug-7feb82765554d728", "chopstick-cb987a664aeeba91", "one food waste paper bag-d21c1a1f6f84d7fb", "eggshell-c08839df3778e8ff", "blue plastic bag-3422821684ce83ab", "food waste bin-4bad12162c13724a", "sponge-0dd841b9997653d1", "washing up liquid bottle-a3e90fc19a4317dd", "pan-1dd5d7741c7265ed", "large bowl2-75fe73e9a85a889a", "plastic calendar-63c80e3f9cdf75b5", "potato1-a2e73c6ae3d8264d", "potato2-1e44164bfbaa9cad", "potato3-a4ace2eabb1c6a34", "potato4-aa365386e5442e73", "potato5-140dcd14519daf63", "potato6-1ade5ad88a5fb7c8", "potato7-a8956f52788169a1", "potato8-7ce9f636df79b299", "chopping board-0fa725b89e5753bd", "knife2-6e02fef99df274b3", "knife3-5aa57c591cb06a1d", "cloth2-08bd78e9df7e00fd", "potato9-f9df4d0f9b83b31c", "potato10-a2ed32ce73f688fc", "potato11-310f57a9a9cf7339", "potato12-e57379c4dc03dc67", "can of olive oil-3ded77db4988444c", "salt-32d76cb271cd9d21", "olive oil can cap-ac705c30088b78fd", "glass lid-924ba845005deac0", "hand wash dispenser-e454f6197c20d6d4", "plate3-dc739385c55942c8", "plate4-3277214c484e2b40", "orange cloth-a316ea7c38bb7e34", "bag of flour-55b8a2e0f7b61d16", "scale-71b4f54bf2930f23", "scissors-1cc56ec76660d927", "scissor safety cover-d689b523849381f6", "bag of flour top of bag-1582a3609c7495e6", "clip-cb17faf107a72a23", "large plastic box-5adeb358c8c9ea55", "yeast box2-a8228c1e3ffccb72", "folded sachet-fec3b7c95948d0b2", "wooden spoon2-9e1e2402cdbdda71", "wooden pizza peel-dec25775f59bf7f8", "wooden rolling pin-bc989fd347e588a1", "square plastic container-680dc3051eb0c50f", "cake mould-db4ac615fc94220f", "dough scraper-db658d2e4487aab7", "egg carton-0e0dcb4ecdf92e37", "egg-688e78a44cefb142", "egg2-205ae9b80988d4b1", "egg3-da39fb088198419b", "egg4-fc8097f80e4b2cbe", "egg5-7a615aef1c3a47b5", "white bowl3-11c6cc86c3de1854", "egg6-a3582ca144add83a", "fork5-32a1e3a313eaa6b7", "smaller baking tray-2c9c033205f0a1b5", "white baking tray-a81532f9e3d87905", "apron-f40a30089727ad06", "spoon3-f0ea632888bf9f67", "oven mitts-1f178a7dad4a1f3a", "container lid-b094d8e1f9b9a3cc", "bag of onions-14e044d9f9c02dd4", "one onion-a20caf9e35684670", "onion half-3dc3bb9a9acac0b3", "tea towel-2f785cd10ad2da5c", "tea towel2-fdb00402c2d6473b", "fork6-2ed7aa38d9a7aa5c", "knife4-c47a15c6ca400d00", "knife5-10bb43e6546b75c9", "pizza dough-d086c99d3d0752c8", "tissue box-01cec62e4005940b", "yellow glass-b0036c30465b1dd5", "cork pot stand-6aa9a0ad87fcd995", "fork7-efea21a6c8ba72ef", "spoon4-48bc0864caeb5f85" ], "reasoning": "1. I watched to see when the user peeled the last 4 potatoes.\n2. I counted the user peeling a total of 12 potatoes from 14:04-23:47.\n3. I identified the last 4 potatoes getting peeled from 21:27-23:47.\n4. I calculated the average speed of peeling the last 4 potatoes by subtracting 21:27 from 23:47 to reach a total of 2:20, and divided that by 4 to calculate an average peeling speed of 35 seconds per potato.\n5. I multiplied the 12 total potatoes by 35 seconds to reach a total time of 7 minutes.", "referenced_object_masks": [] }, { "key": "P03-20240216-185832:ca2e557fc064e2feb40a5afd766d8bb1f651d663", "video_id": "P03-20240216-185832", "question": "What is directly underneath the fourth item I stored in the fridge?", "answer_choice_0": "Salad greens", "answer_choice_1": "Cauliflower", "answer_choice_2": "Tomatoes", "answer_choice_3": "Mushrooms", "answer_choice_4": "Ginger beer", "answer_id": 2, "answer": "Tomatoes", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Counting" ], "objects_of_interest": [ "bag of spring greens-d94464beb250fb38", "bag of kale-9cd0eb0af15afe72", "vegetable-65a2e3a47c6302d0", "box of mushrooms-5480891ea71366cd", "spring onion-fd3fe499e30ff53c", "box of food-3b09d3574748315b", "cauliflower-705371010f925839", "large shopping bag-ab44271b31236a56", "bag of salad-a96d954fabf438d9", "bag of mozzarella-8deaddf9cb5a1c14", "single cream pot-4ffbd866d873e1e6", "spaghetti bag-f8193f01def0ea7e", "box of grapes-99d179636e4e26b3", "bananas bag-afa30e91f9a2b2ea", "bag of potatoes-8664af8e0c4512fa", "bag of mandarins-57e9d0df0122ca8b", "bag of mandarin2-05b21532ace5fc01", "garlic-124c0fee31ec5bc3", "bag of pears-2691488edebe6c10", "bag of kiwis-270bfcee080ea7af", "squash-bf5700a01518efad", "pack of pasta-107b336f60723036", "box of yeast-e96b26f5f93c789f", "bag of lentils-fc82371146cac8f8", "paper food waste bag-5f3983f5e379ca63", "superglue box-63c2e054d6869957", "bread-6039bb22e14eacbd", "tomato concentrated puree-8b68051c77a6200d", "coconut milk can-06a9fc40ecc82644", "tomato sauce bottle-ea967e3e7ff63770", "tomato sauce bottle2-ea67d9e08b7b48aa", "bottle with green lid-f75030cbed5ecaad", "tomato sauce bottle3-ca3fa9d369bd7a9b", "paper receipt-ea5837ad2dd012b3", "large shopping bag2-c1a67a7d60bed8f5", "black shoe left-b9c8a09619f10cc7", "black shoe right-fb84ffe2c21f414c", "dirty plastic spatula-39bf8dc418d3c568", "plate-ccac08aa8e50801a", "plate2-3bfd83d35c5b7ce4", "saucepan-da8b913a944ea3a1", "smaller saucepan-5b6f8054c1661133", "small bowl-f354cc2d8e7f353f", "white bowl-297b16122e17d1ba", "small bowl2-bf5691912176ed31", "coffee cup-1856f70dbc366538", "cup2-34d665c224823126", "plastic box container-16c45dc5640b85eb", "plastic container2-66b0bd40422b43a6", "cloth-e905582ea3e0d5cb", "chopstick1-384d046924eb0850", "chopstick2-740e52f3bd7b0104", "chopstick3-83bfc567ece98531", "chopstick4-00a31c1bcd4d68b3", "chopstick5-af86dc66d58998df", "chopstick6-7e52c88efe127a22", "fork1-c48edec98d98bae4", "fork2-955ca204be163621", "knife1-30debca82d73d304", "spoon1-3c0a243af93fba2e", "spoon2-a3da42ad4dd94bd9", "red spoon-546cfa010b78f974", "metal colander-f9f1b975f21e8568", "large bowl-e297ef66a083c612", "plastic colander-7d62c20f4ec67122", "bowl3-bfd73628becf1677", "bowl4-28b8b3314216e252", "bowl5-1309234381bf1d70", "white bowl2-08d9cb477606e8a6", "bowl6-c94a84c2091f7a90", "cup3-0bacf6cf9105f10d", "glass-3cefa6050e618946", "glass2-71459ef5d1e087a8", "glass3-13dffc2ecc98b459", "glass4-125b49dacdb84198", "glass5-5552f629a26efb08", "white medium sized bowl-173a8b87bb07cb50", "glass ramekin-27976f43e25f4162", "small grey plate-018ce8550f0704b7", "two identical plastic box lids-317227f0df46f5b1", "red plate-3e5c9ed8e2252755", "one small grey plate-252423245bc68222", "v60 glass filter-bb96da8cfbd03752", "v60 metal filter-7c833fe84fc512ee", "plastic food bag-7dca5a67acbec046", "plastic food box-a6ec4ab01b9766d3", "plastic food box lid-0da741d3f28b4d44", "fork3-dc4e1cc6003d165c", "dishwasher brush-8d37bfb2df0fa22c", "fork4-866bca4acda794b3", "white plate-ff6e78f4c64444d5", "grater-7fa171dbaf625b84", "small dirty bowl-a19b985558c5e8c8", "green coffee cup1-c9c6724b8274ff67", "wooden spoon-69be1eab24a0cc61", "metal teaspoon-5846ee31ef4a5033", "checkered cup-bdb131456dfe461a", "coffee jug-7feb82765554d728", "chopstick-cb987a664aeeba91", "one food waste paper bag-d21c1a1f6f84d7fb", "eggshell-c08839df3778e8ff", "blue plastic bag-3422821684ce83ab", "food waste bin-4bad12162c13724a", "sponge-0dd841b9997653d1", "washing up liquid bottle-a3e90fc19a4317dd", "pan-1dd5d7741c7265ed", "large bowl2-75fe73e9a85a889a", "plastic calendar-63c80e3f9cdf75b5", "potato1-a2e73c6ae3d8264d", "potato2-1e44164bfbaa9cad", "potato3-a4ace2eabb1c6a34", "potato4-aa365386e5442e73", "potato5-140dcd14519daf63", "potato6-1ade5ad88a5fb7c8", "potato7-a8956f52788169a1", "potato8-7ce9f636df79b299", "chopping board-0fa725b89e5753bd", "knife2-6e02fef99df274b3", "knife3-5aa57c591cb06a1d", "cloth2-08bd78e9df7e00fd", "potato9-f9df4d0f9b83b31c", "potato10-a2ed32ce73f688fc", "potato11-310f57a9a9cf7339", "potato12-e57379c4dc03dc67", "can of olive oil-3ded77db4988444c", "salt-32d76cb271cd9d21", "olive oil can cap-ac705c30088b78fd", "glass lid-924ba845005deac0", "hand wash dispenser-e454f6197c20d6d4", "plate3-dc739385c55942c8", "plate4-3277214c484e2b40", "orange cloth-a316ea7c38bb7e34", "bag of flour-55b8a2e0f7b61d16", "scale-71b4f54bf2930f23", "scissors-1cc56ec76660d927", "scissor safety cover-d689b523849381f6", "bag of flour top of bag-1582a3609c7495e6", "clip-cb17faf107a72a23", "large plastic box-5adeb358c8c9ea55", "yeast box2-a8228c1e3ffccb72", "folded sachet-fec3b7c95948d0b2", "wooden spoon2-9e1e2402cdbdda71", "wooden pizza peel-dec25775f59bf7f8", "wooden rolling pin-bc989fd347e588a1", "square plastic container-680dc3051eb0c50f", "cake mould-db4ac615fc94220f", "dough scraper-db658d2e4487aab7", "egg carton-0e0dcb4ecdf92e37", "egg-688e78a44cefb142", "egg2-205ae9b80988d4b1", "egg3-da39fb088198419b", "egg4-fc8097f80e4b2cbe", "egg5-7a615aef1c3a47b5", "white bowl3-11c6cc86c3de1854", "egg6-a3582ca144add83a", "fork5-32a1e3a313eaa6b7", "smaller baking tray-2c9c033205f0a1b5", "white baking tray-a81532f9e3d87905", "apron-f40a30089727ad06", "spoon3-f0ea632888bf9f67", "oven mitts-1f178a7dad4a1f3a", "container lid-b094d8e1f9b9a3cc", "bag of onions-14e044d9f9c02dd4", "one onion-a20caf9e35684670", "onion half-3dc3bb9a9acac0b3", "tea towel-2f785cd10ad2da5c", "tea towel2-fdb00402c2d6473b", "fork6-2ed7aa38d9a7aa5c", "knife4-c47a15c6ca400d00", "knife5-10bb43e6546b75c9", "pizza dough-d086c99d3d0752c8", "tissue box-01cec62e4005940b", "yellow glass-b0036c30465b1dd5", "cork pot stand-6aa9a0ad87fcd995", "fork7-efea21a6c8ba72ef", "spoon4-48bc0864caeb5f85" ], "reasoning": "1. I counted the items the user stored in the fridge.\n2. I saw them store a bag of greens (bag of spring greens-d94464beb250fb38) at 00:13, a box of mushrooms (box of mushrooms-5480891ea71366cd) at 00:28, a bunch of spring onions (spring onion-fd3fe499e30ff53c) at 0:30, and some deli meat (box of food-3b09d3574748315b) at 00:36.\n3. I went back in the video for a better view of what was around the deli meat (box of food-3b09d3574748315b) in the fridge, and at 00:35, I could more clearly see that the user stored the deli meat (box of food-3b09d3574748315b) directly on top of a box of tomatoes .", "referenced_object_masks": [ "box of mushrooms-5480891ea71366cd", "bag of spring greens-d94464beb250fb38", "box of food-3b09d3574748315b", "spring onion-fd3fe499e30ff53c" ] }, { "key": "P03-20240216-205923:0ab40b5271dea703e388b5fd0b8f619455a6c3fd", "video_id": "P03-20240216-205923", "question": "Which of the following set of actions did I take while holding the box of plastic wrap?", "answer_choice_0": "Wrap the food, open the refrigerator, press the dishwasher", "answer_choice_1": "Open the refrigerator, wrap the food, pull out the chair", "answer_choice_2": "Press the dishwasher, wrap the food, touch my face", "answer_choice_3": "Open the refrigerator, wrap the food, touch my face", "answer_choice_4": "Pull the plate, wrap the food, open the refrigerator", "answer_id": 2, "answer": "Press the dishwasher, wrap the food, touch my face", "question_type": [ "Action Recognition", "Object Recognition", "State Changes" ], "objects_of_interest": [ "red plastic wrapping-514fe3f820b81cc3", "red plastic wrapping2-1e16148624f43cc6", "receipt-1767f107f2703993", "baking tray-cbdef058f6304d21", "mandarin peels-74e4dd3ae38236f4", "more mandarin peels-681c085d87f1dced", "plate-0cff09877fdaee8c", "cutlery-6e178c73b0f74860", "plate2-0903275c9558c237", "cloth-454c499f42afffda", "white bowl-1525e0c983913700", "large bowl-f8024bd5904d941b", "chocolate-a6aca8dd303fd074", "cloth2-cb3cc57f80844fbe", "plastic box-e3d1a75f812f4e50", "dough scraper-f61e170b287c43bc", "cup-d7da0d3fd46ab842", "chair-dad799f50ac463b6", "chair2-4b8b62479f537e90", "cork pot stand-f07c655b01b7f78e", "phone-ff7adada9e20bd38", "dishwasher brush-38aad133135d10ca", "spoon-a842ea6338c3ab5d", "spoon2-f23d09787407ff51", "knife-18c208541a819db5", "fork-282c5015dc24e049", "knife2-bb7371d0b14c4b9d", "fork2-a07cd0dd302adaae", "coffee jug-bca386177593e659", "food box lid-9221cf9681ec8788", "chopping board-7ef65dd634818375", "peeler-dbb685063c0971f1", "wooden spoon-4fe1d002d616c358", "large bowl2-32313b1e44f54c2b", "grater-e04046b1e88e53fc", "checkered cup-8407dd6e3f183796", "spatula-c7bbb444bdd2fa08", "saucepan-cbaaa1f6dbd3bcae", "cup2-3e92f2e268100c60", "small bowl-19531017fe40f9f5", "fork3-e8fe95a2f79fbfc2", "glass lid-9645cf62b8c1bc25", "dishwasher tablet-7f3083bde5edf50f", "cling film box-f32a44e067e9495b", "plate3-db702c45e6481567", "cling film piece-bbc098e6d6256d4f", "salt container-52ef6e268931ad61", "olive oil can-0daf5fb1239c5a5e", "olive oil can lid-48e45174525cffac", "kitchen roll holder-4affb1ffbf0f53b7", "kitchen towel piece-3135440d60f34abc", "plastic box lid-395f0f65992dd7fd", "box of blueberries-4cbd3555e20cf2f6", "box of ginger ale-6d701c37e7081781", "sponge-8ba11674c08e1f05", "washing up liquid bottle-cbb406d282035d2d", "food plastic bag-62b489a241acecb6", "sunflower oil bottle-016a25c27adcfd70", "tea towel-b2b3bbbedbaf8438", "cleaner bottle-ed97c76412b01b78", "orange cloth-d88d45db26ef6ef3", "glass olive dispenser-0468a115a90f8695", "red box-9616027c0a41fabd", "soap tray-f9e374a459555f1f", "hand wash bottle-2277008d9bd4c399", "cutlery holder-d79a1d316057b23a", "sink strainer-130e2792c79ad064", "blue glass-fb829a159344e6ee", "yellow glass-c0668e82c9bfeeb3", "bread-928d4821edf64c71", "bread knife-a80f75e33111e600", "tea towel1-de0ef5905b3424f2", "tea towel2-0fccf24518b4ef9d", "cloth on the chair-3bcbc70f8a052f96", "broccoli-ba55634c53d529a8", "plant based burger box-53112e995b2cc80a", "plate of food-2bfe93ad34625d4e", "box of tomato-9603657c5f1eb59e", "box-f4c92874533fdbc5", "yellow box-54ca18951e555dad" ], "reasoning": "1. The user picked up the box of plastic wrap (cling film box-f32a44e067e9495b) at 09:52 and set down the box at 10:05, before picking it up again at 10:08, setting it down at 10:32, picking it up again at 10:40, and storing it away at 10:44. These give the timeframes of 09:52-10:05, 10:08-10:32, and 10:40-10:44.\n2. I observed within these timeframes for the answer options provided. The user pressed a button on the dishwasher at 09:54, wrapped the food at 10:23, and touched their face at 10:41. The user did not take the other provided actions.", "referenced_object_masks": [ "cling film box-f32a44e067e9495b" ] }, { "key": "P03-20240216-205923:13a046dca5502cfa55bc383a27880e73c85f8705", "video_id": "P03-20240216-205923", "question": "If the buttons on the dishwasher were numbered from right to left, what is the product of the numbers created from the sets of pressed and unpressed buttons?", "answer_choice_0": "279,882", "answer_choice_1": "281,232", "answer_choice_2": "290,062", "answer_choice_3": "285,422", "answer_choice_4": "283,502", "answer_id": 1, "answer": "281,232", "question_type": [ "Counterfactual", "Action Recognition", "Situational Awareness", "Object Recognition", "Numerical Reasoning" ], "objects_of_interest": [ "red plastic wrapping-514fe3f820b81cc3", "red plastic wrapping2-1e16148624f43cc6", "receipt-1767f107f2703993", "baking tray-cbdef058f6304d21", "mandarin peels-74e4dd3ae38236f4", "more mandarin peels-681c085d87f1dced", "plate-0cff09877fdaee8c", "cutlery-6e178c73b0f74860", "plate2-0903275c9558c237", "cloth-454c499f42afffda", "white bowl-1525e0c983913700", "large bowl-f8024bd5904d941b", "chocolate-a6aca8dd303fd074", "cloth2-cb3cc57f80844fbe", "plastic box-e3d1a75f812f4e50", "dough scraper-f61e170b287c43bc", "cup-d7da0d3fd46ab842", "chair-dad799f50ac463b6", "chair2-4b8b62479f537e90", "cork pot stand-f07c655b01b7f78e", "phone-ff7adada9e20bd38", "dishwasher brush-38aad133135d10ca", "spoon-a842ea6338c3ab5d", "spoon2-f23d09787407ff51", "knife-18c208541a819db5", "fork-282c5015dc24e049", "knife2-bb7371d0b14c4b9d", "fork2-a07cd0dd302adaae", "coffee jug-bca386177593e659", "food box lid-9221cf9681ec8788", "chopping board-7ef65dd634818375", "peeler-dbb685063c0971f1", "wooden spoon-4fe1d002d616c358", "large bowl2-32313b1e44f54c2b", "grater-e04046b1e88e53fc", "checkered cup-8407dd6e3f183796", "spatula-c7bbb444bdd2fa08", "saucepan-cbaaa1f6dbd3bcae", "cup2-3e92f2e268100c60", "small bowl-19531017fe40f9f5", "fork3-e8fe95a2f79fbfc2", "glass lid-9645cf62b8c1bc25", "dishwasher tablet-7f3083bde5edf50f", "cling film box-f32a44e067e9495b", "plate3-db702c45e6481567", "cling film piece-bbc098e6d6256d4f", "salt container-52ef6e268931ad61", "olive oil can-0daf5fb1239c5a5e", "olive oil can lid-48e45174525cffac", "kitchen roll holder-4affb1ffbf0f53b7", "kitchen towel piece-3135440d60f34abc", "plastic box lid-395f0f65992dd7fd", "box of blueberries-4cbd3555e20cf2f6", "box of ginger ale-6d701c37e7081781", "sponge-8ba11674c08e1f05", "washing up liquid bottle-cbb406d282035d2d", "food plastic bag-62b489a241acecb6", "sunflower oil bottle-016a25c27adcfd70", "tea towel-b2b3bbbedbaf8438", "cleaner bottle-ed97c76412b01b78", "orange cloth-d88d45db26ef6ef3", "glass olive dispenser-0468a115a90f8695", "red box-9616027c0a41fabd", "soap tray-f9e374a459555f1f", "hand wash bottle-2277008d9bd4c399", "cutlery holder-d79a1d316057b23a", "sink strainer-130e2792c79ad064", "blue glass-fb829a159344e6ee", "yellow glass-c0668e82c9bfeeb3", "bread-928d4821edf64c71", "bread knife-a80f75e33111e600", "tea towel1-de0ef5905b3424f2", "tea towel2-0fccf24518b4ef9d", "cloth on the chair-3bcbc70f8a052f96", "broccoli-ba55634c53d529a8", "plant based burger box-53112e995b2cc80a", "plate of food-2bfe93ad34625d4e", "box of tomato-9603657c5f1eb59e", "box-f4c92874533fdbc5", "yellow box-54ca18951e555dad" ], "reasoning": "1. I observed the user preparing the dishwasher at 09:30 when they placed the dishwasher tablet (dishwasher tablet-7f3083bde5edf50f) into the dishwasher and closed the door.\n2. At 09:35, I counted 6 buttons on the face of the dishwasher.\n3. If the buttons were labelled 1-6 from right to left, between 09:35 and 09:40, the user pressed buttons 6, 5, and 1, resulting in the number 651. This leaves buttons 4, 3, and 2 as the unpressed number, 432.\n4. 651 multiplied by 432 is 281,232.", "referenced_object_masks": [ "dishwasher tablet-7f3083bde5edf50f" ] }, { "key": "P03-20240216-205923:1723d49ee77a09f0e88a61980601f9632dd8bb20", "video_id": "P03-20240216-205923", "question": "From my perspective looking down, if the initial two cutlery handles were the ends of the hands of a clock on the first plate I set down on the counter, with the fork being the minute hand and the knife being the hour hand, and the last two forks during the time when I store cutlery from this plate into a dishwasher were the hands of a clock, with the minute hand being the fork that sticks out over the edge of the plate, approximately how many hours would have passed according to the handles of the cutlery?", "answer_choice_0": "2", "answer_choice_1": "5", "answer_choice_2": "4", "answer_choice_3": "1", "answer_choice_4": "3", "answer_id": 2, "answer": "4", "question_type": [ "Counterfactual", "Egocentric & Spatial Perception", "Object Recognition", "Numerical Reasoning" ], "objects_of_interest": [ "red plastic wrapping-514fe3f820b81cc3", "red plastic wrapping2-1e16148624f43cc6", "receipt-1767f107f2703993", "baking tray-cbdef058f6304d21", "mandarin peels-74e4dd3ae38236f4", "more mandarin peels-681c085d87f1dced", "plate-0cff09877fdaee8c", "cutlery-6e178c73b0f74860", "plate2-0903275c9558c237", "cloth-454c499f42afffda", "white bowl-1525e0c983913700", "large bowl-f8024bd5904d941b", "chocolate-a6aca8dd303fd074", "cloth2-cb3cc57f80844fbe", "plastic box-e3d1a75f812f4e50", "dough scraper-f61e170b287c43bc", "cup-d7da0d3fd46ab842", "chair-dad799f50ac463b6", "chair2-4b8b62479f537e90", "cork pot stand-f07c655b01b7f78e", "phone-ff7adada9e20bd38", "dishwasher brush-38aad133135d10ca", "spoon-a842ea6338c3ab5d", "spoon2-f23d09787407ff51", "knife-18c208541a819db5", "fork-282c5015dc24e049", "knife2-bb7371d0b14c4b9d", "fork2-a07cd0dd302adaae", "coffee jug-bca386177593e659", "food box lid-9221cf9681ec8788", "chopping board-7ef65dd634818375", "peeler-dbb685063c0971f1", "wooden spoon-4fe1d002d616c358", "large bowl2-32313b1e44f54c2b", "grater-e04046b1e88e53fc", "checkered cup-8407dd6e3f183796", "spatula-c7bbb444bdd2fa08", "saucepan-cbaaa1f6dbd3bcae", "cup2-3e92f2e268100c60", "small bowl-19531017fe40f9f5", "fork3-e8fe95a2f79fbfc2", "glass lid-9645cf62b8c1bc25", "dishwasher tablet-7f3083bde5edf50f", "cling film box-f32a44e067e9495b", "plate3-db702c45e6481567", "cling film piece-bbc098e6d6256d4f", "salt container-52ef6e268931ad61", "olive oil can-0daf5fb1239c5a5e", "olive oil can lid-48e45174525cffac", "kitchen roll holder-4affb1ffbf0f53b7", "kitchen towel piece-3135440d60f34abc", "plastic box lid-395f0f65992dd7fd", "box of blueberries-4cbd3555e20cf2f6", "box of ginger ale-6d701c37e7081781", "sponge-8ba11674c08e1f05", "washing up liquid bottle-cbb406d282035d2d", "food plastic bag-62b489a241acecb6", "sunflower oil bottle-016a25c27adcfd70", "tea towel-b2b3bbbedbaf8438", "cleaner bottle-ed97c76412b01b78", "orange cloth-d88d45db26ef6ef3", "glass olive dispenser-0468a115a90f8695", "red box-9616027c0a41fabd", "soap tray-f9e374a459555f1f", "hand wash bottle-2277008d9bd4c399", "cutlery holder-d79a1d316057b23a", "sink strainer-130e2792c79ad064", "blue glass-fb829a159344e6ee", "yellow glass-c0668e82c9bfeeb3", "bread-928d4821edf64c71", "bread knife-a80f75e33111e600", "tea towel1-de0ef5905b3424f2", "tea towel2-0fccf24518b4ef9d", "cloth on the chair-3bcbc70f8a052f96", "broccoli-ba55634c53d529a8", "plant based burger box-53112e995b2cc80a", "plate of food-2bfe93ad34625d4e", "box of tomato-9603657c5f1eb59e", "box-f4c92874533fdbc5", "yellow box-54ca18951e555dad" ], "reasoning": "1. The user grabbed the first plate (plate-0cff09877fdaee8c) at 00:46. I continued watching until the user set down this plate (plate-0cff09877fdaee8c) on the counter, which the user does at 00:50.At this moment, the knife (knife2-bb7371d0b14c4b9d), being the hour hand, and the initial fork (fork2-a07cd0dd302adaae), being the minute hand, display an approximate time of 01:45.\n2. I watched when the user began storing the cutlery (cutlery-6e178c73b0f74860) from the first plate (plate-0cff09877fdaee8c), which began at 03:25.\n3. I waited until there were two forks left on the plate (plate-0cff09877fdaee8c), which is clearly visible at 03:39 when the user leans toward the plate (plate-0cff09877fdaee8c). Because the handle of the initial fork (fork2-a07cd0dd302adaae), the one pointing away from the user, sticks out over the edge of the plate, I counted this as the minute hand. With the second fork (fork-282c5015dc24e049), they create an approximate time of 05:55.\n4. From 01:45 to 05:55, approximately 4 hours would have passed.", "referenced_object_masks": [ "cutlery-6e178c73b0f74860", "knife2-bb7371d0b14c4b9d", "fork-282c5015dc24e049", "fork2-a07cd0dd302adaae", "plate-0cff09877fdaee8c" ] }, { "key": "P03-20240216-205923:5427a5f6dc82c69f2f2ae60e9b701edcf5ca7eb4", "video_id": "P03-20240216-205923", "question": "Where is the sink in relation to me after I turn when I'm folding the second gray towel?", "answer_choice_0": "Behind me", "answer_choice_1": "There is no sink", "answer_choice_2": "On my right", "answer_choice_3": "Across the room", "answer_choice_4": "On my left", "answer_id": 2, "answer": "On my right", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Counting", "Action Recognition", "Situational Awareness" ], "objects_of_interest": [ "red plastic wrapping-514fe3f820b81cc3", "red plastic wrapping2-1e16148624f43cc6", "receipt-1767f107f2703993", "baking tray-cbdef058f6304d21", "mandarin peels-74e4dd3ae38236f4", "more mandarin peels-681c085d87f1dced", "plate-0cff09877fdaee8c", "cutlery-6e178c73b0f74860", "plate2-0903275c9558c237", "cloth-454c499f42afffda", "white bowl-1525e0c983913700", "large bowl-f8024bd5904d941b", "chocolate-a6aca8dd303fd074", "cloth2-cb3cc57f80844fbe", "plastic box-e3d1a75f812f4e50", "dough scraper-f61e170b287c43bc", "cup-d7da0d3fd46ab842", "chair-dad799f50ac463b6", "chair2-4b8b62479f537e90", "cork pot stand-f07c655b01b7f78e", "phone-ff7adada9e20bd38", "dishwasher brush-38aad133135d10ca", "spoon-a842ea6338c3ab5d", "spoon2-f23d09787407ff51", "knife-18c208541a819db5", "fork-282c5015dc24e049", "knife2-bb7371d0b14c4b9d", "fork2-a07cd0dd302adaae", "coffee jug-bca386177593e659", "food box lid-9221cf9681ec8788", "chopping board-7ef65dd634818375", "peeler-dbb685063c0971f1", "wooden spoon-4fe1d002d616c358", "large bowl2-32313b1e44f54c2b", "grater-e04046b1e88e53fc", "checkered cup-8407dd6e3f183796", "spatula-c7bbb444bdd2fa08", "saucepan-cbaaa1f6dbd3bcae", "cup2-3e92f2e268100c60", "small bowl-19531017fe40f9f5", "fork3-e8fe95a2f79fbfc2", "glass lid-9645cf62b8c1bc25", "dishwasher tablet-7f3083bde5edf50f", "cling film box-f32a44e067e9495b", "plate3-db702c45e6481567", "cling film piece-bbc098e6d6256d4f", "salt container-52ef6e268931ad61", "olive oil can-0daf5fb1239c5a5e", "olive oil can lid-48e45174525cffac", "kitchen roll holder-4affb1ffbf0f53b7", "kitchen towel piece-3135440d60f34abc", "plastic box lid-395f0f65992dd7fd", "box of blueberries-4cbd3555e20cf2f6", "box of ginger ale-6d701c37e7081781", "sponge-8ba11674c08e1f05", "washing up liquid bottle-cbb406d282035d2d", "food plastic bag-62b489a241acecb6", "sunflower oil bottle-016a25c27adcfd70", "tea towel-b2b3bbbedbaf8438", "cleaner bottle-ed97c76412b01b78", "orange cloth-d88d45db26ef6ef3", "glass olive dispenser-0468a115a90f8695", "red box-9616027c0a41fabd", "soap tray-f9e374a459555f1f", "hand wash bottle-2277008d9bd4c399", "cutlery holder-d79a1d316057b23a", "sink strainer-130e2792c79ad064", "blue glass-fb829a159344e6ee", "yellow glass-c0668e82c9bfeeb3", "bread-928d4821edf64c71", "bread knife-a80f75e33111e600", "tea towel1-de0ef5905b3424f2", "tea towel2-0fccf24518b4ef9d", "cloth on the chair-3bcbc70f8a052f96", "broccoli-ba55634c53d529a8", "plant based burger box-53112e995b2cc80a", "plate of food-2bfe93ad34625d4e", "box of tomato-9603657c5f1eb59e", "box-f4c92874533fdbc5", "yellow box-54ca18951e555dad" ], "reasoning": "1. I determine when the user folds the first gray towel (tea towel1-de0ef5905b3424f2). This occurs from 21:47-21:53. \n2. The user folds the second gray towel (tea towel2-0fccf24518b4ef9d) from 21:56-22:04.\n3. As the user folds the second towel (tea towel2-0fccf24518b4ef9d), the sink is positioned in front of the user until they turn slightly left at 22:00. The sink is then positioned on their right.", "referenced_object_masks": [ "tea towel1-de0ef5905b3424f2", "tea towel2-0fccf24518b4ef9d" ] }, { "key": "P03-20240216-205923:638b74625303f3ccf67b389a8f3d51f1db28711b", "video_id": "P03-20240216-205923", "question": "After positioning the casserole dish in the dishwasher, what is the third item that I did not scrub off before placing in the dishwasher?", "answer_choice_0": "Second knife", "answer_choice_1": "First spoon", "answer_choice_2": "First fork", "answer_choice_3": "Second fork", "answer_choice_4": "Second spoon", "answer_id": 2, "answer": "First fork", "question_type": [ "Action Recognition", "Counting", "Object Recognition" ], "objects_of_interest": [ "red plastic wrapping-514fe3f820b81cc3", "red plastic wrapping2-1e16148624f43cc6", "receipt-1767f107f2703993", "baking tray-cbdef058f6304d21", "mandarin peels-74e4dd3ae38236f4", "more mandarin peels-681c085d87f1dced", "plate-0cff09877fdaee8c", "cutlery-6e178c73b0f74860", "plate2-0903275c9558c237", "cloth-454c499f42afffda", "white bowl-1525e0c983913700", "large bowl-f8024bd5904d941b", "chocolate-a6aca8dd303fd074", "cloth2-cb3cc57f80844fbe", "plastic box-e3d1a75f812f4e50", "dough scraper-f61e170b287c43bc", "cup-d7da0d3fd46ab842", "chair-dad799f50ac463b6", "chair2-4b8b62479f537e90", "cork pot stand-f07c655b01b7f78e", "phone-ff7adada9e20bd38", "dishwasher brush-38aad133135d10ca", "spoon-a842ea6338c3ab5d", "spoon2-f23d09787407ff51", "knife-18c208541a819db5", "fork-282c5015dc24e049", "knife2-bb7371d0b14c4b9d", "fork2-a07cd0dd302adaae", "coffee jug-bca386177593e659", "food box lid-9221cf9681ec8788", "chopping board-7ef65dd634818375", "peeler-dbb685063c0971f1", "wooden spoon-4fe1d002d616c358", "large bowl2-32313b1e44f54c2b", "grater-e04046b1e88e53fc", "checkered cup-8407dd6e3f183796", "spatula-c7bbb444bdd2fa08", "saucepan-cbaaa1f6dbd3bcae", "cup2-3e92f2e268100c60", "small bowl-19531017fe40f9f5", "fork3-e8fe95a2f79fbfc2", "glass lid-9645cf62b8c1bc25", "dishwasher tablet-7f3083bde5edf50f", "cling film box-f32a44e067e9495b", "plate3-db702c45e6481567", "cling film piece-bbc098e6d6256d4f", "salt container-52ef6e268931ad61", "olive oil can-0daf5fb1239c5a5e", "olive oil can lid-48e45174525cffac", "kitchen roll holder-4affb1ffbf0f53b7", "kitchen towel piece-3135440d60f34abc", "plastic box lid-395f0f65992dd7fd", "box of blueberries-4cbd3555e20cf2f6", "box of ginger ale-6d701c37e7081781", "sponge-8ba11674c08e1f05", "washing up liquid bottle-cbb406d282035d2d", "food plastic bag-62b489a241acecb6", "sunflower oil bottle-016a25c27adcfd70", "tea towel-b2b3bbbedbaf8438", "cleaner bottle-ed97c76412b01b78", "orange cloth-d88d45db26ef6ef3", "glass olive dispenser-0468a115a90f8695", "red box-9616027c0a41fabd", "soap tray-f9e374a459555f1f", "hand wash bottle-2277008d9bd4c399", "cutlery holder-d79a1d316057b23a", "sink strainer-130e2792c79ad064", "blue glass-fb829a159344e6ee", "yellow glass-c0668e82c9bfeeb3", "bread-928d4821edf64c71", "bread knife-a80f75e33111e600", "tea towel1-de0ef5905b3424f2", "tea towel2-0fccf24518b4ef9d", "cloth on the chair-3bcbc70f8a052f96", "broccoli-ba55634c53d529a8", "plant based burger box-53112e995b2cc80a", "plate of food-2bfe93ad34625d4e", "box of tomato-9603657c5f1eb59e", "box-f4c92874533fdbc5", "yellow box-54ca18951e555dad" ], "reasoning": "1. I determine when the user positions the casserole dish (baking tray-cbdef058f6304d21) in the dishwasher. This occurs from 03:12-03:23. \n2. Next, I determine the items the user puts into the dishwasher. The first item includes a knife at 03:30, second, a spoon (spoon-a842ea6338c3ab5d) at 03:33, third, another spoon (spoon2-f23d09787407ff51) at 03:35, fourth, a plastic knife (knife-18c208541a819db5) at 03:39, fifth, a fork (fork-282c5015dc24e049) at 03:43, sixth, another knife (knife2-bb7371d0b14c4b9d) at 03:48, and finally, seventh, another fork (fork2-a07cd0dd302adaae) at 03:51.\n3. The user does not scrub the spoon (spoon-a842ea6338c3ab5d) at 03:33, the spoon (spoon2-f23d09787407ff51) at 03:35, the fork (fork-282c5015dc24e049) at 03:43, or the fork (fork2-a07cd0dd302adaae) at 03:51. \n4. Therefore, the first fork (fork-282c5015dc24e049) is the third item the user does not scrub.", "referenced_object_masks": [ "spoon2-f23d09787407ff51", "baking tray-cbdef058f6304d21", "knife2-bb7371d0b14c4b9d", "fork-282c5015dc24e049", "fork2-a07cd0dd302adaae", "knife-18c208541a819db5", "spoon-a842ea6338c3ab5d" ] }, { "key": "P03-20240216-205923:71e881f8fb35d7dd56f7b117703d4ccd8b22f492", "video_id": "P03-20240216-205923", "question": "Throughout the video, what is the correct order of each location of the item that I initially moved to make room for the ninth item that I brushed over the sink?", "answer_choice_0": "Top rack on the left, top rack on the right, counter, bottom rack on the right", "answer_choice_1": "Top rack on the left, top rack on the right, counter, top rack on the right", "answer_choice_2": "Top rack on the left, bottom rack on the right, counter, bottom rack on the right", "answer_choice_3": "Top rack on the left, top rack on the right, counter, top rack on the left", "answer_choice_4": "Top rack on the left, bottom rack on the left, counter, bottom rack on the right", "answer_id": 1, "answer": "Top rack on the left, top rack on the right, counter, top rack on the right", "question_type": [ "State Changes", "Situational Awareness", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "red plastic wrapping-514fe3f820b81cc3", "red plastic wrapping2-1e16148624f43cc6", "receipt-1767f107f2703993", "baking tray-cbdef058f6304d21", "mandarin peels-74e4dd3ae38236f4", "more mandarin peels-681c085d87f1dced", "plate-0cff09877fdaee8c", "cutlery-6e178c73b0f74860", "plate2-0903275c9558c237", "cloth-454c499f42afffda", "white bowl-1525e0c983913700", "large bowl-f8024bd5904d941b", "chocolate-a6aca8dd303fd074", "cloth2-cb3cc57f80844fbe", "plastic box-e3d1a75f812f4e50", "dough scraper-f61e170b287c43bc", "cup-d7da0d3fd46ab842", "chair-dad799f50ac463b6", "chair2-4b8b62479f537e90", "cork pot stand-f07c655b01b7f78e", "phone-ff7adada9e20bd38", "dishwasher brush-38aad133135d10ca", "spoon-a842ea6338c3ab5d", "spoon2-f23d09787407ff51", "knife-18c208541a819db5", "fork-282c5015dc24e049", "knife2-bb7371d0b14c4b9d", "fork2-a07cd0dd302adaae", "coffee jug-bca386177593e659", "food box lid-9221cf9681ec8788", "chopping board-7ef65dd634818375", "peeler-dbb685063c0971f1", "wooden spoon-4fe1d002d616c358", "large bowl2-32313b1e44f54c2b", "grater-e04046b1e88e53fc", "checkered cup-8407dd6e3f183796", "spatula-c7bbb444bdd2fa08", "saucepan-cbaaa1f6dbd3bcae", "cup2-3e92f2e268100c60", "small bowl-19531017fe40f9f5", "fork3-e8fe95a2f79fbfc2", "glass lid-9645cf62b8c1bc25", "dishwasher tablet-7f3083bde5edf50f", "cling film box-f32a44e067e9495b", "plate3-db702c45e6481567", "cling film piece-bbc098e6d6256d4f", "salt container-52ef6e268931ad61", "olive oil can-0daf5fb1239c5a5e", "olive oil can lid-48e45174525cffac", "kitchen roll holder-4affb1ffbf0f53b7", "kitchen towel piece-3135440d60f34abc", "plastic box lid-395f0f65992dd7fd", "box of blueberries-4cbd3555e20cf2f6", "box of ginger ale-6d701c37e7081781", "sponge-8ba11674c08e1f05", "washing up liquid bottle-cbb406d282035d2d", "food plastic bag-62b489a241acecb6", "sunflower oil bottle-016a25c27adcfd70", "tea towel-b2b3bbbedbaf8438", "cleaner bottle-ed97c76412b01b78", "orange cloth-d88d45db26ef6ef3", "glass olive dispenser-0468a115a90f8695", "red box-9616027c0a41fabd", "soap tray-f9e374a459555f1f", "hand wash bottle-2277008d9bd4c399", "cutlery holder-d79a1d316057b23a", "sink strainer-130e2792c79ad064", "blue glass-fb829a159344e6ee", "yellow glass-c0668e82c9bfeeb3", "bread-928d4821edf64c71", "bread knife-a80f75e33111e600", "tea towel1-de0ef5905b3424f2", "tea towel2-0fccf24518b4ef9d", "cloth on the chair-3bcbc70f8a052f96", "broccoli-ba55634c53d529a8", "plant based burger box-53112e995b2cc80a", "plate of food-2bfe93ad34625d4e", "box of tomato-9603657c5f1eb59e", "box-f4c92874533fdbc5", "yellow box-54ca18951e555dad" ], "reasoning": "1. I counted each item that the user brushed over the sink beginning with the baking tray (baking tray-cbdef058f6304d21) at 02:40.\n2. At 04:58, the user begins brushing the ninth item over the sink, the large bowl (large bowl2-32313b1e44f54c2b).\n3. I watched the user put the large bowl (large bowl2-32313b1e44f54c2b) in the dishwasher at 05:07. The user had to move the grater (grater-e04046b1e88e53fc) from the left on the top rack to the right on the top rack to make space.\n4. I continued watching the video to determine the positions of the grater (grater-e04046b1e88e53fc).\n5. At 05:39, the user moves the grater (grater-e04046b1e88e53fc) from the top rack on the right to the counter.\n6. At 08:31, the user places the grater (grater-e04046b1e88e53fc) in the top rack on the right where it remains until the end of the video.", "referenced_object_masks": [ "baking tray-cbdef058f6304d21", "grater-e04046b1e88e53fc", "large bowl2-32313b1e44f54c2b" ] }, { "key": "P03-20240216-205923:7fd656f7f15a7e974331d31b913dc7afbedb100a", "video_id": "P03-20240216-205923", "question": "If I reversed the location where I disposed of the red wrapper and the orange peels, where would the refrigerator be located when I toss away the red wrapper?", "answer_choice_0": "On the left", "answer_choice_1": "Behind and on the right", "answer_choice_2": "On the right", "answer_choice_3": "In front of me", "answer_choice_4": "Behind and on the left", "answer_id": 4, "answer": "Behind and on the left", "question_type": [ "Action Recognition", "Object Recognition", "Egocentric & Spatial Perception", "Counterfactual" ], "objects_of_interest": [ "red plastic wrapping-514fe3f820b81cc3", "red plastic wrapping2-1e16148624f43cc6", "receipt-1767f107f2703993", "baking tray-cbdef058f6304d21", "mandarin peels-74e4dd3ae38236f4", "more mandarin peels-681c085d87f1dced", "plate-0cff09877fdaee8c", "cutlery-6e178c73b0f74860", "plate2-0903275c9558c237", "cloth-454c499f42afffda", "white bowl-1525e0c983913700", "large bowl-f8024bd5904d941b", "chocolate-a6aca8dd303fd074", "cloth2-cb3cc57f80844fbe", "plastic box-e3d1a75f812f4e50", "dough scraper-f61e170b287c43bc", "cup-d7da0d3fd46ab842", "chair-dad799f50ac463b6", "chair2-4b8b62479f537e90", "cork pot stand-f07c655b01b7f78e", "phone-ff7adada9e20bd38", "dishwasher brush-38aad133135d10ca", "spoon-a842ea6338c3ab5d", "spoon2-f23d09787407ff51", "knife-18c208541a819db5", "fork-282c5015dc24e049", "knife2-bb7371d0b14c4b9d", "fork2-a07cd0dd302adaae", "coffee jug-bca386177593e659", "food box lid-9221cf9681ec8788", "chopping board-7ef65dd634818375", "peeler-dbb685063c0971f1", "wooden spoon-4fe1d002d616c358", "large bowl2-32313b1e44f54c2b", "grater-e04046b1e88e53fc", "checkered cup-8407dd6e3f183796", "spatula-c7bbb444bdd2fa08", "saucepan-cbaaa1f6dbd3bcae", "cup2-3e92f2e268100c60", "small bowl-19531017fe40f9f5", "fork3-e8fe95a2f79fbfc2", "glass lid-9645cf62b8c1bc25", "dishwasher tablet-7f3083bde5edf50f", "cling film box-f32a44e067e9495b", "plate3-db702c45e6481567", "cling film piece-bbc098e6d6256d4f", "salt container-52ef6e268931ad61", "olive oil can-0daf5fb1239c5a5e", "olive oil can lid-48e45174525cffac", "kitchen roll holder-4affb1ffbf0f53b7", "kitchen towel piece-3135440d60f34abc", "plastic box lid-395f0f65992dd7fd", "box of blueberries-4cbd3555e20cf2f6", "box of ginger ale-6d701c37e7081781", "sponge-8ba11674c08e1f05", "washing up liquid bottle-cbb406d282035d2d", "food plastic bag-62b489a241acecb6", "sunflower oil bottle-016a25c27adcfd70", "tea towel-b2b3bbbedbaf8438", "cleaner bottle-ed97c76412b01b78", "orange cloth-d88d45db26ef6ef3", "glass olive dispenser-0468a115a90f8695", "red box-9616027c0a41fabd", "soap tray-f9e374a459555f1f", "hand wash bottle-2277008d9bd4c399", "cutlery holder-d79a1d316057b23a", "sink strainer-130e2792c79ad064", "blue glass-fb829a159344e6ee", "yellow glass-c0668e82c9bfeeb3", "bread-928d4821edf64c71", "bread knife-a80f75e33111e600", "tea towel1-de0ef5905b3424f2", "tea towel2-0fccf24518b4ef9d", "cloth on the chair-3bcbc70f8a052f96", "broccoli-ba55634c53d529a8", "plant based burger box-53112e995b2cc80a", "plate of food-2bfe93ad34625d4e", "box of tomato-9603657c5f1eb59e", "box-f4c92874533fdbc5", "yellow box-54ca18951e555dad" ], "reasoning": "1. The user picks up the red wrapper (red plastic wrapping-514fe3f820b81cc3) at 00:04.\n2. The user then crumples the receipt (receipt-1767f107f2703993) with the red wrapper (red plastic wrapping-514fe3f820b81cc3) inside and tosses it into the trash bin next to the refrigerator at 00:10.\n3. The user grabs the orange peels (mandarin peels-74e4dd3ae38236f4) at 00:20, then walks to the trash bin under the sink to dispose of them at 00:31.\n4. If these locations were reversed, the user would have disposed of the red wrapper (red plastic wrapping-514fe3f820b81cc3) in the trash bin under the sink .\n5. When the user stands in front of the sink, the refrigerator is located behind them and on the left.", "referenced_object_masks": [ "receipt-1767f107f2703993", "red plastic wrapping-514fe3f820b81cc3", "mandarin peels-74e4dd3ae38236f4" ] }, { "key": "P03-20240216-205923:83783d14ca8a9eb3b8dfc4d8dbffba2c82236b99", "video_id": "P03-20240216-205923", "question": "How many plates are on the table when I enter the kitchen multiplied by the number of bowls visible in the sink when I'm scrubbing the casserole dish?", "answer_choice_0": "7", "answer_choice_1": "9", "answer_choice_2": "5", "answer_choice_3": "4", "answer_choice_4": "6", "answer_id": 4, "answer": "6", "question_type": [ "Counting", "Numerical Reasoning", "Object Recognition", "Situational Awareness", "Action Recognition" ], "objects_of_interest": [ "red plastic wrapping-514fe3f820b81cc3", "red plastic wrapping2-1e16148624f43cc6", "receipt-1767f107f2703993", "baking tray-cbdef058f6304d21", "mandarin peels-74e4dd3ae38236f4", "more mandarin peels-681c085d87f1dced", "plate-0cff09877fdaee8c", "cutlery-6e178c73b0f74860", "plate2-0903275c9558c237", "cloth-454c499f42afffda", "white bowl-1525e0c983913700", "large bowl-f8024bd5904d941b", "chocolate-a6aca8dd303fd074", "cloth2-cb3cc57f80844fbe", "plastic box-e3d1a75f812f4e50", "dough scraper-f61e170b287c43bc", "cup-d7da0d3fd46ab842", "chair-dad799f50ac463b6", "chair2-4b8b62479f537e90", "cork pot stand-f07c655b01b7f78e", "phone-ff7adada9e20bd38", "dishwasher brush-38aad133135d10ca", "spoon-a842ea6338c3ab5d", "spoon2-f23d09787407ff51", "knife-18c208541a819db5", "fork-282c5015dc24e049", "knife2-bb7371d0b14c4b9d", "fork2-a07cd0dd302adaae", "coffee jug-bca386177593e659", "food box lid-9221cf9681ec8788", "chopping board-7ef65dd634818375", "peeler-dbb685063c0971f1", "wooden spoon-4fe1d002d616c358", "large bowl2-32313b1e44f54c2b", "grater-e04046b1e88e53fc", "checkered cup-8407dd6e3f183796", "spatula-c7bbb444bdd2fa08", "saucepan-cbaaa1f6dbd3bcae", "cup2-3e92f2e268100c60", "small bowl-19531017fe40f9f5", "fork3-e8fe95a2f79fbfc2", "glass lid-9645cf62b8c1bc25", "dishwasher tablet-7f3083bde5edf50f", "cling film box-f32a44e067e9495b", "plate3-db702c45e6481567", "cling film piece-bbc098e6d6256d4f", "salt container-52ef6e268931ad61", "olive oil can-0daf5fb1239c5a5e", "olive oil can lid-48e45174525cffac", "kitchen roll holder-4affb1ffbf0f53b7", "kitchen towel piece-3135440d60f34abc", "plastic box lid-395f0f65992dd7fd", "box of blueberries-4cbd3555e20cf2f6", "box of ginger ale-6d701c37e7081781", "sponge-8ba11674c08e1f05", "washing up liquid bottle-cbb406d282035d2d", "food plastic bag-62b489a241acecb6", "sunflower oil bottle-016a25c27adcfd70", "tea towel-b2b3bbbedbaf8438", "cleaner bottle-ed97c76412b01b78", "orange cloth-d88d45db26ef6ef3", "glass olive dispenser-0468a115a90f8695", "red box-9616027c0a41fabd", "soap tray-f9e374a459555f1f", "hand wash bottle-2277008d9bd4c399", "cutlery holder-d79a1d316057b23a", "sink strainer-130e2792c79ad064", "blue glass-fb829a159344e6ee", "yellow glass-c0668e82c9bfeeb3", "bread-928d4821edf64c71", "bread knife-a80f75e33111e600", "tea towel1-de0ef5905b3424f2", "tea towel2-0fccf24518b4ef9d", "cloth on the chair-3bcbc70f8a052f96", "broccoli-ba55634c53d529a8", "plant based burger box-53112e995b2cc80a", "plate of food-2bfe93ad34625d4e", "box of tomato-9603657c5f1eb59e", "box-f4c92874533fdbc5", "yellow box-54ca18951e555dad" ], "reasoning": "1. First, I locate a shot where the table is visible as the user enters the kitchen. I found it at 00:03, on the right, with 3 white plates (plate-0cff09877fdaee8c, plate2-0903275c9558c237, plate3-db702c45e6481567) on top. \n2. Next, I locate the timestamp where the user is scrubbing a casserole dish (baking tray-cbdef058f6304d21). This occurs from 02:41-03:09. \n3. I observed this segment to see the items visible in the sink. Two white bowls (large bowl-f8024bd5904d941b, white bowl-1525e0c983913700) are in the sink. \n4. I multiplied the 3 plates on the table by the 2 bowls in the sink to get 6.", "referenced_object_masks": [ "baking tray-cbdef058f6304d21", "large bowl-f8024bd5904d941b", "white bowl-1525e0c983913700", "plate-0cff09877fdaee8c", "plate2-0903275c9558c237", "plate3-db702c45e6481567" ] }, { "key": "P03-20240216-205923:acc870744de04027488fbe285964bb045b92cd74", "video_id": "P03-20240216-205923", "question": "How many times do I push the camera up on my face with my index finger before scraping the dough?", "answer_choice_0": "7", "answer_choice_1": "5", "answer_choice_2": "4", "answer_choice_3": "3", "answer_choice_4": "9", "answer_id": 3, "answer": "3", "question_type": [ "Counting", "Action Recognition", "Object Recognition", "Temporal Reasoning", "Situational Awareness" ], "objects_of_interest": [ "red plastic wrapping-514fe3f820b81cc3", "red plastic wrapping2-1e16148624f43cc6", "receipt-1767f107f2703993", "baking tray-cbdef058f6304d21", "mandarin peels-74e4dd3ae38236f4", "more mandarin peels-681c085d87f1dced", "plate-0cff09877fdaee8c", "cutlery-6e178c73b0f74860", "plate2-0903275c9558c237", "cloth-454c499f42afffda", "white bowl-1525e0c983913700", "large bowl-f8024bd5904d941b", "chocolate-a6aca8dd303fd074", "cloth2-cb3cc57f80844fbe", "plastic box-e3d1a75f812f4e50", "dough scraper-f61e170b287c43bc", "cup-d7da0d3fd46ab842", "chair-dad799f50ac463b6", "chair2-4b8b62479f537e90", "cork pot stand-f07c655b01b7f78e", "phone-ff7adada9e20bd38", "dishwasher brush-38aad133135d10ca", "spoon-a842ea6338c3ab5d", "spoon2-f23d09787407ff51", "knife-18c208541a819db5", "fork-282c5015dc24e049", "knife2-bb7371d0b14c4b9d", "fork2-a07cd0dd302adaae", "coffee jug-bca386177593e659", "food box lid-9221cf9681ec8788", "chopping board-7ef65dd634818375", "peeler-dbb685063c0971f1", "wooden spoon-4fe1d002d616c358", "large bowl2-32313b1e44f54c2b", "grater-e04046b1e88e53fc", "checkered cup-8407dd6e3f183796", "spatula-c7bbb444bdd2fa08", "saucepan-cbaaa1f6dbd3bcae", "cup2-3e92f2e268100c60", "small bowl-19531017fe40f9f5", "fork3-e8fe95a2f79fbfc2", "glass lid-9645cf62b8c1bc25", "dishwasher tablet-7f3083bde5edf50f", "cling film box-f32a44e067e9495b", "plate3-db702c45e6481567", "cling film piece-bbc098e6d6256d4f", "salt container-52ef6e268931ad61", "olive oil can-0daf5fb1239c5a5e", "olive oil can lid-48e45174525cffac", "kitchen roll holder-4affb1ffbf0f53b7", "kitchen towel piece-3135440d60f34abc", "plastic box lid-395f0f65992dd7fd", "box of blueberries-4cbd3555e20cf2f6", "box of ginger ale-6d701c37e7081781", "sponge-8ba11674c08e1f05", "washing up liquid bottle-cbb406d282035d2d", "food plastic bag-62b489a241acecb6", "sunflower oil bottle-016a25c27adcfd70", "tea towel-b2b3bbbedbaf8438", "cleaner bottle-ed97c76412b01b78", "orange cloth-d88d45db26ef6ef3", "glass olive dispenser-0468a115a90f8695", "red box-9616027c0a41fabd", "soap tray-f9e374a459555f1f", "hand wash bottle-2277008d9bd4c399", "cutlery holder-d79a1d316057b23a", "sink strainer-130e2792c79ad064", "blue glass-fb829a159344e6ee", "yellow glass-c0668e82c9bfeeb3", "bread-928d4821edf64c71", "bread knife-a80f75e33111e600", "tea towel1-de0ef5905b3424f2", "tea towel2-0fccf24518b4ef9d", "cloth on the chair-3bcbc70f8a052f96", "broccoli-ba55634c53d529a8", "plant based burger box-53112e995b2cc80a", "plate of food-2bfe93ad34625d4e", "box of tomato-9603657c5f1eb59e", "box-f4c92874533fdbc5", "yellow box-54ca18951e555dad" ], "reasoning": "1. First, I determine when the user scrapes the dough . This occurs at 01:37 when the user picks up the scraper (dough scraper-f61e170b287c43bc) and scrapes and turns the dough until 01:44. \n2. Next, I rewind the video to determine all the times the user pushes the camera up on their face before 01:37. This occurs at 00:02, 00:36, and 00:43. \n3. I conclude that the user pushes the camera up with their index finger 3 times before scraping the dough .", "referenced_object_masks": [ "dough scraper-f61e170b287c43bc" ] }, { "key": "P03-20240216-205923:c18930cb49e2f36937c2ac340ad0f65099c66542", "video_id": "P03-20240216-205923", "question": "Where is my gaze when I put the oil canister away compared to when I'm wiping the backsplash tiles and the stovetop disappears partially out of view?", "answer_choice_0": "Down both times", "answer_choice_1": "Up both times", "answer_choice_2": "Down; Up", "answer_choice_3": "Behind; Straight Ahead", "answer_choice_4": "Left; Right", "answer_id": 2, "answer": "Down; Up", "question_type": [ "Egocentric & Spatial Perception", "Action Recognition", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "red plastic wrapping-514fe3f820b81cc3", "red plastic wrapping2-1e16148624f43cc6", "receipt-1767f107f2703993", "baking tray-cbdef058f6304d21", "mandarin peels-74e4dd3ae38236f4", "more mandarin peels-681c085d87f1dced", "plate-0cff09877fdaee8c", "cutlery-6e178c73b0f74860", "plate2-0903275c9558c237", "cloth-454c499f42afffda", "white bowl-1525e0c983913700", "large bowl-f8024bd5904d941b", "chocolate-a6aca8dd303fd074", "cloth2-cb3cc57f80844fbe", "plastic box-e3d1a75f812f4e50", "dough scraper-f61e170b287c43bc", "cup-d7da0d3fd46ab842", "chair-dad799f50ac463b6", "chair2-4b8b62479f537e90", "cork pot stand-f07c655b01b7f78e", "phone-ff7adada9e20bd38", "dishwasher brush-38aad133135d10ca", "spoon-a842ea6338c3ab5d", "spoon2-f23d09787407ff51", "knife-18c208541a819db5", "fork-282c5015dc24e049", "knife2-bb7371d0b14c4b9d", "fork2-a07cd0dd302adaae", "coffee jug-bca386177593e659", "food box lid-9221cf9681ec8788", "chopping board-7ef65dd634818375", "peeler-dbb685063c0971f1", "wooden spoon-4fe1d002d616c358", "large bowl2-32313b1e44f54c2b", "grater-e04046b1e88e53fc", "checkered cup-8407dd6e3f183796", "spatula-c7bbb444bdd2fa08", "saucepan-cbaaa1f6dbd3bcae", "cup2-3e92f2e268100c60", "small bowl-19531017fe40f9f5", "fork3-e8fe95a2f79fbfc2", "glass lid-9645cf62b8c1bc25", "dishwasher tablet-7f3083bde5edf50f", "cling film box-f32a44e067e9495b", "plate3-db702c45e6481567", "cling film piece-bbc098e6d6256d4f", "salt container-52ef6e268931ad61", "olive oil can-0daf5fb1239c5a5e", "olive oil can lid-48e45174525cffac", "kitchen roll holder-4affb1ffbf0f53b7", "kitchen towel piece-3135440d60f34abc", "plastic box lid-395f0f65992dd7fd", "box of blueberries-4cbd3555e20cf2f6", "box of ginger ale-6d701c37e7081781", "sponge-8ba11674c08e1f05", "washing up liquid bottle-cbb406d282035d2d", "food plastic bag-62b489a241acecb6", "sunflower oil bottle-016a25c27adcfd70", "tea towel-b2b3bbbedbaf8438", "cleaner bottle-ed97c76412b01b78", "orange cloth-d88d45db26ef6ef3", "glass olive dispenser-0468a115a90f8695", "red box-9616027c0a41fabd", "soap tray-f9e374a459555f1f", "hand wash bottle-2277008d9bd4c399", "cutlery holder-d79a1d316057b23a", "sink strainer-130e2792c79ad064", "blue glass-fb829a159344e6ee", "yellow glass-c0668e82c9bfeeb3", "bread-928d4821edf64c71", "bread knife-a80f75e33111e600", "tea towel1-de0ef5905b3424f2", "tea towel2-0fccf24518b4ef9d", "cloth on the chair-3bcbc70f8a052f96", "broccoli-ba55634c53d529a8", "plant based burger box-53112e995b2cc80a", "plate of food-2bfe93ad34625d4e", "box of tomato-9603657c5f1eb59e", "box-f4c92874533fdbc5", "yellow box-54ca18951e555dad" ], "reasoning": "1. First, I determine when the user puts the oil canister (sunflower oil bottle-016a25c27adcfd70) away. This occurs at 16:04 when the user is looking down to put the canister (sunflower oil bottle-016a25c27adcfd70) into a lower cabinet drawer. \n2. Next, I determine when the user is wiping the backsplash. This occurs at 18:49. At 18:51, the user looks up and their gaze captures the recess lighting positioned underneath the cabinet. At this moment, the stovetop disappears partially from view.\n3. Therefore, the user's gaze is first looking down and then up.", "referenced_object_masks": [ "sunflower oil bottle-016a25c27adcfd70" ] }, { "key": "P03-20240216-223126:0b7b0afa3c1943e4bf24ab038e0a06ad21c6c56c", "video_id": "P03-20240216-223126", "question": "What was the third item I opened?", "answer_choice_0": "The dishwasher", "answer_choice_1": "A bag of chips", "answer_choice_2": "A kitchen cabinet", "answer_choice_3": "A ginger ale bottle", "answer_choice_4": "A bottle of beer", "answer_id": 1, "answer": "A bag of chips", "question_type": [ "Object Recognition", "Counting", "Action Recognition", "State Changes" ], "objects_of_interest": [ "bag of crisps-6ba2599c74d27ce7", "bowl-0d6cc7c42afdcbab", "clip-6fad3ed0f6869c34", "bottle of beer-2701bc1262645e00", "other bottle-adec558f224af3f4", "ginger ale pack-0c982fe6cdc8db1c", "lid-bc870763b99ea600", "ginger ale bottle-96b9be650939e6d6", "glass-6d7989bad9d5466a", "bottle opener-fec002bb286db18f", "bottle lid-8e2b8fecd7a21841", "glass2-3715a082d26c02b4" ], "reasoning": "1. I watched the video and counted each item the user opened.\n2. I saw the user open the first item at 00:11, which was a dishwasher.\n3. I saw the user open the second item at 00:21, which was a cabinet.\n4. I saw the user open the third item at 00:36, which was a bag of chips (bag of crisps-6ba2599c74d27ce7).\n5. Therefore, the third item the user opened was a bag of chips (bag of crisps-6ba2599c74d27ce7).", "referenced_object_masks": [ "bag of crisps-6ba2599c74d27ce7" ] }, { "key": "P03-20240216-223126:175d2e7e7457e9b70ce4aa4f99962897b40dfe73", "video_id": "P03-20240216-223126", "question": "Where did I pour the bag of chips into the bowl?", "answer_choice_0": "By the refrigerator", "answer_choice_1": "By the sink", "answer_choice_2": "By the table", "answer_choice_3": "By the dishwasher", "answer_choice_4": "By the stove", "answer_id": 2, "answer": "By the table", "question_type": [ "Object Recognition", "Action Recognition", "Temporal Reasoning", "Situational Awareness", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "bag of crisps-6ba2599c74d27ce7", "bowl-0d6cc7c42afdcbab", "clip-6fad3ed0f6869c34", "bottle of beer-2701bc1262645e00", "other bottle-adec558f224af3f4", "ginger ale pack-0c982fe6cdc8db1c", "lid-bc870763b99ea600", "ginger ale bottle-96b9be650939e6d6", "glass-6d7989bad9d5466a", "bottle opener-fec002bb286db18f", "bottle lid-8e2b8fecd7a21841", "glass2-3715a082d26c02b4" ], "reasoning": "1. I watched the video and found the bag of chips (bag of crisps-6ba2599c74d27ce7) at 00:18.\n2. I found the bowl (bowl-0d6cc7c42afdcbab) at 00:27.\n3. I saw the user open and pour the bag of chips (bag of crisps-6ba2599c74d27ce7) into the bowl (bowl-0d6cc7c42afdcbab) at 00:36-00:46.\n4. At 00:36-00:46, I saw the user was standing by the table as they poured the bag of chips (bag of crisps-6ba2599c74d27ce7) into the bowl (bowl-0d6cc7c42afdcbab).\n5. Therefore, the user poured the bag of chips (bag of crisps-6ba2599c74d27ce7) into the bowl (bowl-0d6cc7c42afdcbab) by the table .", "referenced_object_masks": [ "bowl-0d6cc7c42afdcbab", "bag of crisps-6ba2599c74d27ce7" ] }, { "key": "P03-20240216-223126:36d2cb82a7de3f76a4be3f0c53077b5ef05f63cf", "video_id": "P03-20240216-223126", "question": "What was the last object I used in the kitchen?", "answer_choice_0": "The dishwasher", "answer_choice_1": "The microwave", "answer_choice_2": "The bottle opener", "answer_choice_3": "The toaster", "answer_choice_4": "The sink", "answer_id": 2, "answer": "The bottle opener", "question_type": [ "Situational Awareness", "Object Recognition", "Action Recognition", "Temporal Reasoning", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "bag of crisps-6ba2599c74d27ce7", "bowl-0d6cc7c42afdcbab", "clip-6fad3ed0f6869c34", "bottle of beer-2701bc1262645e00", "other bottle-adec558f224af3f4", "ginger ale pack-0c982fe6cdc8db1c", "lid-bc870763b99ea600", "ginger ale bottle-96b9be650939e6d6", "glass-6d7989bad9d5466a", "bottle opener-fec002bb286db18f", "bottle lid-8e2b8fecd7a21841", "glass2-3715a082d26c02b4" ], "reasoning": "1. I watched the video and found the kitchen at 00:03.\n2. At 02:09, I saw the user open a bottle (bottle of beer-2701bc1262645e00) by using a bottle opener (bottle opener-fec002bb286db18f) in the kitchen.\n3. I saw the user did not use anything else in the kitchen after 02:09.\n4. Therefore, the last object the user used in the kitchen was the bottle opener (bottle opener-fec002bb286db18f).", "referenced_object_masks": [ "bottle opener-fec002bb286db18f", "bottle of beer-2701bc1262645e00" ] }, { "key": "P03-20240216-223126:7620afbcc69ed556169b11d44584c0d1a7b3f1fb", "video_id": "P03-20240216-223126", "question": "If I doubled the number of bottles I opened, how many bottles would I have opened?", "answer_choice_0": "2", "answer_choice_1": "4", "answer_choice_2": "6", "answer_choice_3": "0", "answer_choice_4": "8", "answer_id": 1, "answer": "4", "question_type": [ "Counting", "Action Recognition", "Object Recognition", "State Changes", "Counterfactual", "Numerical Reasoning" ], "objects_of_interest": [ "bag of crisps-6ba2599c74d27ce7", "bowl-0d6cc7c42afdcbab", "clip-6fad3ed0f6869c34", "bottle of beer-2701bc1262645e00", "other bottle-adec558f224af3f4", "ginger ale pack-0c982fe6cdc8db1c", "lid-bc870763b99ea600", "ginger ale bottle-96b9be650939e6d6", "glass-6d7989bad9d5466a", "bottle opener-fec002bb286db18f", "bottle lid-8e2b8fecd7a21841", "glass2-3715a082d26c02b4" ], "reasoning": "1. I watched the video and counted each time the user opened a bottle.\n2. At 01:33, I saw the user open the first bottle, which was a bottle of ginger ale (ginger ale bottle-96b9be650939e6d6), by removing its lid (lid-bc870763b99ea600) with their hand.\n3. At 02:09, I saw the user open the second bottle (bottle of beer-2701bc1262645e00) by using a bottle opener (bottle opener-fec002bb286db18f).\n4. Since the user did not open another bottle after 02:09, I counted a total of 2 bottles they opened.\n5. I calculated 2*2=4.\n6. Therefore, if the user doubled the number of bottles they opened, they would have opened 4 bottles.", "referenced_object_masks": [ "ginger ale bottle-96b9be650939e6d6", "bottle opener-fec002bb286db18f", "bottle of beer-2701bc1262645e00", "lid-bc870763b99ea600" ] }, { "key": "P03-20240216-223126:ae10ea148321ac4eb3b9c5107860ce0992739b32", "video_id": "P03-20240216-223126", "question": "What did the label of the second bottle I took out of the refrigerator read?", "answer_choice_0": "Kettle", "answer_choice_1": "50", "answer_choice_2": "Otter Ale", "answer_choice_3": "Bundaberg", "answer_choice_4": "Old Peculier", "answer_id": 4, "answer": "Old Peculier", "question_type": [ "Situational Awareness", "Counting", "Action Recognition", "Object Recognition", "Reading" ], "objects_of_interest": [ "bag of crisps-6ba2599c74d27ce7", "bowl-0d6cc7c42afdcbab", "clip-6fad3ed0f6869c34", "bottle of beer-2701bc1262645e00", "other bottle-adec558f224af3f4", "ginger ale pack-0c982fe6cdc8db1c", "lid-bc870763b99ea600", "ginger ale bottle-96b9be650939e6d6", "glass-6d7989bad9d5466a", "bottle opener-fec002bb286db18f", "bottle lid-8e2b8fecd7a21841", "glass2-3715a082d26c02b4" ], "reasoning": "1. I watched the video and found the refrigerator at 00:03.\n2. I counted each bottle the user took out of the refrigerator.\n3. At 01:03, I saw the user take out the first bottle (bottle of beer-2701bc1262645e00) from the refrigerator.\n4. At 01:11, I saw the user take out the second bottle (other bottle-adec558f224af3f4) from the refrigerator.\n5. At 01:12, I saw the label of the second bottle (other bottle-adec558f224af3f4), which read \"Old Peculier\".\n6. Therefore, the label of the second bottle (other bottle-adec558f224af3f4) the user took out of the refrigerator read \"Old Peculier\".", "referenced_object_masks": [ "other bottle-adec558f224af3f4", "bottle of beer-2701bc1262645e00" ] }, { "key": "P03-20240217-091657:1fd13c47cb359c3e68133c0527bb261fb8409ecf", "video_id": "P03-20240217-091657", "question": "What do I put in the fourth appliance that I touch after I move the kettle nearer to the coffee apparatus?", "answer_choice_0": "Water", "answer_choice_1": "Bread", "answer_choice_2": "A jam jar", "answer_choice_3": "A baking sheet", "answer_choice_4": "A frying pan", "answer_id": 1, "answer": "Bread", "question_type": [ "Counting", "Action Recognition", "State Changes", "Situational Awareness" ], "objects_of_interest": [ "frying pan-405d160840634cfb", "frying pan2-2dd3a16b4b6f4e1b", "padding-88c981374c3c6bf5", "saucepan-0fc0658024be4dfd", "saucepan2-efa44d5019a5781a", "saucepan3-2fda0c1414ae81ba", "padding2-9e9bcb9f2d3062bd", "white frying pan-15c94665fd4a5e7a", "chopping board-205a5f1b33f64956", "coffee jug-985ad0496fabbc78", "white cloth-cc933f3f9822f73d", "plate-2d28f9df21d602c3", "plate2-189fd71224e591c5", "plates-a94b17b7a8c120c0", "another small red plate-c6bfcd14d2b0b462", "v60 metal filter-fd679d53ccf83254", "baking tray-856f86297fda4392", "small baking tray-8030d4be9ffd5084", "fork1-268f341ab52f0e88", "fork2-ddce3c99203dbfaf", "knife1-060fc90b72e47802", "knife2-cc0ff36db5858793", "fork3-2c98452c8898c078", "knife3-82074987422e9d6b", "big knife-583db68069e332d0", "spoon-471659376f65a407", "fork4-efc8e0c8b8dbc6d2", "chopstick-bc81a60b24d9d1e9", "peeler-fe0b5688b459b032", "wooden spoon-d51cea6533c8e6c5", "bread knife-4908c577cf432b1e", "wooden spatula-67395dfddf4dd29c", "dough scraper-31023b0adaa452b4", "grater-475f2f7e332df2f9", "glass lid-8ddb1afb32445c5d", "large bowl-2693948f4567dde6", "metal colander-07d298c7b4f5c37c", "plastic bowl-a7256e64c80aea80", "plastic colander-7edbcbcd9c04ae37", "stacked items-6689b07674c82a26", "food box-f366e03bd08b9635", "food box lid-e2886201e097c77d", "white bowl-35447f7477060411", "two small bowls-bb62b309d010184c", "small bowl2-b491a788046e1c3a", "green cup-e4699835a8979c79", "chequered cup-886f200378361a73", "green cup2-a3d3ef6eccdbb780", "plastic spatula-0eae4fe2df0b0cab", "ground coffee bag-d461d3d07c9490e9", "coffee beans bag-4dd751e0afe63ea5", "kettle-6a4b06b02bb29e59", "black coffee spoon-324dbc0c33490cf2", "bowl2-02fe110ea8ba4ecf", "dishwasher brush-9c1ac46b8e50edcc", "yellow glass-3f63e880d86b7a4c", "blue glass-5ec3f6e2b7d1381f", "food packaging-cffd15e27ddbefa7", "jar of jam-56c37c4dbf973793", "plate3-c1ee5a65bff5ad66", "small grey plate-5648852b662c6f25", "fork5-c391ec68b04eabbc", "plastic water bottle-e2489240827335d7", "one potato-ff2228c2f8d469d4", "potato2-606b68750c45c05a", "tea towel-2b68019ed5a8e2cd", "tea towel2-5735fd4844e78f5b", "butter-4a2e3ecd281b992b", "toaster-e5a4b4f8519092b5", "small cup-a859fc75321a3d37", "bread bag-e7c2484062a73444", "slice of bread-52efeb2d85d122c6", "second slice of bread-ba0cd408c7feb556", "food bag-4c2167da55e0d918", "bread2-3dae0b01849b1ba5", "bread knife2-c97eb826bec64124", "cloth2-5e88508a87976d8d", "apron-0668b9eba416c2f5", "teaspoon-0db2e5f59e2f180a", "knife4-7952054c9aff22d1" ], "reasoning": "1. I watched for the user to move the red kettle (kettle-6a4b06b02bb29e59) closer to the coffee apparatus .\n2. The user moves the red kettle (kettle-6a4b06b02bb29e59) to the other side of the sink at 08:20.\n3. After this, the user first touches the microwave at 08:23.\n4. The user then touches the refrigerator at 09:09.\n5. The user then touches the dishwasher at 09:36.\n6. The user then retrieves a toaster (toaster-e5a4b4f8519092b5) at 09:52. \n7. At 10:53, the user puts a slice of bread (slice of bread-52efeb2d85d122c6) into the toaster (toaster-e5a4b4f8519092b5).", "referenced_object_masks": [ "kettle-6a4b06b02bb29e59", "slice of bread-52efeb2d85d122c6", "toaster-e5a4b4f8519092b5" ] }, { "key": "P03-20240217-091657:3f6edc791a41fa8d95f2da9ae9b5be2f35a6e61c", "video_id": "P03-20240217-091657", "question": "If I had moved the jam jar from the first place I put it to the cabinet where I put away the first frying pan, how would I have gotten there?", "answer_choice_0": "Turned 180 degrees", "answer_choice_1": "No movement needed", "answer_choice_2": "Turned to the left, walked forward, turned right", "answer_choice_3": "Turned right, walked forward", "answer_choice_4": "Stooped down", "answer_id": 2, "answer": "Turned to the left, walked forward, turned right", "question_type": [ "Action Recognition", "Object Recognition", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "frying pan-405d160840634cfb", "frying pan2-2dd3a16b4b6f4e1b", "padding-88c981374c3c6bf5", "saucepan-0fc0658024be4dfd", "saucepan2-efa44d5019a5781a", "saucepan3-2fda0c1414ae81ba", "padding2-9e9bcb9f2d3062bd", "white frying pan-15c94665fd4a5e7a", "chopping board-205a5f1b33f64956", "coffee jug-985ad0496fabbc78", "white cloth-cc933f3f9822f73d", "plate-2d28f9df21d602c3", "plate2-189fd71224e591c5", "plates-a94b17b7a8c120c0", "another small red plate-c6bfcd14d2b0b462", "v60 metal filter-fd679d53ccf83254", "baking tray-856f86297fda4392", "small baking tray-8030d4be9ffd5084", "fork1-268f341ab52f0e88", "fork2-ddce3c99203dbfaf", "knife1-060fc90b72e47802", "knife2-cc0ff36db5858793", "fork3-2c98452c8898c078", "knife3-82074987422e9d6b", "big knife-583db68069e332d0", "spoon-471659376f65a407", "fork4-efc8e0c8b8dbc6d2", "chopstick-bc81a60b24d9d1e9", "peeler-fe0b5688b459b032", "wooden spoon-d51cea6533c8e6c5", "bread knife-4908c577cf432b1e", "wooden spatula-67395dfddf4dd29c", "dough scraper-31023b0adaa452b4", "grater-475f2f7e332df2f9", "glass lid-8ddb1afb32445c5d", "large bowl-2693948f4567dde6", "metal colander-07d298c7b4f5c37c", "plastic bowl-a7256e64c80aea80", "plastic colander-7edbcbcd9c04ae37", "stacked items-6689b07674c82a26", "food box-f366e03bd08b9635", "food box lid-e2886201e097c77d", "white bowl-35447f7477060411", "two small bowls-bb62b309d010184c", "small bowl2-b491a788046e1c3a", "green cup-e4699835a8979c79", "chequered cup-886f200378361a73", "green cup2-a3d3ef6eccdbb780", "plastic spatula-0eae4fe2df0b0cab", "ground coffee bag-d461d3d07c9490e9", "coffee beans bag-4dd751e0afe63ea5", "kettle-6a4b06b02bb29e59", "black coffee spoon-324dbc0c33490cf2", "bowl2-02fe110ea8ba4ecf", "dishwasher brush-9c1ac46b8e50edcc", "yellow glass-3f63e880d86b7a4c", "blue glass-5ec3f6e2b7d1381f", "food packaging-cffd15e27ddbefa7", "jar of jam-56c37c4dbf973793", "plate3-c1ee5a65bff5ad66", "small grey plate-5648852b662c6f25", "fork5-c391ec68b04eabbc", "plastic water bottle-e2489240827335d7", "one potato-ff2228c2f8d469d4", "potato2-606b68750c45c05a", "tea towel-2b68019ed5a8e2cd", "tea towel2-5735fd4844e78f5b", "butter-4a2e3ecd281b992b", "toaster-e5a4b4f8519092b5", "small cup-a859fc75321a3d37", "bread bag-e7c2484062a73444", "slice of bread-52efeb2d85d122c6", "second slice of bread-ba0cd408c7feb556", "food bag-4c2167da55e0d918", "bread2-3dae0b01849b1ba5", "bread knife2-c97eb826bec64124", "cloth2-5e88508a87976d8d", "apron-0668b9eba416c2f5", "teaspoon-0db2e5f59e2f180a", "knife4-7952054c9aff22d1" ], "reasoning": "1. I watched for the user to pick up the jam jar (jar of jam-56c37c4dbf973793).\n2. The user takes the jam jar (jar of jam-56c37c4dbf973793) out of the fridge and puts it on the table from 06:26-06:27.\n3. I rewatched the video to see where the user put away the first frying pan (frying pan-405d160840634cfb), which the user removes from the dishwasher at 00:13 and puts it in a cabinet at 00:38.\n4. To reach this cabinet from the table with the jam har (jar of jam-56c37c4dbf973793), the user would need to turn left, walk forward, and turn right.", "referenced_object_masks": [ "frying pan-405d160840634cfb", "jar of jam-56c37c4dbf973793" ] }, { "key": "P03-20240217-091657:40d83f534dc1c61927ab41fd1c1750a6469a3156", "video_id": "P03-20240217-091657", "question": "How many items do I take out of the dishwasher in the first four minutes, if I were counting any items that fit together as one?", "answer_choice_0": "24", "answer_choice_1": "12", "answer_choice_2": "18", "answer_choice_3": "36", "answer_choice_4": "30", "answer_id": 0, "answer": "24", "question_type": [ "Action Recognition", "Counting", "Temporal Reasoning", "Numerical Reasoning", "Object Recognition" ], "objects_of_interest": [ "frying pan-405d160840634cfb", "frying pan2-2dd3a16b4b6f4e1b", "padding-88c981374c3c6bf5", "saucepan-0fc0658024be4dfd", "saucepan2-efa44d5019a5781a", "saucepan3-2fda0c1414ae81ba", "padding2-9e9bcb9f2d3062bd", "white frying pan-15c94665fd4a5e7a", "chopping board-205a5f1b33f64956", "coffee jug-985ad0496fabbc78", "white cloth-cc933f3f9822f73d", "plate-2d28f9df21d602c3", "plate2-189fd71224e591c5", "plates-a94b17b7a8c120c0", "another small red plate-c6bfcd14d2b0b462", "v60 metal filter-fd679d53ccf83254", "baking tray-856f86297fda4392", "small baking tray-8030d4be9ffd5084", "fork1-268f341ab52f0e88", "fork2-ddce3c99203dbfaf", "knife1-060fc90b72e47802", "knife2-cc0ff36db5858793", "fork3-2c98452c8898c078", "knife3-82074987422e9d6b", "big knife-583db68069e332d0", "spoon-471659376f65a407", "fork4-efc8e0c8b8dbc6d2", "chopstick-bc81a60b24d9d1e9", "peeler-fe0b5688b459b032", "wooden spoon-d51cea6533c8e6c5", "bread knife-4908c577cf432b1e", "wooden spatula-67395dfddf4dd29c", "dough scraper-31023b0adaa452b4", "grater-475f2f7e332df2f9", "glass lid-8ddb1afb32445c5d", "large bowl-2693948f4567dde6", "metal colander-07d298c7b4f5c37c", "plastic bowl-a7256e64c80aea80", "plastic colander-7edbcbcd9c04ae37", "stacked items-6689b07674c82a26", "food box-f366e03bd08b9635", "food box lid-e2886201e097c77d", "white bowl-35447f7477060411", "two small bowls-bb62b309d010184c", "small bowl2-b491a788046e1c3a", "green cup-e4699835a8979c79", "chequered cup-886f200378361a73", "green cup2-a3d3ef6eccdbb780", "plastic spatula-0eae4fe2df0b0cab", "ground coffee bag-d461d3d07c9490e9", "coffee beans bag-4dd751e0afe63ea5", "kettle-6a4b06b02bb29e59", "black coffee spoon-324dbc0c33490cf2", "bowl2-02fe110ea8ba4ecf", "dishwasher brush-9c1ac46b8e50edcc", "yellow glass-3f63e880d86b7a4c", "blue glass-5ec3f6e2b7d1381f", "food packaging-cffd15e27ddbefa7", "jar of jam-56c37c4dbf973793", "plate3-c1ee5a65bff5ad66", "small grey plate-5648852b662c6f25", "fork5-c391ec68b04eabbc", "plastic water bottle-e2489240827335d7", "one potato-ff2228c2f8d469d4", "potato2-606b68750c45c05a", "tea towel-2b68019ed5a8e2cd", "tea towel2-5735fd4844e78f5b", "butter-4a2e3ecd281b992b", "toaster-e5a4b4f8519092b5", "small cup-a859fc75321a3d37", "bread bag-e7c2484062a73444", "slice of bread-52efeb2d85d122c6", "second slice of bread-ba0cd408c7feb556", "food bag-4c2167da55e0d918", "bread2-3dae0b01849b1ba5", "bread knife2-c97eb826bec64124", "cloth2-5e88508a87976d8d", "apron-0668b9eba416c2f5", "teaspoon-0db2e5f59e2f180a", "knife4-7952054c9aff22d1" ], "reasoning": "1. I watched the video looking for the user to take items out of the dishwasher.\n2. The first item, a frying pan (frying pan-405d160840634cfb) is removed at 00:12, and the last item to be removed in the first 4 minutes, a green cup (green cup-e4699835a8979c79) is removed at 03:59.\n3. In between 00:12 and 03:59, a total of 26 separate items are removed from the dishwasher by the user.\n4. At 02:53, the user removes two parts of a salad spinner (plastic colander-7edbcbcd9c04ae37).\n5. Between 03:32-03:43, the user removes a plastic container (food box-f366e03bd08b9635) and its lid (food box lid-e2886201e097c77d).\n6. If each of these pairs counted as one, this would make 24 separate items.", "referenced_object_masks": [ "food box lid-e2886201e097c77d", "plastic colander-7edbcbcd9c04ae37", "food box-f366e03bd08b9635", "frying pan-405d160840634cfb", "green cup-e4699835a8979c79" ] }, { "key": "P03-20240217-091657:448b61896e9dc4249c414b98bad731c4c56f0d3c", "video_id": "P03-20240217-091657", "question": "What is on the far right of the same shelf I place the last thing I return to the refrigerator?", "answer_choice_0": "Eggs", "answer_choice_1": "Quiche", "answer_choice_2": "Jam", "answer_choice_3": "Cheese", "answer_choice_4": "Ginger beer", "answer_id": 4, "answer": "Ginger beer", "question_type": [ "Egocentric & Spatial Perception", "Situational Awareness", "Object Recognition" ], "objects_of_interest": [ "frying pan-405d160840634cfb", "frying pan2-2dd3a16b4b6f4e1b", "padding-88c981374c3c6bf5", "saucepan-0fc0658024be4dfd", "saucepan2-efa44d5019a5781a", "saucepan3-2fda0c1414ae81ba", "padding2-9e9bcb9f2d3062bd", "white frying pan-15c94665fd4a5e7a", "chopping board-205a5f1b33f64956", "coffee jug-985ad0496fabbc78", "white cloth-cc933f3f9822f73d", "plate-2d28f9df21d602c3", "plate2-189fd71224e591c5", "plates-a94b17b7a8c120c0", "another small red plate-c6bfcd14d2b0b462", "v60 metal filter-fd679d53ccf83254", "baking tray-856f86297fda4392", "small baking tray-8030d4be9ffd5084", "fork1-268f341ab52f0e88", "fork2-ddce3c99203dbfaf", "knife1-060fc90b72e47802", "knife2-cc0ff36db5858793", "fork3-2c98452c8898c078", "knife3-82074987422e9d6b", "big knife-583db68069e332d0", "spoon-471659376f65a407", "fork4-efc8e0c8b8dbc6d2", "chopstick-bc81a60b24d9d1e9", "peeler-fe0b5688b459b032", "wooden spoon-d51cea6533c8e6c5", "bread knife-4908c577cf432b1e", "wooden spatula-67395dfddf4dd29c", "dough scraper-31023b0adaa452b4", "grater-475f2f7e332df2f9", "glass lid-8ddb1afb32445c5d", "large bowl-2693948f4567dde6", "metal colander-07d298c7b4f5c37c", "plastic bowl-a7256e64c80aea80", "plastic colander-7edbcbcd9c04ae37", "stacked items-6689b07674c82a26", "food box-f366e03bd08b9635", "food box lid-e2886201e097c77d", "white bowl-35447f7477060411", "two small bowls-bb62b309d010184c", "small bowl2-b491a788046e1c3a", "green cup-e4699835a8979c79", "chequered cup-886f200378361a73", "green cup2-a3d3ef6eccdbb780", "plastic spatula-0eae4fe2df0b0cab", "ground coffee bag-d461d3d07c9490e9", "coffee beans bag-4dd751e0afe63ea5", "kettle-6a4b06b02bb29e59", "black coffee spoon-324dbc0c33490cf2", "bowl2-02fe110ea8ba4ecf", "dishwasher brush-9c1ac46b8e50edcc", "yellow glass-3f63e880d86b7a4c", "blue glass-5ec3f6e2b7d1381f", "food packaging-cffd15e27ddbefa7", "jar of jam-56c37c4dbf973793", "plate3-c1ee5a65bff5ad66", "small grey plate-5648852b662c6f25", "fork5-c391ec68b04eabbc", "plastic water bottle-e2489240827335d7", "one potato-ff2228c2f8d469d4", "potato2-606b68750c45c05a", "tea towel-2b68019ed5a8e2cd", "tea towel2-5735fd4844e78f5b", "butter-4a2e3ecd281b992b", "toaster-e5a4b4f8519092b5", "small cup-a859fc75321a3d37", "bread bag-e7c2484062a73444", "slice of bread-52efeb2d85d122c6", "second slice of bread-ba0cd408c7feb556", "food bag-4c2167da55e0d918", "bread2-3dae0b01849b1ba5", "bread knife2-c97eb826bec64124", "cloth2-5e88508a87976d8d", "apron-0668b9eba416c2f5", "teaspoon-0db2e5f59e2f180a", "knife4-7952054c9aff22d1" ], "reasoning": "1. I watched for when the user put things back in the fridge. I noted the last time they opened the fridge was at 11:20.\n2. I saw the user return a bag of bread (bread bag-e7c2484062a73444) to the left side of the top shelf at 11:21. There were no other items added after this point.\n3. I looked to the far right of the top shelf and saw a pack of ginger beer .", "referenced_object_masks": [ "bread bag-e7c2484062a73444" ] }, { "key": "P03-20240217-091657:7d2756dd34ad88e929ff3bf016254a07af1bd8e1", "video_id": "P03-20240217-091657", "question": "If there had been twice as much quiche to begin with, how much would remain after I cut a slice?", "answer_choice_0": "7/8", "answer_choice_1": "2/3", "answer_choice_2": "5/8", "answer_choice_3": "3/4", "answer_choice_4": "1/2", "answer_id": 0, "answer": "7/8", "question_type": [ "Counterfactual", "Numerical Reasoning", "Object Recognition" ], "objects_of_interest": [ "frying pan-405d160840634cfb", "frying pan2-2dd3a16b4b6f4e1b", "padding-88c981374c3c6bf5", "saucepan-0fc0658024be4dfd", "saucepan2-efa44d5019a5781a", "saucepan3-2fda0c1414ae81ba", "padding2-9e9bcb9f2d3062bd", "white frying pan-15c94665fd4a5e7a", "chopping board-205a5f1b33f64956", "coffee jug-985ad0496fabbc78", "white cloth-cc933f3f9822f73d", "plate-2d28f9df21d602c3", "plate2-189fd71224e591c5", "plates-a94b17b7a8c120c0", "another small red plate-c6bfcd14d2b0b462", "v60 metal filter-fd679d53ccf83254", "baking tray-856f86297fda4392", "small baking tray-8030d4be9ffd5084", "fork1-268f341ab52f0e88", "fork2-ddce3c99203dbfaf", "knife1-060fc90b72e47802", "knife2-cc0ff36db5858793", "fork3-2c98452c8898c078", "knife3-82074987422e9d6b", "big knife-583db68069e332d0", "spoon-471659376f65a407", "fork4-efc8e0c8b8dbc6d2", "chopstick-bc81a60b24d9d1e9", "peeler-fe0b5688b459b032", "wooden spoon-d51cea6533c8e6c5", "bread knife-4908c577cf432b1e", "wooden spatula-67395dfddf4dd29c", "dough scraper-31023b0adaa452b4", "grater-475f2f7e332df2f9", "glass lid-8ddb1afb32445c5d", "large bowl-2693948f4567dde6", "metal colander-07d298c7b4f5c37c", "plastic bowl-a7256e64c80aea80", "plastic colander-7edbcbcd9c04ae37", "stacked items-6689b07674c82a26", "food box-f366e03bd08b9635", "food box lid-e2886201e097c77d", "white bowl-35447f7477060411", "two small bowls-bb62b309d010184c", "small bowl2-b491a788046e1c3a", "green cup-e4699835a8979c79", "chequered cup-886f200378361a73", "green cup2-a3d3ef6eccdbb780", "plastic spatula-0eae4fe2df0b0cab", "ground coffee bag-d461d3d07c9490e9", "coffee beans bag-4dd751e0afe63ea5", "kettle-6a4b06b02bb29e59", "black coffee spoon-324dbc0c33490cf2", "bowl2-02fe110ea8ba4ecf", "dishwasher brush-9c1ac46b8e50edcc", "yellow glass-3f63e880d86b7a4c", "blue glass-5ec3f6e2b7d1381f", "food packaging-cffd15e27ddbefa7", "jar of jam-56c37c4dbf973793", "plate3-c1ee5a65bff5ad66", "small grey plate-5648852b662c6f25", "fork5-c391ec68b04eabbc", "plastic water bottle-e2489240827335d7", "one potato-ff2228c2f8d469d4", "potato2-606b68750c45c05a", "tea towel-2b68019ed5a8e2cd", "tea towel2-5735fd4844e78f5b", "butter-4a2e3ecd281b992b", "toaster-e5a4b4f8519092b5", "small cup-a859fc75321a3d37", "bread bag-e7c2484062a73444", "slice of bread-52efeb2d85d122c6", "second slice of bread-ba0cd408c7feb556", "food bag-4c2167da55e0d918", "bread2-3dae0b01849b1ba5", "bread knife2-c97eb826bec64124", "cloth2-5e88508a87976d8d", "apron-0668b9eba416c2f5", "teaspoon-0db2e5f59e2f180a", "knife4-7952054c9aff22d1" ], "reasoning": "1. I saw the user set down a plate (small grey plate-5648852b662c6f25) with half of a quiche on it at 06:38.\n2. I saw the user cut a slice sized roughly 1/4 of the remaining quiche on the plate at 06:55.\n3. If there had been twice as much quiche to begin with, the slice would be equivalent to 1/8.\n4. Thus, there would be 7/8 of the quiche remaining.", "referenced_object_masks": [ "small grey plate-5648852b662c6f25" ] }, { "key": "P03-20240217-091657:957119170653f4cdcbadd8fce158dbdedc0b1663", "video_id": "P03-20240217-091657", "question": "What color handle did the seventh utensil I removed from the dishwasher have?", "answer_choice_0": "Black", "answer_choice_1": "Red", "answer_choice_2": "Green", "answer_choice_3": "Blue", "answer_choice_4": "Silver", "answer_id": 2, "answer": "Green", "question_type": [ "Action Recognition", "Counting", "Egocentric & Spatial Perception", "Object Recognition" ], "objects_of_interest": [ "frying pan-405d160840634cfb", "frying pan2-2dd3a16b4b6f4e1b", "padding-88c981374c3c6bf5", "saucepan-0fc0658024be4dfd", "saucepan2-efa44d5019a5781a", "saucepan3-2fda0c1414ae81ba", "padding2-9e9bcb9f2d3062bd", "white frying pan-15c94665fd4a5e7a", "chopping board-205a5f1b33f64956", "coffee jug-985ad0496fabbc78", "white cloth-cc933f3f9822f73d", "plate-2d28f9df21d602c3", "plate2-189fd71224e591c5", "plates-a94b17b7a8c120c0", "another small red plate-c6bfcd14d2b0b462", "v60 metal filter-fd679d53ccf83254", "baking tray-856f86297fda4392", "small baking tray-8030d4be9ffd5084", "fork1-268f341ab52f0e88", "fork2-ddce3c99203dbfaf", "knife1-060fc90b72e47802", "knife2-cc0ff36db5858793", "fork3-2c98452c8898c078", "knife3-82074987422e9d6b", "big knife-583db68069e332d0", "spoon-471659376f65a407", "fork4-efc8e0c8b8dbc6d2", "chopstick-bc81a60b24d9d1e9", "peeler-fe0b5688b459b032", "wooden spoon-d51cea6533c8e6c5", "bread knife-4908c577cf432b1e", "wooden spatula-67395dfddf4dd29c", "dough scraper-31023b0adaa452b4", "grater-475f2f7e332df2f9", "glass lid-8ddb1afb32445c5d", "large bowl-2693948f4567dde6", "metal colander-07d298c7b4f5c37c", "plastic bowl-a7256e64c80aea80", "plastic colander-7edbcbcd9c04ae37", "stacked items-6689b07674c82a26", "food box-f366e03bd08b9635", "food box lid-e2886201e097c77d", "white bowl-35447f7477060411", "two small bowls-bb62b309d010184c", "small bowl2-b491a788046e1c3a", "green cup-e4699835a8979c79", "chequered cup-886f200378361a73", "green cup2-a3d3ef6eccdbb780", "plastic spatula-0eae4fe2df0b0cab", "ground coffee bag-d461d3d07c9490e9", "coffee beans bag-4dd751e0afe63ea5", "kettle-6a4b06b02bb29e59", "black coffee spoon-324dbc0c33490cf2", "bowl2-02fe110ea8ba4ecf", "dishwasher brush-9c1ac46b8e50edcc", "yellow glass-3f63e880d86b7a4c", "blue glass-5ec3f6e2b7d1381f", "food packaging-cffd15e27ddbefa7", "jar of jam-56c37c4dbf973793", "plate3-c1ee5a65bff5ad66", "small grey plate-5648852b662c6f25", "fork5-c391ec68b04eabbc", "plastic water bottle-e2489240827335d7", "one potato-ff2228c2f8d469d4", "potato2-606b68750c45c05a", "tea towel-2b68019ed5a8e2cd", "tea towel2-5735fd4844e78f5b", "butter-4a2e3ecd281b992b", "toaster-e5a4b4f8519092b5", "small cup-a859fc75321a3d37", "bread bag-e7c2484062a73444", "slice of bread-52efeb2d85d122c6", "second slice of bread-ba0cd408c7feb556", "food bag-4c2167da55e0d918", "bread2-3dae0b01849b1ba5", "bread knife2-c97eb826bec64124", "cloth2-5e88508a87976d8d", "apron-0668b9eba416c2f5", "teaspoon-0db2e5f59e2f180a", "knife4-7952054c9aff22d1" ], "reasoning": "1. I watched for the user to remove utensils and not other objects from the dishwasher.\n2. I saw the user remove two forks at 01:32, a knife (knife1-060fc90b72e47802) at 01:34, another fork (knife2-cc0ff36db5858793) at 01:35, a fourth fork (fork3-2c98452c8898c078) at 01:36, and a second knife (knife3-82074987422e9d6b) at 01:38. This meant the next utensil removed would be the seventh.\n3. At 01:47, I saw the user remove the next utensil, a large knife with a green handle (big knife-583db68069e332d0).", "referenced_object_masks": [ "fork3-2c98452c8898c078", "big knife-583db68069e332d0", "knife1-060fc90b72e47802", "knife2-cc0ff36db5858793", "knife3-82074987422e9d6b" ] }, { "key": "P03-20240217-091657:9707bb79588c171b6b84c05476900bf59b3436fe", "video_id": "P03-20240217-091657", "question": "How many items did I remove from the top rack of the dishwasher before I removed the first mug?", "answer_choice_0": "10", "answer_choice_1": "8", "answer_choice_2": "11", "answer_choice_3": "9", "answer_choice_4": "12", "answer_id": 3, "answer": "9", "question_type": [ "Temporal Reasoning", "Counting", "Object Recognition", "Action Recognition", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "frying pan-405d160840634cfb", "frying pan2-2dd3a16b4b6f4e1b", "padding-88c981374c3c6bf5", "saucepan-0fc0658024be4dfd", "saucepan2-efa44d5019a5781a", "saucepan3-2fda0c1414ae81ba", "padding2-9e9bcb9f2d3062bd", "white frying pan-15c94665fd4a5e7a", "chopping board-205a5f1b33f64956", "coffee jug-985ad0496fabbc78", "white cloth-cc933f3f9822f73d", "plate-2d28f9df21d602c3", "plate2-189fd71224e591c5", "plates-a94b17b7a8c120c0", "another small red plate-c6bfcd14d2b0b462", "v60 metal filter-fd679d53ccf83254", "baking tray-856f86297fda4392", "small baking tray-8030d4be9ffd5084", "fork1-268f341ab52f0e88", "fork2-ddce3c99203dbfaf", "knife1-060fc90b72e47802", "knife2-cc0ff36db5858793", "fork3-2c98452c8898c078", "knife3-82074987422e9d6b", "big knife-583db68069e332d0", "spoon-471659376f65a407", "fork4-efc8e0c8b8dbc6d2", "chopstick-bc81a60b24d9d1e9", "peeler-fe0b5688b459b032", "wooden spoon-d51cea6533c8e6c5", "bread knife-4908c577cf432b1e", "wooden spatula-67395dfddf4dd29c", "dough scraper-31023b0adaa452b4", "grater-475f2f7e332df2f9", "glass lid-8ddb1afb32445c5d", "large bowl-2693948f4567dde6", "metal colander-07d298c7b4f5c37c", "plastic bowl-a7256e64c80aea80", "plastic colander-7edbcbcd9c04ae37", "stacked items-6689b07674c82a26", "food box-f366e03bd08b9635", "food box lid-e2886201e097c77d", "white bowl-35447f7477060411", "two small bowls-bb62b309d010184c", "small bowl2-b491a788046e1c3a", "green cup-e4699835a8979c79", "chequered cup-886f200378361a73", "green cup2-a3d3ef6eccdbb780", "plastic spatula-0eae4fe2df0b0cab", "ground coffee bag-d461d3d07c9490e9", "coffee beans bag-4dd751e0afe63ea5", "kettle-6a4b06b02bb29e59", "black coffee spoon-324dbc0c33490cf2", "bowl2-02fe110ea8ba4ecf", "dishwasher brush-9c1ac46b8e50edcc", "yellow glass-3f63e880d86b7a4c", "blue glass-5ec3f6e2b7d1381f", "food packaging-cffd15e27ddbefa7", "jar of jam-56c37c4dbf973793", "plate3-c1ee5a65bff5ad66", "small grey plate-5648852b662c6f25", "fork5-c391ec68b04eabbc", "plastic water bottle-e2489240827335d7", "one potato-ff2228c2f8d469d4", "potato2-606b68750c45c05a", "tea towel-2b68019ed5a8e2cd", "tea towel2-5735fd4844e78f5b", "butter-4a2e3ecd281b992b", "toaster-e5a4b4f8519092b5", "small cup-a859fc75321a3d37", "bread bag-e7c2484062a73444", "slice of bread-52efeb2d85d122c6", "second slice of bread-ba0cd408c7feb556", "food bag-4c2167da55e0d918", "bread2-3dae0b01849b1ba5", "bread knife2-c97eb826bec64124", "cloth2-5e88508a87976d8d", "apron-0668b9eba416c2f5", "teaspoon-0db2e5f59e2f180a", "knife4-7952054c9aff22d1" ], "reasoning": "1. I watched until I saw the user begin unloading the dishwasher at 00:11. I continued watching until they began unloading the top rack at 02:32.\n2. I saw the user remove a mug (green cup-e4699835a8979c79) from the top rack at 04:00, signifying that this is the end of the period I need to examine for items unloaded from the top rack.\n3. Between 02:32 and 04:00 I saw the user remove a grater (grater-475f2f7e332df2f9) at 02:32, a lid (glass lid-8ddb1afb32445c5d) at 02:38, a large bowl (large bowl-2693948f4567dde6) at 02:45, the outer part of a colander (plastic bowl-a7256e64c80aea80) and its inner part (plastic colander-7edbcbcd9c04ae37) at 02:52, a tupperware container (food box-f366e03bd08b9635) at 03:32, a tupperware lid (food box lid-e2886201e097c77d) at 03:42, a white bowl (white bowl-35447f7477060411) at 03:50, and a small bowl (small bowl2-b491a788046e1c3a) at 03:57 for a total of 9 items.", "referenced_object_masks": [ "white bowl-35447f7477060411", "small bowl2-b491a788046e1c3a", "plastic bowl-a7256e64c80aea80", "food box lid-e2886201e097c77d", "large bowl-2693948f4567dde6", "grater-475f2f7e332df2f9", "glass lid-8ddb1afb32445c5d", "plastic colander-7edbcbcd9c04ae37", "food box-f366e03bd08b9635", "green cup-e4699835a8979c79" ] }, { "key": "P03-20240217-091657:ae89e069068038c6d84de3fc2d4fc53657f06000", "video_id": "P03-20240217-091657", "question": "If I had brewed my coffee for twice as long, how long would I have steeped it?", "answer_choice_0": "7:16", "answer_choice_1": "6:58", "answer_choice_2": "9:06", "answer_choice_3": "8:32", "answer_choice_4": "4:16", "answer_id": 3, "answer": "8:32", "question_type": [ "Counterfactual", "Numerical Reasoning", "Action Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "frying pan-405d160840634cfb", "frying pan2-2dd3a16b4b6f4e1b", "padding-88c981374c3c6bf5", "saucepan-0fc0658024be4dfd", "saucepan2-efa44d5019a5781a", "saucepan3-2fda0c1414ae81ba", "padding2-9e9bcb9f2d3062bd", "white frying pan-15c94665fd4a5e7a", "chopping board-205a5f1b33f64956", "coffee jug-985ad0496fabbc78", "white cloth-cc933f3f9822f73d", "plate-2d28f9df21d602c3", "plate2-189fd71224e591c5", "plates-a94b17b7a8c120c0", "another small red plate-c6bfcd14d2b0b462", "v60 metal filter-fd679d53ccf83254", "baking tray-856f86297fda4392", "small baking tray-8030d4be9ffd5084", "fork1-268f341ab52f0e88", "fork2-ddce3c99203dbfaf", "knife1-060fc90b72e47802", "knife2-cc0ff36db5858793", "fork3-2c98452c8898c078", "knife3-82074987422e9d6b", "big knife-583db68069e332d0", "spoon-471659376f65a407", "fork4-efc8e0c8b8dbc6d2", "chopstick-bc81a60b24d9d1e9", "peeler-fe0b5688b459b032", "wooden spoon-d51cea6533c8e6c5", "bread knife-4908c577cf432b1e", "wooden spatula-67395dfddf4dd29c", "dough scraper-31023b0adaa452b4", "grater-475f2f7e332df2f9", "glass lid-8ddb1afb32445c5d", "large bowl-2693948f4567dde6", "metal colander-07d298c7b4f5c37c", "plastic bowl-a7256e64c80aea80", "plastic colander-7edbcbcd9c04ae37", "stacked items-6689b07674c82a26", "food box-f366e03bd08b9635", "food box lid-e2886201e097c77d", "white bowl-35447f7477060411", "two small bowls-bb62b309d010184c", "small bowl2-b491a788046e1c3a", "green cup-e4699835a8979c79", "chequered cup-886f200378361a73", "green cup2-a3d3ef6eccdbb780", "plastic spatula-0eae4fe2df0b0cab", "ground coffee bag-d461d3d07c9490e9", "coffee beans bag-4dd751e0afe63ea5", "kettle-6a4b06b02bb29e59", "black coffee spoon-324dbc0c33490cf2", "bowl2-02fe110ea8ba4ecf", "dishwasher brush-9c1ac46b8e50edcc", "yellow glass-3f63e880d86b7a4c", "blue glass-5ec3f6e2b7d1381f", "food packaging-cffd15e27ddbefa7", "jar of jam-56c37c4dbf973793", "plate3-c1ee5a65bff5ad66", "small grey plate-5648852b662c6f25", "fork5-c391ec68b04eabbc", "plastic water bottle-e2489240827335d7", "one potato-ff2228c2f8d469d4", "potato2-606b68750c45c05a", "tea towel-2b68019ed5a8e2cd", "tea towel2-5735fd4844e78f5b", "butter-4a2e3ecd281b992b", "toaster-e5a4b4f8519092b5", "small cup-a859fc75321a3d37", "bread bag-e7c2484062a73444", "slice of bread-52efeb2d85d122c6", "second slice of bread-ba0cd408c7feb556", "food bag-4c2167da55e0d918", "bread2-3dae0b01849b1ba5", "bread knife2-c97eb826bec64124", "cloth2-5e88508a87976d8d", "apron-0668b9eba416c2f5", "teaspoon-0db2e5f59e2f180a", "knife4-7952054c9aff22d1" ], "reasoning": "1. I watched the video for when the user brewed their coffee. I saw them first pour hot water from their kettle (kettle-6a4b06b02bb29e59) over their coffee at 08:52.\n2. The user continued periodically adding water to their coffee until they removed the cone filter (v60 metal filter-fd679d53ccf83254) at 13:08, indicating their coffee was finished brewing.\n3. I subtracted 8:52 from 13:08 to reach a total brew time of 4:16.\n4. I multiplied the brew time of 4:16 by 2 to reach a total of 8:32.", "referenced_object_masks": [ "v60 metal filter-fd679d53ccf83254", "kettle-6a4b06b02bb29e59" ] }, { "key": "P03-20240217-091657:d25508b1aa5cc876f58e0e4ae76f04246affbbf8", "video_id": "P03-20240217-091657", "question": "After the final time I pour water for coffee, how many seconds would I have spent standing and just looking at the coffee if I multiplied that number by the number of times I poured water from the kettle?", "answer_choice_0": "36", "answer_choice_1": "49", "answer_choice_2": "18", "answer_choice_3": "81", "answer_choice_4": "72", "answer_id": 3, "answer": "81", "question_type": [ "Egocentric & Spatial Perception", "Action Recognition", "Event Occurence", "Counting", "Temporal Reasoning" ], "objects_of_interest": [ "frying pan-405d160840634cfb", "frying pan2-2dd3a16b4b6f4e1b", "padding-88c981374c3c6bf5", "saucepan-0fc0658024be4dfd", "saucepan2-efa44d5019a5781a", "saucepan3-2fda0c1414ae81ba", "padding2-9e9bcb9f2d3062bd", "white frying pan-15c94665fd4a5e7a", "chopping board-205a5f1b33f64956", "coffee jug-985ad0496fabbc78", "white cloth-cc933f3f9822f73d", "plate-2d28f9df21d602c3", "plate2-189fd71224e591c5", "plates-a94b17b7a8c120c0", "another small red plate-c6bfcd14d2b0b462", "v60 metal filter-fd679d53ccf83254", "baking tray-856f86297fda4392", "small baking tray-8030d4be9ffd5084", "fork1-268f341ab52f0e88", "fork2-ddce3c99203dbfaf", "knife1-060fc90b72e47802", "knife2-cc0ff36db5858793", "fork3-2c98452c8898c078", "knife3-82074987422e9d6b", "big knife-583db68069e332d0", "spoon-471659376f65a407", "fork4-efc8e0c8b8dbc6d2", "chopstick-bc81a60b24d9d1e9", "peeler-fe0b5688b459b032", "wooden spoon-d51cea6533c8e6c5", "bread knife-4908c577cf432b1e", "wooden spatula-67395dfddf4dd29c", "dough scraper-31023b0adaa452b4", "grater-475f2f7e332df2f9", "glass lid-8ddb1afb32445c5d", "large bowl-2693948f4567dde6", "metal colander-07d298c7b4f5c37c", "plastic bowl-a7256e64c80aea80", "plastic colander-7edbcbcd9c04ae37", "stacked items-6689b07674c82a26", "food box-f366e03bd08b9635", "food box lid-e2886201e097c77d", "white bowl-35447f7477060411", "two small bowls-bb62b309d010184c", "small bowl2-b491a788046e1c3a", "green cup-e4699835a8979c79", "chequered cup-886f200378361a73", "green cup2-a3d3ef6eccdbb780", "plastic spatula-0eae4fe2df0b0cab", "ground coffee bag-d461d3d07c9490e9", "coffee beans bag-4dd751e0afe63ea5", "kettle-6a4b06b02bb29e59", "black coffee spoon-324dbc0c33490cf2", "bowl2-02fe110ea8ba4ecf", "dishwasher brush-9c1ac46b8e50edcc", "yellow glass-3f63e880d86b7a4c", "blue glass-5ec3f6e2b7d1381f", "food packaging-cffd15e27ddbefa7", "jar of jam-56c37c4dbf973793", "plate3-c1ee5a65bff5ad66", "small grey plate-5648852b662c6f25", "fork5-c391ec68b04eabbc", "plastic water bottle-e2489240827335d7", "one potato-ff2228c2f8d469d4", "potato2-606b68750c45c05a", "tea towel-2b68019ed5a8e2cd", "tea towel2-5735fd4844e78f5b", "butter-4a2e3ecd281b992b", "toaster-e5a4b4f8519092b5", "small cup-a859fc75321a3d37", "bread bag-e7c2484062a73444", "slice of bread-52efeb2d85d122c6", "second slice of bread-ba0cd408c7feb556", "food bag-4c2167da55e0d918", "bread2-3dae0b01849b1ba5", "bread knife2-c97eb826bec64124", "cloth2-5e88508a87976d8d", "apron-0668b9eba416c2f5", "teaspoon-0db2e5f59e2f180a", "knife4-7952054c9aff22d1" ], "reasoning": "1. I watched the video looking for the user to pour water from the kettle (kettle-6a4b06b02bb29e59).\n2. The user pours water from the kettle (kettle-6a4b06b02bb29e59) at 08:52, 09:19, 09:42, 10:07, 10:37, 11:09, 11:38, 12:07, and 12:37.\n3. After the final time the user pours coffee at 12:37, I watched for the user to stand and stare at the coffee apparatus, which occurs from 12:53-13:02.\n4. The number of times the water is poured, 9, times the number of seconds that the user stares at the coffee apparatus, 9, is 81.", "referenced_object_masks": [ "kettle-6a4b06b02bb29e59" ] }, { "key": "P03-20240217-091657:d566719e627b38eda4652e71718675a61062dfb4", "video_id": "P03-20240217-091657", "question": "When I cut the first slice of pizza, what kitchen implement is the knife I use pointing towards?", "answer_choice_0": "The dish drainer", "answer_choice_1": "The air fryer", "answer_choice_2": "The frying pan", "answer_choice_3": "The electric kettle", "answer_choice_4": "The water dispenser", "answer_id": 4, "answer": "The water dispenser", "question_type": [ "Egocentric & Spatial Perception", "Action Recognition", "Object Recognition" ], "objects_of_interest": [ "frying pan-405d160840634cfb", "frying pan2-2dd3a16b4b6f4e1b", "padding-88c981374c3c6bf5", "saucepan-0fc0658024be4dfd", "saucepan2-efa44d5019a5781a", "saucepan3-2fda0c1414ae81ba", "padding2-9e9bcb9f2d3062bd", "white frying pan-15c94665fd4a5e7a", "chopping board-205a5f1b33f64956", "coffee jug-985ad0496fabbc78", "white cloth-cc933f3f9822f73d", "plate-2d28f9df21d602c3", "plate2-189fd71224e591c5", "plates-a94b17b7a8c120c0", "another small red plate-c6bfcd14d2b0b462", "v60 metal filter-fd679d53ccf83254", "baking tray-856f86297fda4392", "small baking tray-8030d4be9ffd5084", "fork1-268f341ab52f0e88", "fork2-ddce3c99203dbfaf", "knife1-060fc90b72e47802", "knife2-cc0ff36db5858793", "fork3-2c98452c8898c078", "knife3-82074987422e9d6b", "big knife-583db68069e332d0", "spoon-471659376f65a407", "fork4-efc8e0c8b8dbc6d2", "chopstick-bc81a60b24d9d1e9", "peeler-fe0b5688b459b032", "wooden spoon-d51cea6533c8e6c5", "bread knife-4908c577cf432b1e", "wooden spatula-67395dfddf4dd29c", "dough scraper-31023b0adaa452b4", "grater-475f2f7e332df2f9", "glass lid-8ddb1afb32445c5d", "large bowl-2693948f4567dde6", "metal colander-07d298c7b4f5c37c", "plastic bowl-a7256e64c80aea80", "plastic colander-7edbcbcd9c04ae37", "stacked items-6689b07674c82a26", "food box-f366e03bd08b9635", "food box lid-e2886201e097c77d", "white bowl-35447f7477060411", "two small bowls-bb62b309d010184c", "small bowl2-b491a788046e1c3a", "green cup-e4699835a8979c79", "chequered cup-886f200378361a73", "green cup2-a3d3ef6eccdbb780", "plastic spatula-0eae4fe2df0b0cab", "ground coffee bag-d461d3d07c9490e9", "coffee beans bag-4dd751e0afe63ea5", "kettle-6a4b06b02bb29e59", "black coffee spoon-324dbc0c33490cf2", "bowl2-02fe110ea8ba4ecf", "dishwasher brush-9c1ac46b8e50edcc", "yellow glass-3f63e880d86b7a4c", "blue glass-5ec3f6e2b7d1381f", "food packaging-cffd15e27ddbefa7", "jar of jam-56c37c4dbf973793", "plate3-c1ee5a65bff5ad66", "small grey plate-5648852b662c6f25", "fork5-c391ec68b04eabbc", "plastic water bottle-e2489240827335d7", "one potato-ff2228c2f8d469d4", "potato2-606b68750c45c05a", "tea towel-2b68019ed5a8e2cd", "tea towel2-5735fd4844e78f5b", "butter-4a2e3ecd281b992b", "toaster-e5a4b4f8519092b5", "small cup-a859fc75321a3d37", "bread bag-e7c2484062a73444", "slice of bread-52efeb2d85d122c6", "second slice of bread-ba0cd408c7feb556", "food bag-4c2167da55e0d918", "bread2-3dae0b01849b1ba5", "bread knife2-c97eb826bec64124", "cloth2-5e88508a87976d8d", "apron-0668b9eba416c2f5", "teaspoon-0db2e5f59e2f180a", "knife4-7952054c9aff22d1" ], "reasoning": "1. I watched for the user to get out the pizza, which is on a plate (plate3-c1ee5a65bff5ad66) they retrieve from the fridge at 06:30.\n2. I watched the user put the plate (plate3-c1ee5a65bff5ad66) on the counter at 06:38.\n3. At 06:47, the user retrieves a knife from the cabinet.\n4. At 06:52, the user makes the first cut with the knife .\n5. I observed the direction in which the knife was pointing, and it was pointing at the water dispense on the counter.", "referenced_object_masks": [ "plate3-c1ee5a65bff5ad66" ] }, { "key": "P03-20240217-094713:08af2398e88cfeae1741224b471e451e51271be7", "video_id": "P03-20240217-094713", "question": "If I would have placed the jam on the first fridge shelf I temporarily placed the jam on, how many items would have been on that shelf?", "answer_choice_0": "6", "answer_choice_1": "1", "answer_choice_2": "3", "answer_choice_3": "2", "answer_choice_4": "4", "answer_id": 2, "answer": "3", "question_type": [ "Counting", "Counterfactual", "Object Recognition", "Situational Awareness", "Action Recognition" ], "objects_of_interest": [ "blueberry box-e2dd928d8f4f11bd", "bowl-8767c8d06486e0d1", "coffee jug-5f5b275604ca825d", "cling film-4873777ec5aa0bb3", "coffee filter-562d46fb8748fceb", "cloth-3ffb1037d2a77790", "jam jar-8f70da62ae072436", "butter-c7c2d03fd16bcc1e", "bread-63bd2bcc2a697ad8", "bread piece-8206cdb39dd1853a" ], "reasoning": "1. I watched the video to find that the user grabs the jam jar (jam jar-8f70da62ae072436) at 02:11.\n2. I then observed the user raise the jam jar (jam jar-8f70da62ae072436) and turn left towards the refrigerator .\n3. After opening the fridge door, I saw the user temporarily place the jam jar (jam jar-8f70da62ae072436) on the second shelf at 02:15, which has a carton of eggs and another jar .\n4. I then saw the user firmly place the jam jar (jam jar-8f70da62ae072436) on the top shelf at 02:16.\n5. There are two items on the middle shelf, so the jam jar (jam jar-8f70da62ae072436) staying there would make it a total of 3 items.", "referenced_object_masks": [ "jam jar-8f70da62ae072436" ] }, { "key": "P03-20240217-094713:3c306fb4a01e75968fc397d2f01c214b64d54da0", "video_id": "P03-20240217-094713", "question": "What is the fourth thing I touch with my right hand?", "answer_choice_0": "The kitchen faucet", "answer_choice_1": "The white bowl", "answer_choice_2": "The French press", "answer_choice_3": "The cabinet door", "answer_choice_4": "The fridge door", "answer_id": 1, "answer": "The white bowl", "question_type": [ "Object Recognition", "Temporal Reasoning", "Situational Awareness" ], "objects_of_interest": [ "blueberry box-e2dd928d8f4f11bd", "bowl-8767c8d06486e0d1", "coffee jug-5f5b275604ca825d", "cling film-4873777ec5aa0bb3", "coffee filter-562d46fb8748fceb", "cloth-3ffb1037d2a77790", "jam jar-8f70da62ae072436", "butter-c7c2d03fd16bcc1e", "bread-63bd2bcc2a697ad8", "bread piece-8206cdb39dd1853a" ], "reasoning": "1. I watched the video and noted that the first thing the user touches with their right hand is the camera lens at 00:02.\n2. I continued to watch and noted that the second thing the user touches is the handle of the refrigerator at 00:05.\n3. The user then grabs the blueberry box (blueberry box-e2dd928d8f4f11bd) and touches it with their right hand at 00:11.\n4. I continued to watch and saw that the user grabs the white bowl (bowl-8767c8d06486e0d1) with their right hand at 00:18, marking the fourth object touched with the user's right hand.", "referenced_object_masks": [ "blueberry box-e2dd928d8f4f11bd", "bowl-8767c8d06486e0d1" ] }, { "key": "P03-20240217-094713:68820f2c35f7b247f106a765260bc36dd81a1cff", "video_id": "P03-20240217-094713", "question": "What is the product of the number of colored cups on the middle shelf of the cabinet and the number of times I drain the blueberries while rinsing them?", "answer_choice_0": "16", "answer_choice_1": "28", "answer_choice_2": "25", "answer_choice_3": "40", "answer_choice_4": "32", "answer_id": 2, "answer": "25", "question_type": [ "Counting", "Numerical Reasoning", "Object Recognition", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "blueberry box-e2dd928d8f4f11bd", "bowl-8767c8d06486e0d1", "coffee jug-5f5b275604ca825d", "cling film-4873777ec5aa0bb3", "coffee filter-562d46fb8748fceb", "cloth-3ffb1037d2a77790", "jam jar-8f70da62ae072436", "butter-c7c2d03fd16bcc1e", "bread-63bd2bcc2a697ad8", "bread piece-8206cdb39dd1853a" ], "reasoning": "1. I watched the video to observe the user open the cabinet at \n00:17.\n2. Once opened, I looked at the middle shelf and counted five colored cups on the left side of the shelf.\n3. I continued to watch to identify when the user begins washing the blueberries (blueberry box-e2dd928d8f4f11bd), which begins at 01:01.\n4. While washing the berries (blueberry box-e2dd928d8f4f11bd), I observed that the user tilts the bowl to drain the water first from 01:06-01:07.\n5. The user continues to wash the berries (blueberry box-e2dd928d8f4f11bd) and drains them again from 01:16-01:17.\n6. Then, the user washed the berries (blueberry box-e2dd928d8f4f11bd) more, draining them for a third time at 01:24-01:25\n7. I watched further to 01:36-01:37, when the user drains the berries (blueberry box-e2dd928d8f4f11bd) a fourth time.\n8. I continued to watch and saw the user drain the berries (blueberry box-e2dd928d8f4f11bd) for the fifth time from 01:41-01:46.\n9. At 01:48, I observed the user stop washing the berries (blueberry box-e2dd928d8f4f11bd), confirming that they drain them 5 times.\n10. I then did the math, which is 5 x 5 = 25.", "referenced_object_masks": [ "blueberry box-e2dd928d8f4f11bd" ] }, { "key": "P03-20240217-094713:a904a129840c495eebb365c8cff00c94b0607312", "video_id": "P03-20240217-094713", "question": "If I turned right while standing next to the refrigerator at 00:01 instead of walking forward, what would I have been looking at?", "answer_choice_0": "A microwave", "answer_choice_1": "The kitchen sink", "answer_choice_2": "A washing machine", "answer_choice_3": "The oven", "answer_choice_4": "A dining table", "answer_id": 4, "answer": "A dining table", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Counterfactual", "Situational Awareness" ], "objects_of_interest": [ "blueberry box-e2dd928d8f4f11bd", "bowl-8767c8d06486e0d1", "coffee jug-5f5b275604ca825d", "cling film-4873777ec5aa0bb3", "coffee filter-562d46fb8748fceb", "cloth-3ffb1037d2a77790", "jam jar-8f70da62ae072436", "butter-c7c2d03fd16bcc1e", "bread-63bd2bcc2a697ad8", "bread piece-8206cdb39dd1853a" ], "reasoning": "1. I watched the video to note the refrigerator to the left of the user at 00:01.\n2. I continued to watch the video and saw the user turn back to look down the hallway at 00:43.\n3. At this point, I observed a dining table across from the refrigerator , indicating that this would have been the item seen if the user turned right at the refrigerator.", "referenced_object_masks": [] }, { "key": "P03-20240217-094713:afbc643ef707ed861a2444baf60912f79a62f17c", "video_id": "P03-20240217-094713", "question": "What do I do after placing the bread scraps on the little cart?", "answer_choice_0": "Pat the hands together", "answer_choice_1": "Open the fridge", "answer_choice_2": "Take a drink of water", "answer_choice_3": "Eat berries", "answer_choice_4": "Grab the bread knife", "answer_id": 0, "answer": "Pat the hands together", "question_type": [ "Event Occurence", "Situational Awareness", "Egocentric & Spatial Perception", "Action Recognition" ], "objects_of_interest": [ "blueberry box-e2dd928d8f4f11bd", "bowl-8767c8d06486e0d1", "coffee jug-5f5b275604ca825d", "cling film-4873777ec5aa0bb3", "coffee filter-562d46fb8748fceb", "cloth-3ffb1037d2a77790", "jam jar-8f70da62ae072436", "butter-c7c2d03fd16bcc1e", "bread-63bd2bcc2a697ad8", "bread piece-8206cdb39dd1853a" ], "reasoning": "1. I watched the video and saw the user put a small piece of bread (bread piece-8206cdb39dd1853a) in cling wrap with a larger piece of bread (bread-63bd2bcc2a697ad8) from 02:24-02:33.\n2. After closing up the bread (bread-63bd2bcc2a697ad8), the user turns and places the bread (bread-63bd2bcc2a697ad8) on the small cart at 02:36.\n3. Immediately following this action, the user pats their hands together from 02:38-02:40.", "referenced_object_masks": [ "bread-63bd2bcc2a697ad8", "bread piece-8206cdb39dd1853a" ] }, { "key": "P03-20240217-100448:1ca100354c7d03874130189898a07be5903a2a99", "video_id": "P03-20240217-100448", "question": "Where did I put the second plate after cleaning it?", "answer_choice_0": "In the dishwasher's bottom shelf", "answer_choice_1": "Back on the countertop", "answer_choice_2": "In the middle of the table", "answer_choice_3": "In the dishwasher's top shelf", "answer_choice_4": "In the empty side of the sink", "answer_id": 3, "answer": "In the dishwasher's top shelf", "question_type": [ "Object Recognition", "Action Recognition", "Counting", "Situational Awareness", "Temporal Reasoning" ], "objects_of_interest": [ "fork-8533d662db0de062", "knife-74728e85091dbd5b", "plates-59cee0d653b529ee", "plate-46b3f438ab06d4c1", "spoon-eacfc4447b5ed571", "dishwasher brush-b506e2b0042fff96", "chair-5d031f4cf28f3b57", "other chair-7224ccca5c378a26", "cup-848dcd8d0011fa21", "coffee filter-7a85f6816b0e9091", "coffee filter stand-a50a5c02fe37fc8d", "sponge-f6ad5670ccdf6e4c", "jug-44fa0861eef25286", "plastic bag-ade154ac720bdc8f", "white cloth-295eafbb3797f59d", "kettle-6c54beb2420fb491", "second tea towel-ec36da1b0b915883", "first tea towel-67bf9d279e648dd6", "toaster-08e26c80282a86d5", "basket-41b825faf37029c5", "yeast box-b105f7855f885592", "flour bag-03040b15dc738f3f", "bag-6834ba7d107c8a1b", "box-53db585994653685", "another bag-6e1e93afb332c5d9", "unknown-124dbbcf6621e48a", "clips-ea2cda8f7eb5c9b7" ], "reasoning": "1. I watched the video and found 2 plates (plates-59cee0d653b529ee, plate-46b3f438ab06d4c1) at 00:01.\n2. I saw the user clean the first plate (plates-59cee0d653b529ee) at 00:31.\n3. I saw the user clean the second plate (plate-46b3f438ab06d4c1) at 00:43.\n4. I saw the user put the second plate (plate-46b3f438ab06d4c1) in the dishwasher's top shelf at 00:46.\n5. Therefore, the user put the second plate (plate-46b3f438ab06d4c1) in the dishwasher's top shelf after cleaning it.", "referenced_object_masks": [ "plates-59cee0d653b529ee", "plate-46b3f438ab06d4c1" ] }, { "key": "P03-20240217-100448:3679817f93fb755e60568383a4b8e012f4d5f569", "video_id": "P03-20240217-100448", "question": "What did I fold the second tea towel into?", "answer_choice_0": "Sixteenths", "answer_choice_1": "Eighths", "answer_choice_2": "No fold", "answer_choice_3": "Half", "answer_choice_4": "Fourths", "answer_id": 1, "answer": "Eighths", "question_type": [ "Object Recognition", "Counting", "Action Recognition", "Egocentric & Spatial Perception", "State Changes" ], "objects_of_interest": [ "fork-8533d662db0de062", "knife-74728e85091dbd5b", "plates-59cee0d653b529ee", "plate-46b3f438ab06d4c1", "spoon-eacfc4447b5ed571", "dishwasher brush-b506e2b0042fff96", "chair-5d031f4cf28f3b57", "other chair-7224ccca5c378a26", "cup-848dcd8d0011fa21", "coffee filter-7a85f6816b0e9091", "coffee filter stand-a50a5c02fe37fc8d", "sponge-f6ad5670ccdf6e4c", "jug-44fa0861eef25286", "plastic bag-ade154ac720bdc8f", "white cloth-295eafbb3797f59d", "kettle-6c54beb2420fb491", "second tea towel-ec36da1b0b915883", "first tea towel-67bf9d279e648dd6", "toaster-08e26c80282a86d5", "basket-41b825faf37029c5", "yeast box-b105f7855f885592", "flour bag-03040b15dc738f3f", "bag-6834ba7d107c8a1b", "box-53db585994653685", "another bag-6e1e93afb332c5d9", "unknown-124dbbcf6621e48a", "clips-ea2cda8f7eb5c9b7" ], "reasoning": "1. I watched the video and found 2 tea towels (second tea towel-ec36da1b0b915883, first tea towel-67bf9d279e648dd6) at 03:05.\n2. I saw the user fold the first tea towel (second tea towel-ec36da1b0b915883) at 03:20.\n3. I saw the user fold the second tea towel (first tea towel-67bf9d279e648dd6) in half at 03:38.\n4. I saw the user fold the second tea towel (first tea towel-67bf9d279e648dd6) into fourths at 03:42.\n5. I saw the user fold the second tea towel (first tea towel-67bf9d279e648dd6) into eighths at 03:43.\n6. Therefore, the user folded the second tea towel (first tea towel-67bf9d279e648dd6) into eighths.", "referenced_object_masks": [ "second tea towel-ec36da1b0b915883", "first tea towel-67bf9d279e648dd6" ] }, { "key": "P03-20240217-100448:681cb3fcd571b89ac064028960972b3d407b6e82", "video_id": "P03-20240217-100448", "question": "What was the last thing I touched on the kitchen countertop?", "answer_choice_0": "The microwave", "answer_choice_1": "The toaster", "answer_choice_2": "A white rag", "answer_choice_3": "The paper towels", "answer_choice_4": "An orange towel", "answer_id": 2, "answer": "A white rag", "question_type": [ "Situational Awareness", "Object Recognition", "Action Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "fork-8533d662db0de062", "knife-74728e85091dbd5b", "plates-59cee0d653b529ee", "plate-46b3f438ab06d4c1", "spoon-eacfc4447b5ed571", "dishwasher brush-b506e2b0042fff96", "chair-5d031f4cf28f3b57", "other chair-7224ccca5c378a26", "cup-848dcd8d0011fa21", "coffee filter-7a85f6816b0e9091", "coffee filter stand-a50a5c02fe37fc8d", "sponge-f6ad5670ccdf6e4c", "jug-44fa0861eef25286", "plastic bag-ade154ac720bdc8f", "white cloth-295eafbb3797f59d", "kettle-6c54beb2420fb491", "second tea towel-ec36da1b0b915883", "first tea towel-67bf9d279e648dd6", "toaster-08e26c80282a86d5", "basket-41b825faf37029c5", "yeast box-b105f7855f885592", "flour bag-03040b15dc738f3f", "bag-6834ba7d107c8a1b", "box-53db585994653685", "another bag-6e1e93afb332c5d9", "unknown-124dbbcf6621e48a", "clips-ea2cda8f7eb5c9b7" ], "reasoning": "1. I watched the video and found the kitchen countertop at 00:01.\n2. At 05:01-05:04, I saw the user grab and fold a white rag on the countertop.\n3. I saw the user did not touch anything else on the countertop after 05:04.\n4. Therefore, the last thing the user touched on the kitchen countertop was a white rag .", "referenced_object_masks": [] }, { "key": "P03-20240217-100448:84e658aba81ae812dede02f09a0dd1aaafc7c8e5", "video_id": "P03-20240217-100448", "question": "If I doubled the total number of items I took off the wooden table, then divided that number by the number of times I folded the tea towels before taking them off the wooden table, what number would I get?", "answer_choice_0": "4.3", "answer_choice_1": "7.2", "answer_choice_2": "5.4", "answer_choice_3": "3.6", "answer_choice_4": "2.5", "answer_id": 3, "answer": "3.6", "question_type": [ "Object Recognition", "Action Recognition", "Counting", "Egocentric & Spatial Perception", "Numerical Reasoning", "Counterfactual", "Situational Awareness" ], "objects_of_interest": [ "fork-8533d662db0de062", "knife-74728e85091dbd5b", "plates-59cee0d653b529ee", "plate-46b3f438ab06d4c1", "spoon-eacfc4447b5ed571", "dishwasher brush-b506e2b0042fff96", "chair-5d031f4cf28f3b57", "other chair-7224ccca5c378a26", "cup-848dcd8d0011fa21", "coffee filter-7a85f6816b0e9091", "coffee filter stand-a50a5c02fe37fc8d", "sponge-f6ad5670ccdf6e4c", "jug-44fa0861eef25286", "plastic bag-ade154ac720bdc8f", "white cloth-295eafbb3797f59d", "kettle-6c54beb2420fb491", "second tea towel-ec36da1b0b915883", "first tea towel-67bf9d279e648dd6", "toaster-08e26c80282a86d5", "basket-41b825faf37029c5", "yeast box-b105f7855f885592", "flour bag-03040b15dc738f3f", "bag-6834ba7d107c8a1b", "box-53db585994653685", "another bag-6e1e93afb332c5d9", "unknown-124dbbcf6621e48a", "clips-ea2cda8f7eb5c9b7" ], "reasoning": "1. I watched the video and found the wooden table at 00:01.\n2. I watched for the user to take items off the wooden table . I saw them grab 6 total items -- 2 plates (plates-59cee0d653b529ee, plate-46b3f438ab06d4c1), 2 knives (knife-74728e85091dbd5b), a fork (fork-8533d662db0de062), and a spoon (spoon-eacfc4447b5ed571) -- from the table at 00:04-00:07. \n3. I continued watching and saw the user grab a mug (cup-848dcd8d0011fa21) from the table at 01:00. \n4. At 03:09, I saw the user grab a tea towel (second tea towel-ec36da1b0b915883) from the table . \n5. At 03:32, the user grabs a second tea towel (first tea towel-67bf9d279e648dd6) from the table . \n6. I added up the total number of items and got 9. \n7. I did the equation, 9*2 and got 18. \n8. Next, I watched for the user to fold the tea towels (second tea towel-ec36da1b0b915883, first tea towel-67bf9d279e648dd6) before taking them off the table . I saw the user fold the first tea towel (second tea towel-ec36da1b0b915883) a total of 4 times at 03:06-03:09. \n9. At 03:31, I saw the user fold the second tea towel (first tea towel-67bf9d279e648dd6) once before taking it off the table . \n10. I added together the total number of times the user folded the tea towels (second tea towel-ec36da1b0b915883, first tea towel-67bf9d279e648dd6) before taking them off the table and got 5. \n11. Finally, I divided 18 by 5 and got a final answer of 3.6.", "referenced_object_masks": [ "spoon-eacfc4447b5ed571", "plates-59cee0d653b529ee", "plate-46b3f438ab06d4c1", "first tea towel-67bf9d279e648dd6", "cup-848dcd8d0011fa21", "fork-8533d662db0de062", "second tea towel-ec36da1b0b915883", "knife-74728e85091dbd5b" ] }, { "key": "P03-20240217-100448:dcf2355484357b40272d2320007735180b0f9deb", "video_id": "P03-20240217-100448", "question": "What was the third piece of silverware I put in the dishwasher?", "answer_choice_0": "Fork", "answer_choice_1": "Knife", "answer_choice_2": "Spoon", "answer_choice_3": "Spork", "answer_choice_4": "Spatula", "answer_id": 1, "answer": "Knife", "question_type": [ "Object Recognition", "Counting", "Action Recognition", "Situational Awareness" ], "objects_of_interest": [ "fork-8533d662db0de062", "knife-74728e85091dbd5b", "plates-59cee0d653b529ee", "plate-46b3f438ab06d4c1", "spoon-eacfc4447b5ed571", "dishwasher brush-b506e2b0042fff96", "chair-5d031f4cf28f3b57", "other chair-7224ccca5c378a26", "cup-848dcd8d0011fa21", "coffee filter-7a85f6816b0e9091", "coffee filter stand-a50a5c02fe37fc8d", "sponge-f6ad5670ccdf6e4c", "jug-44fa0861eef25286", "plastic bag-ade154ac720bdc8f", "white cloth-295eafbb3797f59d", "kettle-6c54beb2420fb491", "second tea towel-ec36da1b0b915883", "first tea towel-67bf9d279e648dd6", "toaster-08e26c80282a86d5", "basket-41b825faf37029c5", "yeast box-b105f7855f885592", "flour bag-03040b15dc738f3f", "bag-6834ba7d107c8a1b", "box-53db585994653685", "another bag-6e1e93afb332c5d9", "unknown-124dbbcf6621e48a", "clips-ea2cda8f7eb5c9b7" ], "reasoning": "1. I watched the video and counted and identified each piece of silverware that the user put in the dishwasher .\n2. I saw the user put the first piece of silverware in the dishwasher at 00:26, which was a fork (fork-8533d662db0de062).\n3. I saw the user put the second piece of silverware in the dishwasher at 00:27, which was a spoon (spoon-eacfc4447b5ed571).\n4. I saw the user put the third piece of silverware in the dishwasher at 00:28, which was a knife .\n5. Therefore, the third piece of silverware that the user put in the dishwasher was a knife .", "referenced_object_masks": [ "spoon-eacfc4447b5ed571", "fork-8533d662db0de062" ] }, { "key": "P03-20240217-130208:41473471aa1c6250a368411660c29dde08261d1e", "video_id": "P03-20240217-130208", "question": "What is the number of plates in the cabinet after I remove the red plate?", "answer_choice_0": "5", "answer_choice_1": "2", "answer_choice_2": "4", "answer_choice_3": "1", "answer_choice_4": "3", "answer_id": 1, "answer": "2", "question_type": [ "Temporal Reasoning", "Object Recognition", "Numerical Reasoning", "Counting", "Action Recognition" ], "objects_of_interest": [ "plate-1c7899ff4cee801b", "plate2-8557be62473be1a2", "knife-0a92bb73e35e2710", "spanish omelette slice-fae1bbcac14786d0", "multiple tea towels-5d0a5b438d7b552f", "bread-39d51042e38358a4", "tea towel-8a7e9d2865461db4", "flower pot-02db80076a461066", "other tea towel-56ebee6089ff0c1c", "glass-bb54ee2cb98b48a7", "glass2-5d742a633a41f3b1", "fork1-5d1e45e20d209537", "fork2-4b12c9750f4bc7c2" ], "reasoning": "1. I determine that the user opens the cabinet at 00:15.\n2. I locate a stack of 3 plates (plate2-8557be62473be1a2) on the middle shelf in the cabinet at 00:15.\n3. I observe the user remove the red plate (plate2-8557be62473be1a2) at 00:16.\n4. I subtract 1 from 3.", "referenced_object_masks": [ "plate2-8557be62473be1a2" ] }, { "key": "P03-20240217-130208:5b117f6be2dc42e5cca04362f537e87ba43bc132", "video_id": "P03-20240217-130208", "question": "How many tea towels make up the table setting?", "answer_choice_0": "4", "answer_choice_1": "2", "answer_choice_2": "1", "answer_choice_3": "5", "answer_choice_4": "3", "answer_id": 1, "answer": "2", "question_type": [ "Temporal Reasoning", "Egocentric & Spatial Perception", "Object Recognition", "Counting", "Action Recognition" ], "objects_of_interest": [ "plate-1c7899ff4cee801b", "plate2-8557be62473be1a2", "knife-0a92bb73e35e2710", "spanish omelette slice-fae1bbcac14786d0", "multiple tea towels-5d0a5b438d7b552f", "bread-39d51042e38358a4", "tea towel-8a7e9d2865461db4", "flower pot-02db80076a461066", "other tea towel-56ebee6089ff0c1c", "glass-bb54ee2cb98b48a7", "glass2-5d742a633a41f3b1", "fork1-5d1e45e20d209537", "fork2-4b12c9750f4bc7c2" ], "reasoning": "1. I locate the tea towels (multiple tea towels-5d0a5b438d7b552f), which first appear in the cabinet at 01:12.\n2. I identify that the user places the tea towels (multiple tea towels-5d0a5b438d7b552f) on the table at 01:19.\n3. I observe that the user places the first tea towel (tea towel-8a7e9d2865461db4) on the left side of the table at 01:22.\n4. I determine that the user places the second tea towel (other tea towel-56ebee6089ff0c1c) on the right side of the table at 01:34.", "referenced_object_masks": [ "other tea towel-56ebee6089ff0c1c", "multiple tea towels-5d0a5b438d7b552f", "tea towel-8a7e9d2865461db4" ] }, { "key": "P03-20240217-130208:9da9480ab315055b371efd3e836a4b7a389479b9", "video_id": "P03-20240217-130208", "question": "What number would I get if I subtracted the number of omelette slices I cut for myself from the number of bowl types in the cabinet bottom shelf?", "answer_choice_0": "3", "answer_choice_1": "4", "answer_choice_2": "5", "answer_choice_3": "1", "answer_choice_4": "2", "answer_id": 4, "answer": "2", "question_type": [ "Temporal Reasoning", "Egocentric & Spatial Perception", "Object Recognition", "Numerical Reasoning", "Counting", "Action Recognition" ], "objects_of_interest": [ "plate-1c7899ff4cee801b", "plate2-8557be62473be1a2", "knife-0a92bb73e35e2710", "spanish omelette slice-fae1bbcac14786d0", "multiple tea towels-5d0a5b438d7b552f", "bread-39d51042e38358a4", "tea towel-8a7e9d2865461db4", "flower pot-02db80076a461066", "other tea towel-56ebee6089ff0c1c", "glass-bb54ee2cb98b48a7", "glass2-5d742a633a41f3b1", "fork1-5d1e45e20d209537", "fork2-4b12c9750f4bc7c2" ], "reasoning": "1. I locate the omelette , which first appears on a plate (plate-1c7899ff4cee801b) in the fridge at 00:03.\n2. I determine that the user cuts themself 1 slice (spanish omelette slice-fae1bbcac14786d0) from 00:36 to 00:38.\n3. I determine that the user opens the cabinet at 00:15, revealing the cabinet's bottom shelf.\n4. I count 3 different types of bowls at 00:15.\n5. I subtract 1 from 3.", "referenced_object_masks": [ "spanish omelette slice-fae1bbcac14786d0", "plate-1c7899ff4cee801b" ] }, { "key": "P03-20240217-130208:a9b814bd3a04b4e914c19a2da984626b6f6c8e3a", "video_id": "P03-20240217-130208", "question": "What number would I get if I subtracted the number of flower colors in the flower vase at the kitchen table from the first visible time on the microwave?", "answer_choice_0": "1298", "answer_choice_1": "97", "answer_choice_2": "98", "answer_choice_3": "1297", "answer_choice_4": "298", "answer_id": 3, "answer": "1297", "question_type": [ "Temporal Reasoning", "Egocentric & Spatial Perception", "Object Recognition", "Numerical Reasoning", "Counting" ], "objects_of_interest": [ "plate-1c7899ff4cee801b", "plate2-8557be62473be1a2", "knife-0a92bb73e35e2710", "spanish omelette slice-fae1bbcac14786d0", "multiple tea towels-5d0a5b438d7b552f", "bread-39d51042e38358a4", "tea towel-8a7e9d2865461db4", "flower pot-02db80076a461066", "other tea towel-56ebee6089ff0c1c", "glass-bb54ee2cb98b48a7", "glass2-5d742a633a41f3b1", "fork1-5d1e45e20d209537", "fork2-4b12c9750f4bc7c2" ], "reasoning": "1. I locate the flower vase (flower pot-02db80076a461066), which appears at 01:25 on top of the kitchen table.\n2. I determine there are 3 different colored flowers in the flower vase (flower pot-02db80076a461066).\n3. I locate the microwave , which comes into view at 00:07 to the left of the sink .\n3. I identify when the time shown on the microwave is first shown, which is at 00:51.\n4. I determine the time on the microwave at 00:51, which is 13:00.\n5. I subtract 3 from 1300.", "referenced_object_masks": [ "flower pot-02db80076a461066" ] }, { "key": "P03-20240217-130208:cff6021bb35af408726557d4f16cec7f30fb42f9", "video_id": "P03-20240217-130208", "question": "Where is the sink in relation to me when I finish setting the table?", "answer_choice_0": "Below", "answer_choice_1": "Behind", "answer_choice_2": "To the right", "answer_choice_3": "To the left", "answer_choice_4": "Directly ahead", "answer_id": 3, "answer": "To the left", "question_type": [ "Temporal Reasoning", "Egocentric & Spatial Perception", "Situational Awareness", "Object Recognition" ], "objects_of_interest": [ "plate-1c7899ff4cee801b", "plate2-8557be62473be1a2", "knife-0a92bb73e35e2710", "spanish omelette slice-fae1bbcac14786d0", "multiple tea towels-5d0a5b438d7b552f", "bread-39d51042e38358a4", "tea towel-8a7e9d2865461db4", "flower pot-02db80076a461066", "other tea towel-56ebee6089ff0c1c", "glass-bb54ee2cb98b48a7", "glass2-5d742a633a41f3b1", "fork1-5d1e45e20d209537", "fork2-4b12c9750f4bc7c2" ], "reasoning": "1. I locate the sink , which first appears at 00:00 in the kitchen directly ahead of the user. \n2. I determine that the table appears at 00:00 directly to the right of the user.\n3. I determine that the user finishes setting the table at 02:08 when they place the meal (spanish omelette slice-fae1bbcac14786d0) down.\n4. I identify the location of the sink in relation to the user as they place the omelette (spanish omelette slice-fae1bbcac14786d0) onto the table at 02:08.", "referenced_object_masks": [ "spanish omelette slice-fae1bbcac14786d0" ] }, { "key": "P03-20240217-131219:1d3e52d450f75c308281801becc5bf7431fcbd71", "video_id": "P03-20240217-131219", "question": "When I reach for the knife in the cabinet, where is the original location of the squash in relation to me?", "answer_choice_0": "To the right", "answer_choice_1": "To the left", "answer_choice_2": "Behind", "answer_choice_3": "Directly in front", "answer_choice_4": "Above", "answer_id": 0, "answer": "To the right", "question_type": [ "Egocentric & Spatial Perception", "Event Occurence", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "right fork-9e719de8129aabab", "left fork-787a051fdb7b62b0", "recipe card-9d6cea18cbad9538", "folded white cloth-6456acd187f9a628", "pack of lentils-f62cf03618da91cd", "small knife-2ef4cd3b6b62247a", "scale-40685c5361cef4b5", "plastic colander-789c0c8f1379eed4", "scissors-7b21ea63ea07beb1", "seal tap-bc05cc4f3c7fb8c9", "cup-b560db3403d371bb", "cloth-7882b5a9a2352ec9", "chopping board-cb59981e62d4365d", "squash-6982230d527110b3", "big knife-d2a0f8a627e9277f", "one fourth left side squash-ce98c9ca9f000394", "right side squash-b9022d18f19c8310", "squash fourth piece-5d7f073b3c81689d", "sponge-1e546335a9c90260", "squash fifth piece-f24765fb6495a8a2", "squash sixth piece-c2371a92b3413242", "squash seventh piece-3d117a5ad3b43588", "squash eighth piece-8de19e02846c4548", "squash ninth piece-37e42be4b32e5985", "quash tenth piece-5d5d80e1cc678ffc", "large bowl-d4a9d64b9a7935bd", "peeled chunks of squash-f8f5e0a8c2fca8e8", "squash eleventh piece-f8d483d24adc468d", "squash twelfth piece-6a525c58c3f1cd55", "squash thirteenth piece-649c7f35a92ab614", "squash fourteenth piece-72a65f7315e1e6b8", "squash fifteenth piece-c17b673e6cc5358f", "squash sixteenth piece-e078956436211ba8", "squash seventeenth piece-652bae6e1c3273f7", "squash eighteenth piece-23a69ab374b090a5", "squash nineteenth piece-7a55f7263deb348b", "squash twentieth piece-e1937bf0abd1b8a4", "squash twenty-first piece-a80dc0c5c7692223", "few chunks of squash-2a980da60ae1f48d", "more chunks of squash-b55258824c020135", "last chunks of squash-6d0adf45129486b8", "remaining squash-8ba189bfb842a210", "squash last piece-0fbdf69f3bb5121c", "squash label-b275e3e17336d694", "squash twenty-second piece-d7fdafa680deb28a", "squash twenth-third piece-65cc8582de3dd261", "squash twenty-fourth piece-74e443355339d754", "squash twenty-fifth piece-1e571e990857ed6f", "squash twenty-sixth piece-0219dd2912f3f647", "squash twenty-seventh piece-b5bd69b93489bdeb", "squash twenty-eighth piece-a8d23b7f67f87c4c", "peeled chunk of squash-b9123a6432d32df4", "peeled chunks of squash 2-57ae00949b9234a7", "few more peeled squash chunks-e58bcdce444d7de4", "few more peeled squash chunks 2-6a0922ed69a6eff0", "more peeled squash chunks-2e21fe2f58a3c7dd", "more peeled squash chunks 3-fef2c998c96d623c", "last bits of squash-cc7d20b0488f5bb9", "bag of lettuce-481dd3dd502eb163", "bunch of spring greens-6ef90806487cc132", "leaf of spring greens-82c25823cb6df600", "second leaf of spring green-6157b96202ddc216", "third leaf of spring green-a32cc06c7c69ec31", "fourth leaf of spring greens-ce46765de085652e", "fifth leaf of spring greens-6458b0ab9d019f25", "spring green leaves-8ba878ac39101f61", "sixth leaf of spring green-308f01ab292133ad", "strips of spring green leaves-efc517aed4e95109", "second large bowl-ea9b6dc08e9a7257", "can of coconut milk-16f046dadbc13f81", "box of cling film-40089125571e5dcf", "partly wrapped squash-a01241c053751ad6", "cling film-32dc53be2190afc1", "plastic bag of onions-5e6cdb503c76d4e3", "onion-6d70efcb6a560beb", "peeled half of onion-54aaecb23e7b5487", "second half of peeled onion-a2e290d4d04c1bbb", "stack of small frying pan-543c1020a503d69a", "frying pan grey-cacd8936c048acfb", "pan padding-4e551b32f278fe2f", "sunflower oil bottle-4141adf1f5c3e0ea", "partly chopped onion first half-bc0ddf57c5d4d86e", "plastic spatula-3b5bc3812ee99ade", "food waste bin-e04a4431ef2ec134", "peels-1ec6b4ae40d55983", "other vegetable peels-87e445b5e4a2b889", "new recipe card-2b32f55b2b537f5c", "recipe kit-84ae2ca3af7bf898", "recipe kit 2-0cf8da91d218cc58", "recipe kit 3-1b5b09077ccd79b1", "recipe kit box-c17f989efac06973", "teaspoon-3002a5fee10cab1f", "yellow plastic tub-85234776d45178d0", "inner part of plastic colander-62700200aa5bcb6d", "outer part of plastic colander-ae784b59e34414f5", "pink tub-62da6e3ddb49e8a4", "can opener-6798748b46b4bf7b", "last green plastic tub-131c5e91f81ba466", "glass-7019e116a8c3433f", "large glass lid-ef5d507a0d964784", "new track 2-d9cdd18e54fae096", "three plastic tubs-1941ddf33f716a85", "dishwasher brush-fef6398cb4354466", "v60 stand-ce24c492374d4d73", "v60 metal filter-6bb6a298e601a4c7", "coffee jug-99955b3e64ae51c0", "plastic food bag-710840db9ad365ed", "bread knife-13cf2382a2dcefbb", "black coffee spoon-33f368ccebba8f3d", "red plate-9a0a9378ec1991a1", "orange cloth-9550ea1395d8b999", "bit of plastic-e574dbaf80217f33", "cutlery stand-c4b5cf177f57dd97", "sink strainer-b4180c4533ecd01a", "rice box-89d873889fbc8de0", "long leaf plant-dc0cd2596d8d5c98", "other bowl-77ee35879fe06a66", "bread-923d61d123f0e8ec", "larger part of bread loaf-658ce2fc8929474d", "bread wrapping-4079edb10a4a41be", "ladle-968795cabd8dc591", "other spoon-9c25ab6fcf4127d4", "one fork-a08bbd8820f753a6", "third spoon-cf952ecf5957cc4f" ], "reasoning": "1. From 04:13 to 04:15, I observe the user pick up the butternut squash (squash-6982230d527110b3) from a kitchen cart and turn right to approach the counter.\n2. I observe the user reach their hand out to place the butternut squash (squash-6982230d527110b3) behind the cutting board (chopping board-cb59981e62d4365d) on the counter at 04:16.\n3. From 04:16 to 04:18, I observe the user turn about 180 degrees to approach the cabinet behind them.\n4. At 04:21, I observe the user reach for the knife (big knife-d2a0f8a627e9277f) in the cabinet.\n5. At 04:21, I determine that the original location of the squash is to the right of the user.", "referenced_object_masks": [ "big knife-d2a0f8a627e9277f", "squash-6982230d527110b3", "chopping board-cb59981e62d4365d" ] }, { "key": "P03-20240217-131219:1f91f50a439c1c57390b4aba82870d882496f62d", "video_id": "P03-20240217-131219", "question": "If I doubled the amount of time I poured the red pack of lentils, approximately how long would I have poured them?", "answer_choice_0": "40 seconds", "answer_choice_1": "10 seconds", "answer_choice_2": "20 seconds", "answer_choice_3": "50 seconds", "answer_choice_4": "30 seconds", "answer_id": 3, "answer": "50 seconds", "question_type": [ "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counting", "Numerical Reasoning", "Counterfactual" ], "objects_of_interest": [ "right fork-9e719de8129aabab", "left fork-787a051fdb7b62b0", "recipe card-9d6cea18cbad9538", "folded white cloth-6456acd187f9a628", "pack of lentils-f62cf03618da91cd", "small knife-2ef4cd3b6b62247a", "scale-40685c5361cef4b5", "plastic colander-789c0c8f1379eed4", "scissors-7b21ea63ea07beb1", "seal tap-bc05cc4f3c7fb8c9", "cup-b560db3403d371bb", "cloth-7882b5a9a2352ec9", "chopping board-cb59981e62d4365d", "squash-6982230d527110b3", "big knife-d2a0f8a627e9277f", "one fourth left side squash-ce98c9ca9f000394", "right side squash-b9022d18f19c8310", "squash fourth piece-5d7f073b3c81689d", "sponge-1e546335a9c90260", "squash fifth piece-f24765fb6495a8a2", "squash sixth piece-c2371a92b3413242", "squash seventh piece-3d117a5ad3b43588", "squash eighth piece-8de19e02846c4548", "squash ninth piece-37e42be4b32e5985", "quash tenth piece-5d5d80e1cc678ffc", "large bowl-d4a9d64b9a7935bd", "peeled chunks of squash-f8f5e0a8c2fca8e8", "squash eleventh piece-f8d483d24adc468d", "squash twelfth piece-6a525c58c3f1cd55", "squash thirteenth piece-649c7f35a92ab614", "squash fourteenth piece-72a65f7315e1e6b8", "squash fifteenth piece-c17b673e6cc5358f", "squash sixteenth piece-e078956436211ba8", "squash seventeenth piece-652bae6e1c3273f7", "squash eighteenth piece-23a69ab374b090a5", "squash nineteenth piece-7a55f7263deb348b", "squash twentieth piece-e1937bf0abd1b8a4", "squash twenty-first piece-a80dc0c5c7692223", "few chunks of squash-2a980da60ae1f48d", "more chunks of squash-b55258824c020135", "last chunks of squash-6d0adf45129486b8", "remaining squash-8ba189bfb842a210", "squash last piece-0fbdf69f3bb5121c", "squash label-b275e3e17336d694", "squash twenty-second piece-d7fdafa680deb28a", "squash twenth-third piece-65cc8582de3dd261", "squash twenty-fourth piece-74e443355339d754", "squash twenty-fifth piece-1e571e990857ed6f", "squash twenty-sixth piece-0219dd2912f3f647", "squash twenty-seventh piece-b5bd69b93489bdeb", "squash twenty-eighth piece-a8d23b7f67f87c4c", "peeled chunk of squash-b9123a6432d32df4", "peeled chunks of squash 2-57ae00949b9234a7", "few more peeled squash chunks-e58bcdce444d7de4", "few more peeled squash chunks 2-6a0922ed69a6eff0", "more peeled squash chunks-2e21fe2f58a3c7dd", "more peeled squash chunks 3-fef2c998c96d623c", "last bits of squash-cc7d20b0488f5bb9", "bag of lettuce-481dd3dd502eb163", "bunch of spring greens-6ef90806487cc132", "leaf of spring greens-82c25823cb6df600", "second leaf of spring green-6157b96202ddc216", "third leaf of spring green-a32cc06c7c69ec31", "fourth leaf of spring greens-ce46765de085652e", "fifth leaf of spring greens-6458b0ab9d019f25", "spring green leaves-8ba878ac39101f61", "sixth leaf of spring green-308f01ab292133ad", "strips of spring green leaves-efc517aed4e95109", "second large bowl-ea9b6dc08e9a7257", "can of coconut milk-16f046dadbc13f81", "box of cling film-40089125571e5dcf", "partly wrapped squash-a01241c053751ad6", "cling film-32dc53be2190afc1", "plastic bag of onions-5e6cdb503c76d4e3", "onion-6d70efcb6a560beb", "peeled half of onion-54aaecb23e7b5487", "second half of peeled onion-a2e290d4d04c1bbb", "stack of small frying pan-543c1020a503d69a", "frying pan grey-cacd8936c048acfb", "pan padding-4e551b32f278fe2f", "sunflower oil bottle-4141adf1f5c3e0ea", "partly chopped onion first half-bc0ddf57c5d4d86e", "plastic spatula-3b5bc3812ee99ade", "food waste bin-e04a4431ef2ec134", "peels-1ec6b4ae40d55983", "other vegetable peels-87e445b5e4a2b889", "new recipe card-2b32f55b2b537f5c", "recipe kit-84ae2ca3af7bf898", "recipe kit 2-0cf8da91d218cc58", "recipe kit 3-1b5b09077ccd79b1", "recipe kit box-c17f989efac06973", "teaspoon-3002a5fee10cab1f", "yellow plastic tub-85234776d45178d0", "inner part of plastic colander-62700200aa5bcb6d", "outer part of plastic colander-ae784b59e34414f5", "pink tub-62da6e3ddb49e8a4", "can opener-6798748b46b4bf7b", "last green plastic tub-131c5e91f81ba466", "glass-7019e116a8c3433f", "large glass lid-ef5d507a0d964784", "new track 2-d9cdd18e54fae096", "three plastic tubs-1941ddf33f716a85", "dishwasher brush-fef6398cb4354466", "v60 stand-ce24c492374d4d73", "v60 metal filter-6bb6a298e601a4c7", "coffee jug-99955b3e64ae51c0", "plastic food bag-710840db9ad365ed", "bread knife-13cf2382a2dcefbb", "black coffee spoon-33f368ccebba8f3d", "red plate-9a0a9378ec1991a1", "orange cloth-9550ea1395d8b999", "bit of plastic-e574dbaf80217f33", "cutlery stand-c4b5cf177f57dd97", "sink strainer-b4180c4533ecd01a", "rice box-89d873889fbc8de0", "long leaf plant-dc0cd2596d8d5c98", "other bowl-77ee35879fe06a66", "bread-923d61d123f0e8ec", "larger part of bread loaf-658ce2fc8929474d", "bread wrapping-4079edb10a4a41be", "ladle-968795cabd8dc591", "other spoon-9c25ab6fcf4127d4", "one fork-a08bbd8820f753a6", "third spoon-cf952ecf5957cc4f" ], "reasoning": "1. I watched the video and found the red pack of lentils (pack of lentils-f62cf03618da91cd) at 00:24.\n2. At 01:38, I saw the user open the pack of lentils (pack of lentils-f62cf03618da91cd) with scissors (scissors-7b21ea63ea07beb1).\n3. I saw the user pour the pack of lentils (pack of lentils-f62cf03618da91cd) into the plastic colander (plastic colander-789c0c8f1379eed4) from approximately 01:44-02:10.\n4. I counted the number of seconds from 01:44-02:10, calculating a total of 26 seconds.\n5. I calculated 26*2=52, which approximates to 50.\n6. Therefore, if the user doubled the amount of time they poured the red pack of lentils (pack of lentils-f62cf03618da91cd), they would have poured them for approximately 50 seconds.", "referenced_object_masks": [ "plastic colander-789c0c8f1379eed4", "pack of lentils-f62cf03618da91cd", "scissors-7b21ea63ea07beb1" ] }, { "key": "P03-20240217-131219:4b66d2d260c144191badce88847f3f0509f98cb0", "video_id": "P03-20240217-131219", "question": "What are the three different textures of the lentils in this video?", "answer_choice_0": "Dry and hard, then soaked, then softened by sauteing", "answer_choice_1": "Soft and grounded, then soaked, then hardened by sauteing", "answer_choice_2": "Soft and grounded, then soaked, then softened by sauteing", "answer_choice_3": "Dry and hard, then soaked, then hardened by sauteing", "answer_choice_4": "Dry and grounded, then soaked, then softened by baking", "answer_id": 0, "answer": "Dry and hard, then soaked, then softened by sauteing", "question_type": [ "Event Occurence", "Object Recognition", "Temporal Reasoning", "Action Recognition", "State Changes" ], "objects_of_interest": [ "right fork-9e719de8129aabab", "left fork-787a051fdb7b62b0", "recipe card-9d6cea18cbad9538", "folded white cloth-6456acd187f9a628", "pack of lentils-f62cf03618da91cd", "small knife-2ef4cd3b6b62247a", "scale-40685c5361cef4b5", "plastic colander-789c0c8f1379eed4", "scissors-7b21ea63ea07beb1", "seal tap-bc05cc4f3c7fb8c9", "cup-b560db3403d371bb", "cloth-7882b5a9a2352ec9", "chopping board-cb59981e62d4365d", "squash-6982230d527110b3", "big knife-d2a0f8a627e9277f", "one fourth left side squash-ce98c9ca9f000394", "right side squash-b9022d18f19c8310", "squash fourth piece-5d7f073b3c81689d", "sponge-1e546335a9c90260", "squash fifth piece-f24765fb6495a8a2", "squash sixth piece-c2371a92b3413242", "squash seventh piece-3d117a5ad3b43588", "squash eighth piece-8de19e02846c4548", "squash ninth piece-37e42be4b32e5985", "quash tenth piece-5d5d80e1cc678ffc", "large bowl-d4a9d64b9a7935bd", "peeled chunks of squash-f8f5e0a8c2fca8e8", "squash eleventh piece-f8d483d24adc468d", "squash twelfth piece-6a525c58c3f1cd55", "squash thirteenth piece-649c7f35a92ab614", "squash fourteenth piece-72a65f7315e1e6b8", "squash fifteenth piece-c17b673e6cc5358f", "squash sixteenth piece-e078956436211ba8", "squash seventeenth piece-652bae6e1c3273f7", "squash eighteenth piece-23a69ab374b090a5", "squash nineteenth piece-7a55f7263deb348b", "squash twentieth piece-e1937bf0abd1b8a4", "squash twenty-first piece-a80dc0c5c7692223", "few chunks of squash-2a980da60ae1f48d", "more chunks of squash-b55258824c020135", "last chunks of squash-6d0adf45129486b8", "remaining squash-8ba189bfb842a210", "squash last piece-0fbdf69f3bb5121c", "squash label-b275e3e17336d694", "squash twenty-second piece-d7fdafa680deb28a", "squash twenth-third piece-65cc8582de3dd261", "squash twenty-fourth piece-74e443355339d754", "squash twenty-fifth piece-1e571e990857ed6f", "squash twenty-sixth piece-0219dd2912f3f647", "squash twenty-seventh piece-b5bd69b93489bdeb", "squash twenty-eighth piece-a8d23b7f67f87c4c", "peeled chunk of squash-b9123a6432d32df4", "peeled chunks of squash 2-57ae00949b9234a7", "few more peeled squash chunks-e58bcdce444d7de4", "few more peeled squash chunks 2-6a0922ed69a6eff0", "more peeled squash chunks-2e21fe2f58a3c7dd", "more peeled squash chunks 3-fef2c998c96d623c", "last bits of squash-cc7d20b0488f5bb9", "bag of lettuce-481dd3dd502eb163", "bunch of spring greens-6ef90806487cc132", "leaf of spring greens-82c25823cb6df600", "second leaf of spring green-6157b96202ddc216", "third leaf of spring green-a32cc06c7c69ec31", "fourth leaf of spring greens-ce46765de085652e", "fifth leaf of spring greens-6458b0ab9d019f25", "spring green leaves-8ba878ac39101f61", "sixth leaf of spring green-308f01ab292133ad", "strips of spring green leaves-efc517aed4e95109", "second large bowl-ea9b6dc08e9a7257", "can of coconut milk-16f046dadbc13f81", "box of cling film-40089125571e5dcf", "partly wrapped squash-a01241c053751ad6", "cling film-32dc53be2190afc1", "plastic bag of onions-5e6cdb503c76d4e3", "onion-6d70efcb6a560beb", "peeled half of onion-54aaecb23e7b5487", "second half of peeled onion-a2e290d4d04c1bbb", "stack of small frying pan-543c1020a503d69a", "frying pan grey-cacd8936c048acfb", "pan padding-4e551b32f278fe2f", "sunflower oil bottle-4141adf1f5c3e0ea", "partly chopped onion first half-bc0ddf57c5d4d86e", "plastic spatula-3b5bc3812ee99ade", "food waste bin-e04a4431ef2ec134", "peels-1ec6b4ae40d55983", "other vegetable peels-87e445b5e4a2b889", "new recipe card-2b32f55b2b537f5c", "recipe kit-84ae2ca3af7bf898", "recipe kit 2-0cf8da91d218cc58", "recipe kit 3-1b5b09077ccd79b1", "recipe kit box-c17f989efac06973", "teaspoon-3002a5fee10cab1f", "yellow plastic tub-85234776d45178d0", "inner part of plastic colander-62700200aa5bcb6d", "outer part of plastic colander-ae784b59e34414f5", "pink tub-62da6e3ddb49e8a4", "can opener-6798748b46b4bf7b", "last green plastic tub-131c5e91f81ba466", "glass-7019e116a8c3433f", "large glass lid-ef5d507a0d964784", "new track 2-d9cdd18e54fae096", "three plastic tubs-1941ddf33f716a85", "dishwasher brush-fef6398cb4354466", "v60 stand-ce24c492374d4d73", "v60 metal filter-6bb6a298e601a4c7", "coffee jug-99955b3e64ae51c0", "plastic food bag-710840db9ad365ed", "bread knife-13cf2382a2dcefbb", "black coffee spoon-33f368ccebba8f3d", "red plate-9a0a9378ec1991a1", "orange cloth-9550ea1395d8b999", "bit of plastic-e574dbaf80217f33", "cutlery stand-c4b5cf177f57dd97", "sink strainer-b4180c4533ecd01a", "rice box-89d873889fbc8de0", "long leaf plant-dc0cd2596d8d5c98", "other bowl-77ee35879fe06a66", "bread-923d61d123f0e8ec", "larger part of bread loaf-658ce2fc8929474d", "bread wrapping-4079edb10a4a41be", "ladle-968795cabd8dc591", "other spoon-9c25ab6fcf4127d4", "one fork-a08bbd8820f753a6", "third spoon-cf952ecf5957cc4f" ], "reasoning": "1. I watched the whole video and saw the user at 01:38 open the bag of lentils (pack of lentils-f62cf03618da91cd).\n2. At this point, the lentils are hard.\n3. At 02:28, the user puts the lentils in a bowl, fills the bowl with water, mixes the lentils in the water, drains them and repeats the process.\n4. At 03:27, the user drains the lentils and puts the bowl of lentils on the counter. This process has begun to soften the lentils.\n5. At 30:38, the user puts the lentils in the pan on the stove that is saut\u00e9ing the onions and butternut squash.\n6. At 52:50, the user adds the sauteed lentils to the bowl (other bowl-77ee35879fe06a66).", "referenced_object_masks": [ "pack of lentils-f62cf03618da91cd", "other bowl-77ee35879fe06a66" ] }, { "key": "P03-20240217-131219:5499ece56385851f3309072b50149a76f2973a3d", "video_id": "P03-20240217-131219", "question": "If I take the number of times the user picks up butternut squash cubes with their hands and puts them in a bowl on the kitchen scale, multiply it by the number of leaves of lettuce the user breaks off from the bunch of lettuce and raise it to the power of the number of seconds it takes the user to fold the white towel and put it on top of the tupperware with the green top, what number would I get?", "answer_choice_0": "24,389", "answer_choice_1": "21,952", "answer_choice_2": "13,824", "answer_choice_3": "8,000", "answer_choice_4": "9,261", "answer_id": 2, "answer": "13,824", "question_type": [ "Action Recognition", "Counting", "Event Occurence", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "right fork-9e719de8129aabab", "left fork-787a051fdb7b62b0", "recipe card-9d6cea18cbad9538", "folded white cloth-6456acd187f9a628", "pack of lentils-f62cf03618da91cd", "small knife-2ef4cd3b6b62247a", "scale-40685c5361cef4b5", "plastic colander-789c0c8f1379eed4", "scissors-7b21ea63ea07beb1", "seal tap-bc05cc4f3c7fb8c9", "cup-b560db3403d371bb", "cloth-7882b5a9a2352ec9", "chopping board-cb59981e62d4365d", "squash-6982230d527110b3", "big knife-d2a0f8a627e9277f", "one fourth left side squash-ce98c9ca9f000394", "right side squash-b9022d18f19c8310", "squash fourth piece-5d7f073b3c81689d", "sponge-1e546335a9c90260", "squash fifth piece-f24765fb6495a8a2", "squash sixth piece-c2371a92b3413242", "squash seventh piece-3d117a5ad3b43588", "squash eighth piece-8de19e02846c4548", "squash ninth piece-37e42be4b32e5985", "quash tenth piece-5d5d80e1cc678ffc", "large bowl-d4a9d64b9a7935bd", "peeled chunks of squash-f8f5e0a8c2fca8e8", "squash eleventh piece-f8d483d24adc468d", "squash twelfth piece-6a525c58c3f1cd55", "squash thirteenth piece-649c7f35a92ab614", "squash fourteenth piece-72a65f7315e1e6b8", "squash fifteenth piece-c17b673e6cc5358f", "squash sixteenth piece-e078956436211ba8", "squash seventeenth piece-652bae6e1c3273f7", "squash eighteenth piece-23a69ab374b090a5", "squash nineteenth piece-7a55f7263deb348b", "squash twentieth piece-e1937bf0abd1b8a4", "squash twenty-first piece-a80dc0c5c7692223", "few chunks of squash-2a980da60ae1f48d", "more chunks of squash-b55258824c020135", "last chunks of squash-6d0adf45129486b8", "remaining squash-8ba189bfb842a210", "squash last piece-0fbdf69f3bb5121c", "squash label-b275e3e17336d694", "squash twenty-second piece-d7fdafa680deb28a", "squash twenth-third piece-65cc8582de3dd261", "squash twenty-fourth piece-74e443355339d754", "squash twenty-fifth piece-1e571e990857ed6f", "squash twenty-sixth piece-0219dd2912f3f647", "squash twenty-seventh piece-b5bd69b93489bdeb", "squash twenty-eighth piece-a8d23b7f67f87c4c", "peeled chunk of squash-b9123a6432d32df4", "peeled chunks of squash 2-57ae00949b9234a7", "few more peeled squash chunks-e58bcdce444d7de4", "few more peeled squash chunks 2-6a0922ed69a6eff0", "more peeled squash chunks-2e21fe2f58a3c7dd", "more peeled squash chunks 3-fef2c998c96d623c", "last bits of squash-cc7d20b0488f5bb9", "bag of lettuce-481dd3dd502eb163", "bunch of spring greens-6ef90806487cc132", "leaf of spring greens-82c25823cb6df600", "second leaf of spring green-6157b96202ddc216", "third leaf of spring green-a32cc06c7c69ec31", "fourth leaf of spring greens-ce46765de085652e", "fifth leaf of spring greens-6458b0ab9d019f25", "spring green leaves-8ba878ac39101f61", "sixth leaf of spring green-308f01ab292133ad", "strips of spring green leaves-efc517aed4e95109", "second large bowl-ea9b6dc08e9a7257", "can of coconut milk-16f046dadbc13f81", "box of cling film-40089125571e5dcf", "partly wrapped squash-a01241c053751ad6", "cling film-32dc53be2190afc1", "plastic bag of onions-5e6cdb503c76d4e3", "onion-6d70efcb6a560beb", "peeled half of onion-54aaecb23e7b5487", "second half of peeled onion-a2e290d4d04c1bbb", "stack of small frying pan-543c1020a503d69a", "frying pan grey-cacd8936c048acfb", "pan padding-4e551b32f278fe2f", "sunflower oil bottle-4141adf1f5c3e0ea", "partly chopped onion first half-bc0ddf57c5d4d86e", "plastic spatula-3b5bc3812ee99ade", "food waste bin-e04a4431ef2ec134", "peels-1ec6b4ae40d55983", "other vegetable peels-87e445b5e4a2b889", "new recipe card-2b32f55b2b537f5c", "recipe kit-84ae2ca3af7bf898", "recipe kit 2-0cf8da91d218cc58", "recipe kit 3-1b5b09077ccd79b1", "recipe kit box-c17f989efac06973", "teaspoon-3002a5fee10cab1f", "yellow plastic tub-85234776d45178d0", "inner part of plastic colander-62700200aa5bcb6d", "outer part of plastic colander-ae784b59e34414f5", "pink tub-62da6e3ddb49e8a4", "can opener-6798748b46b4bf7b", "last green plastic tub-131c5e91f81ba466", "glass-7019e116a8c3433f", "large glass lid-ef5d507a0d964784", "new track 2-d9cdd18e54fae096", "three plastic tubs-1941ddf33f716a85", "dishwasher brush-fef6398cb4354466", "v60 stand-ce24c492374d4d73", "v60 metal filter-6bb6a298e601a4c7", "coffee jug-99955b3e64ae51c0", "plastic food bag-710840db9ad365ed", "bread knife-13cf2382a2dcefbb", "black coffee spoon-33f368ccebba8f3d", "red plate-9a0a9378ec1991a1", "orange cloth-9550ea1395d8b999", "bit of plastic-e574dbaf80217f33", "cutlery stand-c4b5cf177f57dd97", "sink strainer-b4180c4533ecd01a", "rice box-89d873889fbc8de0", "long leaf plant-dc0cd2596d8d5c98", "other bowl-77ee35879fe06a66", "bread-923d61d123f0e8ec", "larger part of bread loaf-658ce2fc8929474d", "bread wrapping-4079edb10a4a41be", "ladle-968795cabd8dc591", "other spoon-9c25ab6fcf4127d4", "one fork-a08bbd8820f753a6", "third spoon-cf952ecf5957cc4f" ], "reasoning": "1. I watched the whole video and saw the user at 15:18 start picking the butternut squash cubes (peeled chunks of squash 2-57ae00949b9234a7) with their hands.\n2. The user picked up the squash cubes 3 more times - at 15:25 (few more peeled squash chunks-e58bcdce444d7de4), at 15:28 (few more peeled squash chunks 2-6a0922ed69a6eff0) and at 15:29 (more peeled squash chunks-2e21fe2f58a3c7dd).\n3. So, the user picked up the squash cubes a total of 4 times.\n4. The user broke off a leaf of lettuce at 16:13 (leaf of spring greens-82c25823cb6df600), 16:17 (second leaf of spring green-6157b96202ddc216), 16:20 (third leaf of spring green-a32cc06c7c69ec31), 16:31 (fourth leaf of spring greens-ce46765de085652e), 16:37 (fifth leaf of spring greens-6458b0ab9d019f25) and 16:44 (sixth leaf of spring green-308f01ab292133ad) for a total of 6 leaves.\n5. At 0:11, the user folded a small, white towel (folded white cloth-6456acd187f9a628) and placed on top of a Tupperware container at 0:14 for a total of 3 seconds.\n6. (4*6)^3 is 13,824.", "referenced_object_masks": [ "fifth leaf of spring greens-6458b0ab9d019f25", "second leaf of spring green-6157b96202ddc216", "fourth leaf of spring greens-ce46765de085652e", "few more peeled squash chunks 2-6a0922ed69a6eff0", "few more peeled squash chunks-e58bcdce444d7de4", "folded white cloth-6456acd187f9a628", "leaf of spring greens-82c25823cb6df600", "more peeled squash chunks-2e21fe2f58a3c7dd", "third leaf of spring green-a32cc06c7c69ec31", "sixth leaf of spring green-308f01ab292133ad", "peeled chunks of squash 2-57ae00949b9234a7" ] }, { "key": "P03-20240217-131219:54ebddc5d2408f1a3fa75012b11d37dd31d80d4b", "video_id": "P03-20240217-131219", "question": "What is the third thing I touch after I finish cutting up the onion?", "answer_choice_0": "White Towel", "answer_choice_1": "Bowl of Butternut Squash", "answer_choice_2": "Bowl of Lentils", "answer_choice_3": "Recipe Card", "answer_choice_4": "Scissors", "answer_id": 2, "answer": "Bowl of Lentils", "question_type": [ "Action Recognition", "Counting", "Event Occurence", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "right fork-9e719de8129aabab", "left fork-787a051fdb7b62b0", "recipe card-9d6cea18cbad9538", "folded white cloth-6456acd187f9a628", "pack of lentils-f62cf03618da91cd", "small knife-2ef4cd3b6b62247a", "scale-40685c5361cef4b5", "plastic colander-789c0c8f1379eed4", "scissors-7b21ea63ea07beb1", "seal tap-bc05cc4f3c7fb8c9", "cup-b560db3403d371bb", "cloth-7882b5a9a2352ec9", "chopping board-cb59981e62d4365d", "squash-6982230d527110b3", "big knife-d2a0f8a627e9277f", "one fourth left side squash-ce98c9ca9f000394", "right side squash-b9022d18f19c8310", "squash fourth piece-5d7f073b3c81689d", "sponge-1e546335a9c90260", "squash fifth piece-f24765fb6495a8a2", "squash sixth piece-c2371a92b3413242", "squash seventh piece-3d117a5ad3b43588", "squash eighth piece-8de19e02846c4548", "squash ninth piece-37e42be4b32e5985", "quash tenth piece-5d5d80e1cc678ffc", "large bowl-d4a9d64b9a7935bd", "peeled chunks of squash-f8f5e0a8c2fca8e8", "squash eleventh piece-f8d483d24adc468d", "squash twelfth piece-6a525c58c3f1cd55", "squash thirteenth piece-649c7f35a92ab614", "squash fourteenth piece-72a65f7315e1e6b8", "squash fifteenth piece-c17b673e6cc5358f", "squash sixteenth piece-e078956436211ba8", "squash seventeenth piece-652bae6e1c3273f7", "squash eighteenth piece-23a69ab374b090a5", "squash nineteenth piece-7a55f7263deb348b", "squash twentieth piece-e1937bf0abd1b8a4", "squash twenty-first piece-a80dc0c5c7692223", "few chunks of squash-2a980da60ae1f48d", "more chunks of squash-b55258824c020135", "last chunks of squash-6d0adf45129486b8", "remaining squash-8ba189bfb842a210", "squash last piece-0fbdf69f3bb5121c", "squash label-b275e3e17336d694", "squash twenty-second piece-d7fdafa680deb28a", "squash twenth-third piece-65cc8582de3dd261", "squash twenty-fourth piece-74e443355339d754", "squash twenty-fifth piece-1e571e990857ed6f", "squash twenty-sixth piece-0219dd2912f3f647", "squash twenty-seventh piece-b5bd69b93489bdeb", "squash twenty-eighth piece-a8d23b7f67f87c4c", "peeled chunk of squash-b9123a6432d32df4", "peeled chunks of squash 2-57ae00949b9234a7", "few more peeled squash chunks-e58bcdce444d7de4", "few more peeled squash chunks 2-6a0922ed69a6eff0", "more peeled squash chunks-2e21fe2f58a3c7dd", "more peeled squash chunks 3-fef2c998c96d623c", "last bits of squash-cc7d20b0488f5bb9", "bag of lettuce-481dd3dd502eb163", "bunch of spring greens-6ef90806487cc132", "leaf of spring greens-82c25823cb6df600", "second leaf of spring green-6157b96202ddc216", "third leaf of spring green-a32cc06c7c69ec31", "fourth leaf of spring greens-ce46765de085652e", "fifth leaf of spring greens-6458b0ab9d019f25", "spring green leaves-8ba878ac39101f61", "sixth leaf of spring green-308f01ab292133ad", "strips of spring green leaves-efc517aed4e95109", "second large bowl-ea9b6dc08e9a7257", "can of coconut milk-16f046dadbc13f81", "box of cling film-40089125571e5dcf", "partly wrapped squash-a01241c053751ad6", "cling film-32dc53be2190afc1", "plastic bag of onions-5e6cdb503c76d4e3", "onion-6d70efcb6a560beb", "peeled half of onion-54aaecb23e7b5487", "second half of peeled onion-a2e290d4d04c1bbb", "stack of small frying pan-543c1020a503d69a", "frying pan grey-cacd8936c048acfb", "pan padding-4e551b32f278fe2f", "sunflower oil bottle-4141adf1f5c3e0ea", "partly chopped onion first half-bc0ddf57c5d4d86e", "plastic spatula-3b5bc3812ee99ade", "food waste bin-e04a4431ef2ec134", "peels-1ec6b4ae40d55983", "other vegetable peels-87e445b5e4a2b889", "new recipe card-2b32f55b2b537f5c", "recipe kit-84ae2ca3af7bf898", "recipe kit 2-0cf8da91d218cc58", "recipe kit 3-1b5b09077ccd79b1", "recipe kit box-c17f989efac06973", "teaspoon-3002a5fee10cab1f", "yellow plastic tub-85234776d45178d0", "inner part of plastic colander-62700200aa5bcb6d", "outer part of plastic colander-ae784b59e34414f5", "pink tub-62da6e3ddb49e8a4", "can opener-6798748b46b4bf7b", "last green plastic tub-131c5e91f81ba466", "glass-7019e116a8c3433f", "large glass lid-ef5d507a0d964784", "new track 2-d9cdd18e54fae096", "three plastic tubs-1941ddf33f716a85", "dishwasher brush-fef6398cb4354466", "v60 stand-ce24c492374d4d73", "v60 metal filter-6bb6a298e601a4c7", "coffee jug-99955b3e64ae51c0", "plastic food bag-710840db9ad365ed", "bread knife-13cf2382a2dcefbb", "black coffee spoon-33f368ccebba8f3d", "red plate-9a0a9378ec1991a1", "orange cloth-9550ea1395d8b999", "bit of plastic-e574dbaf80217f33", "cutlery stand-c4b5cf177f57dd97", "sink strainer-b4180c4533ecd01a", "rice box-89d873889fbc8de0", "long leaf plant-dc0cd2596d8d5c98", "other bowl-77ee35879fe06a66", "bread-923d61d123f0e8ec", "larger part of bread loaf-658ce2fc8929474d", "bread wrapping-4079edb10a4a41be", "ladle-968795cabd8dc591", "other spoon-9c25ab6fcf4127d4", "one fork-a08bbd8820f753a6", "third spoon-cf952ecf5957cc4f" ], "reasoning": "1. I watched the whole video and saw the user finish cutting up the onion (onion-6d70efcb6a560beb) at 21:06.\n2. The first thing the user touches after cutting up the onion (onion-6d70efcb6a560beb) is the white towel (folded white cloth-6456acd187f9a628) at 21:07.\n3. The second thing the user touches is the recipe card (recipe card-9d6cea18cbad9538) at 21:17.\n3. The third thing the user touches is the bowl of lentils at 21:28.", "referenced_object_masks": [ "recipe card-9d6cea18cbad9538", "folded white cloth-6456acd187f9a628", "onion-6d70efcb6a560beb" ] }, { "key": "P03-20240217-131219:681380ff124b37ee319b0b996f91a1f331192be1", "video_id": "P03-20240217-131219", "question": "What was the third item I grabbed from a cabinet?", "answer_choice_0": "A red pair of scissors", "answer_choice_1": "A white weight scale", "answer_choice_2": "A white recipe card", "answer_choice_3": "A red pack of lentils", "answer_choice_4": "A white plastic colander", "answer_id": 4, "answer": "A white plastic colander", "question_type": [ "Counting", "Action Recognition", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "right fork-9e719de8129aabab", "left fork-787a051fdb7b62b0", "recipe card-9d6cea18cbad9538", "folded white cloth-6456acd187f9a628", "pack of lentils-f62cf03618da91cd", "small knife-2ef4cd3b6b62247a", "scale-40685c5361cef4b5", "plastic colander-789c0c8f1379eed4", "scissors-7b21ea63ea07beb1", "seal tap-bc05cc4f3c7fb8c9", "cup-b560db3403d371bb", "cloth-7882b5a9a2352ec9", "chopping board-cb59981e62d4365d", "squash-6982230d527110b3", "big knife-d2a0f8a627e9277f", "one fourth left side squash-ce98c9ca9f000394", "right side squash-b9022d18f19c8310", "squash fourth piece-5d7f073b3c81689d", "sponge-1e546335a9c90260", "squash fifth piece-f24765fb6495a8a2", "squash sixth piece-c2371a92b3413242", "squash seventh piece-3d117a5ad3b43588", "squash eighth piece-8de19e02846c4548", "squash ninth piece-37e42be4b32e5985", "quash tenth piece-5d5d80e1cc678ffc", "large bowl-d4a9d64b9a7935bd", "peeled chunks of squash-f8f5e0a8c2fca8e8", "squash eleventh piece-f8d483d24adc468d", "squash twelfth piece-6a525c58c3f1cd55", "squash thirteenth piece-649c7f35a92ab614", "squash fourteenth piece-72a65f7315e1e6b8", "squash fifteenth piece-c17b673e6cc5358f", "squash sixteenth piece-e078956436211ba8", "squash seventeenth piece-652bae6e1c3273f7", "squash eighteenth piece-23a69ab374b090a5", "squash nineteenth piece-7a55f7263deb348b", "squash twentieth piece-e1937bf0abd1b8a4", "squash twenty-first piece-a80dc0c5c7692223", "few chunks of squash-2a980da60ae1f48d", "more chunks of squash-b55258824c020135", "last chunks of squash-6d0adf45129486b8", "remaining squash-8ba189bfb842a210", "squash last piece-0fbdf69f3bb5121c", "squash label-b275e3e17336d694", "squash twenty-second piece-d7fdafa680deb28a", "squash twenth-third piece-65cc8582de3dd261", "squash twenty-fourth piece-74e443355339d754", "squash twenty-fifth piece-1e571e990857ed6f", "squash twenty-sixth piece-0219dd2912f3f647", "squash twenty-seventh piece-b5bd69b93489bdeb", "squash twenty-eighth piece-a8d23b7f67f87c4c", "peeled chunk of squash-b9123a6432d32df4", "peeled chunks of squash 2-57ae00949b9234a7", "few more peeled squash chunks-e58bcdce444d7de4", "few more peeled squash chunks 2-6a0922ed69a6eff0", "more peeled squash chunks-2e21fe2f58a3c7dd", "more peeled squash chunks 3-fef2c998c96d623c", "last bits of squash-cc7d20b0488f5bb9", "bag of lettuce-481dd3dd502eb163", "bunch of spring greens-6ef90806487cc132", "leaf of spring greens-82c25823cb6df600", "second leaf of spring green-6157b96202ddc216", "third leaf of spring green-a32cc06c7c69ec31", "fourth leaf of spring greens-ce46765de085652e", "fifth leaf of spring greens-6458b0ab9d019f25", "spring green leaves-8ba878ac39101f61", "sixth leaf of spring green-308f01ab292133ad", "strips of spring green leaves-efc517aed4e95109", "second large bowl-ea9b6dc08e9a7257", "can of coconut milk-16f046dadbc13f81", "box of cling film-40089125571e5dcf", "partly wrapped squash-a01241c053751ad6", "cling film-32dc53be2190afc1", "plastic bag of onions-5e6cdb503c76d4e3", "onion-6d70efcb6a560beb", "peeled half of onion-54aaecb23e7b5487", "second half of peeled onion-a2e290d4d04c1bbb", "stack of small frying pan-543c1020a503d69a", "frying pan grey-cacd8936c048acfb", "pan padding-4e551b32f278fe2f", "sunflower oil bottle-4141adf1f5c3e0ea", "partly chopped onion first half-bc0ddf57c5d4d86e", "plastic spatula-3b5bc3812ee99ade", "food waste bin-e04a4431ef2ec134", "peels-1ec6b4ae40d55983", "other vegetable peels-87e445b5e4a2b889", "new recipe card-2b32f55b2b537f5c", "recipe kit-84ae2ca3af7bf898", "recipe kit 2-0cf8da91d218cc58", "recipe kit 3-1b5b09077ccd79b1", "recipe kit box-c17f989efac06973", "teaspoon-3002a5fee10cab1f", "yellow plastic tub-85234776d45178d0", "inner part of plastic colander-62700200aa5bcb6d", "outer part of plastic colander-ae784b59e34414f5", "pink tub-62da6e3ddb49e8a4", "can opener-6798748b46b4bf7b", "last green plastic tub-131c5e91f81ba466", "glass-7019e116a8c3433f", "large glass lid-ef5d507a0d964784", "new track 2-d9cdd18e54fae096", "three plastic tubs-1941ddf33f716a85", "dishwasher brush-fef6398cb4354466", "v60 stand-ce24c492374d4d73", "v60 metal filter-6bb6a298e601a4c7", "coffee jug-99955b3e64ae51c0", "plastic food bag-710840db9ad365ed", "bread knife-13cf2382a2dcefbb", "black coffee spoon-33f368ccebba8f3d", "red plate-9a0a9378ec1991a1", "orange cloth-9550ea1395d8b999", "bit of plastic-e574dbaf80217f33", "cutlery stand-c4b5cf177f57dd97", "sink strainer-b4180c4533ecd01a", "rice box-89d873889fbc8de0", "long leaf plant-dc0cd2596d8d5c98", "other bowl-77ee35879fe06a66", "bread-923d61d123f0e8ec", "larger part of bread loaf-658ce2fc8929474d", "bread wrapping-4079edb10a4a41be", "ladle-968795cabd8dc591", "other spoon-9c25ab6fcf4127d4", "one fork-a08bbd8820f753a6", "third spoon-cf952ecf5957cc4f" ], "reasoning": "1. I watched the video and counted each item the user grabbed from a cabinet.\n2. At 00:24, I saw the user grab the first item from a cabinet, which was a red pack of lentils (pack of lentils-f62cf03618da91cd).\n3. At 00:56, I saw the user grab the second item from another cabinet, which was a scale (scale-40685c5361cef4b5).\n4. At 00:58, I saw the user grab the third item from the same cabinet as the second item, which was a plastic colander (plastic colander-789c0c8f1379eed4).\n5. Therefore, the third item the user grabbed from a cabinet was a plastic colander (plastic colander-789c0c8f1379eed4).", "referenced_object_masks": [ "plastic colander-789c0c8f1379eed4", "pack of lentils-f62cf03618da91cd", "scale-40685c5361cef4b5" ] }, { "key": "P03-20240217-131219:79a7d593916865859f3bbfa9fda7a27f5368d45a", "video_id": "P03-20240217-131219", "question": "What is the ratio of the number of times I hit the sink strainer on the side of the food waste bin to the number of times I hit the spatula on the edge of the pan at 50:45? If needed, reduce the ratio into lowest terms.", "answer_choice_0": "11:4", "answer_choice_1": "3:1", "answer_choice_2": "5:2", "answer_choice_3": "21:8", "answer_choice_4": "4:1", "answer_id": 0, "answer": "11:4", "question_type": [ "Action Recognition", "Counting", "Event Occurence", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "right fork-9e719de8129aabab", "left fork-787a051fdb7b62b0", "recipe card-9d6cea18cbad9538", "folded white cloth-6456acd187f9a628", "pack of lentils-f62cf03618da91cd", "small knife-2ef4cd3b6b62247a", "scale-40685c5361cef4b5", "plastic colander-789c0c8f1379eed4", "scissors-7b21ea63ea07beb1", "seal tap-bc05cc4f3c7fb8c9", "cup-b560db3403d371bb", "cloth-7882b5a9a2352ec9", "chopping board-cb59981e62d4365d", "squash-6982230d527110b3", "big knife-d2a0f8a627e9277f", "one fourth left side squash-ce98c9ca9f000394", "right side squash-b9022d18f19c8310", "squash fourth piece-5d7f073b3c81689d", "sponge-1e546335a9c90260", "squash fifth piece-f24765fb6495a8a2", "squash sixth piece-c2371a92b3413242", "squash seventh piece-3d117a5ad3b43588", "squash eighth piece-8de19e02846c4548", "squash ninth piece-37e42be4b32e5985", "quash tenth piece-5d5d80e1cc678ffc", "large bowl-d4a9d64b9a7935bd", "peeled chunks of squash-f8f5e0a8c2fca8e8", "squash eleventh piece-f8d483d24adc468d", "squash twelfth piece-6a525c58c3f1cd55", "squash thirteenth piece-649c7f35a92ab614", "squash fourteenth piece-72a65f7315e1e6b8", "squash fifteenth piece-c17b673e6cc5358f", "squash sixteenth piece-e078956436211ba8", "squash seventeenth piece-652bae6e1c3273f7", "squash eighteenth piece-23a69ab374b090a5", "squash nineteenth piece-7a55f7263deb348b", "squash twentieth piece-e1937bf0abd1b8a4", "squash twenty-first piece-a80dc0c5c7692223", "few chunks of squash-2a980da60ae1f48d", "more chunks of squash-b55258824c020135", "last chunks of squash-6d0adf45129486b8", "remaining squash-8ba189bfb842a210", "squash last piece-0fbdf69f3bb5121c", "squash label-b275e3e17336d694", "squash twenty-second piece-d7fdafa680deb28a", "squash twenth-third piece-65cc8582de3dd261", "squash twenty-fourth piece-74e443355339d754", "squash twenty-fifth piece-1e571e990857ed6f", "squash twenty-sixth piece-0219dd2912f3f647", "squash twenty-seventh piece-b5bd69b93489bdeb", "squash twenty-eighth piece-a8d23b7f67f87c4c", "peeled chunk of squash-b9123a6432d32df4", "peeled chunks of squash 2-57ae00949b9234a7", "few more peeled squash chunks-e58bcdce444d7de4", "few more peeled squash chunks 2-6a0922ed69a6eff0", "more peeled squash chunks-2e21fe2f58a3c7dd", "more peeled squash chunks 3-fef2c998c96d623c", "last bits of squash-cc7d20b0488f5bb9", "bag of lettuce-481dd3dd502eb163", "bunch of spring greens-6ef90806487cc132", "leaf of spring greens-82c25823cb6df600", "second leaf of spring green-6157b96202ddc216", "third leaf of spring green-a32cc06c7c69ec31", "fourth leaf of spring greens-ce46765de085652e", "fifth leaf of spring greens-6458b0ab9d019f25", "spring green leaves-8ba878ac39101f61", "sixth leaf of spring green-308f01ab292133ad", "strips of spring green leaves-efc517aed4e95109", "second large bowl-ea9b6dc08e9a7257", "can of coconut milk-16f046dadbc13f81", "box of cling film-40089125571e5dcf", "partly wrapped squash-a01241c053751ad6", "cling film-32dc53be2190afc1", "plastic bag of onions-5e6cdb503c76d4e3", "onion-6d70efcb6a560beb", "peeled half of onion-54aaecb23e7b5487", "second half of peeled onion-a2e290d4d04c1bbb", "stack of small frying pan-543c1020a503d69a", "frying pan grey-cacd8936c048acfb", "pan padding-4e551b32f278fe2f", "sunflower oil bottle-4141adf1f5c3e0ea", "partly chopped onion first half-bc0ddf57c5d4d86e", "plastic spatula-3b5bc3812ee99ade", "food waste bin-e04a4431ef2ec134", "peels-1ec6b4ae40d55983", "other vegetable peels-87e445b5e4a2b889", "new recipe card-2b32f55b2b537f5c", "recipe kit-84ae2ca3af7bf898", "recipe kit 2-0cf8da91d218cc58", "recipe kit 3-1b5b09077ccd79b1", "recipe kit box-c17f989efac06973", "teaspoon-3002a5fee10cab1f", "yellow plastic tub-85234776d45178d0", "inner part of plastic colander-62700200aa5bcb6d", "outer part of plastic colander-ae784b59e34414f5", "pink tub-62da6e3ddb49e8a4", "can opener-6798748b46b4bf7b", "last green plastic tub-131c5e91f81ba466", "glass-7019e116a8c3433f", "large glass lid-ef5d507a0d964784", "new track 2-d9cdd18e54fae096", "three plastic tubs-1941ddf33f716a85", "dishwasher brush-fef6398cb4354466", "v60 stand-ce24c492374d4d73", "v60 metal filter-6bb6a298e601a4c7", "coffee jug-99955b3e64ae51c0", "plastic food bag-710840db9ad365ed", "bread knife-13cf2382a2dcefbb", "black coffee spoon-33f368ccebba8f3d", "red plate-9a0a9378ec1991a1", "orange cloth-9550ea1395d8b999", "bit of plastic-e574dbaf80217f33", "cutlery stand-c4b5cf177f57dd97", "sink strainer-b4180c4533ecd01a", "rice box-89d873889fbc8de0", "long leaf plant-dc0cd2596d8d5c98", "other bowl-77ee35879fe06a66", "bread-923d61d123f0e8ec", "larger part of bread loaf-658ce2fc8929474d", "bread wrapping-4079edb10a4a41be", "ladle-968795cabd8dc591", "other spoon-9c25ab6fcf4127d4", "one fork-a08bbd8820f753a6", "third spoon-cf952ecf5957cc4f" ], "reasoning": "1. I watched the whole video and saw the user hit the sink strainer (sink strainer-b4180c4533ecd01a) on the side of the food waste bin (food waste bin-e04a4431ef2ec134) at 44:52. The user did this 22 times.\n2. At 50:45, I saw the user hit the spatula (plastic spatula-3b5bc3812ee99ade) on the edge of the pan (frying pan grey-cacd8936c048acfb) 8 times.\n3. The ratio of the number of times the user hit the sink strainer (sink strainer-b4180c4533ecd01a) on the side of the food waste bin (food waste bin-e04a4431ef2ec134) to the number of times the user hit the spatula (plastic spatula-3b5bc3812ee99ade)on the side of the pan (frying pan grey-cacd8936c048acfb) is 22:8.\n4. This ratio can be reduced by 2 to 11:4.", "referenced_object_masks": [ "food waste bin-e04a4431ef2ec134", "frying pan grey-cacd8936c048acfb", "plastic spatula-3b5bc3812ee99ade", "sink strainer-b4180c4533ecd01a" ] }, { "key": "P03-20240217-131219:ad2c0f2c30ecc85f4c96e34b588fc0fef79e146e", "video_id": "P03-20240217-131219", "question": "What is the 2nd action I take after pouring the red pack of lentils?", "answer_choice_0": "The user picked up and read the recipe card", "answer_choice_1": "The user washed and drained the colander's lentils", "answer_choice_2": "The user rinsed and dried a knife by hand and towel", "answer_choice_3": "The user grabbed a squash and cut it with a knife", "answer_choice_4": "The user sealed and put away the pack of lentils", "answer_id": 1, "answer": "The user washed and drained the colander's lentils", "question_type": [ "Object Recognition", "Action Recognition", "Temporal Reasoning", "Situational Awareness", "State Changes" ], "objects_of_interest": [ "right fork-9e719de8129aabab", "left fork-787a051fdb7b62b0", "recipe card-9d6cea18cbad9538", "folded white cloth-6456acd187f9a628", "pack of lentils-f62cf03618da91cd", "small knife-2ef4cd3b6b62247a", "scale-40685c5361cef4b5", "plastic colander-789c0c8f1379eed4", "scissors-7b21ea63ea07beb1", "seal tap-bc05cc4f3c7fb8c9", "cup-b560db3403d371bb", "cloth-7882b5a9a2352ec9", "chopping board-cb59981e62d4365d", "squash-6982230d527110b3", "big knife-d2a0f8a627e9277f", "one fourth left side squash-ce98c9ca9f000394", "right side squash-b9022d18f19c8310", "squash fourth piece-5d7f073b3c81689d", "sponge-1e546335a9c90260", "squash fifth piece-f24765fb6495a8a2", "squash sixth piece-c2371a92b3413242", "squash seventh piece-3d117a5ad3b43588", "squash eighth piece-8de19e02846c4548", "squash ninth piece-37e42be4b32e5985", "quash tenth piece-5d5d80e1cc678ffc", "large bowl-d4a9d64b9a7935bd", "peeled chunks of squash-f8f5e0a8c2fca8e8", "squash eleventh piece-f8d483d24adc468d", "squash twelfth piece-6a525c58c3f1cd55", "squash thirteenth piece-649c7f35a92ab614", "squash fourteenth piece-72a65f7315e1e6b8", "squash fifteenth piece-c17b673e6cc5358f", "squash sixteenth piece-e078956436211ba8", "squash seventeenth piece-652bae6e1c3273f7", "squash eighteenth piece-23a69ab374b090a5", "squash nineteenth piece-7a55f7263deb348b", "squash twentieth piece-e1937bf0abd1b8a4", "squash twenty-first piece-a80dc0c5c7692223", "few chunks of squash-2a980da60ae1f48d", "more chunks of squash-b55258824c020135", "last chunks of squash-6d0adf45129486b8", "remaining squash-8ba189bfb842a210", "squash last piece-0fbdf69f3bb5121c", "squash label-b275e3e17336d694", "squash twenty-second piece-d7fdafa680deb28a", "squash twenth-third piece-65cc8582de3dd261", "squash twenty-fourth piece-74e443355339d754", "squash twenty-fifth piece-1e571e990857ed6f", "squash twenty-sixth piece-0219dd2912f3f647", "squash twenty-seventh piece-b5bd69b93489bdeb", "squash twenty-eighth piece-a8d23b7f67f87c4c", "peeled chunk of squash-b9123a6432d32df4", "peeled chunks of squash 2-57ae00949b9234a7", "few more peeled squash chunks-e58bcdce444d7de4", "few more peeled squash chunks 2-6a0922ed69a6eff0", "more peeled squash chunks-2e21fe2f58a3c7dd", "more peeled squash chunks 3-fef2c998c96d623c", "last bits of squash-cc7d20b0488f5bb9", "bag of lettuce-481dd3dd502eb163", "bunch of spring greens-6ef90806487cc132", "leaf of spring greens-82c25823cb6df600", "second leaf of spring green-6157b96202ddc216", "third leaf of spring green-a32cc06c7c69ec31", "fourth leaf of spring greens-ce46765de085652e", "fifth leaf of spring greens-6458b0ab9d019f25", "spring green leaves-8ba878ac39101f61", "sixth leaf of spring green-308f01ab292133ad", "strips of spring green leaves-efc517aed4e95109", "second large bowl-ea9b6dc08e9a7257", "can of coconut milk-16f046dadbc13f81", "box of cling film-40089125571e5dcf", "partly wrapped squash-a01241c053751ad6", "cling film-32dc53be2190afc1", "plastic bag of onions-5e6cdb503c76d4e3", "onion-6d70efcb6a560beb", "peeled half of onion-54aaecb23e7b5487", "second half of peeled onion-a2e290d4d04c1bbb", "stack of small frying pan-543c1020a503d69a", "frying pan grey-cacd8936c048acfb", "pan padding-4e551b32f278fe2f", "sunflower oil bottle-4141adf1f5c3e0ea", "partly chopped onion first half-bc0ddf57c5d4d86e", "plastic spatula-3b5bc3812ee99ade", "food waste bin-e04a4431ef2ec134", "peels-1ec6b4ae40d55983", "other vegetable peels-87e445b5e4a2b889", "new recipe card-2b32f55b2b537f5c", "recipe kit-84ae2ca3af7bf898", "recipe kit 2-0cf8da91d218cc58", "recipe kit 3-1b5b09077ccd79b1", "recipe kit box-c17f989efac06973", "teaspoon-3002a5fee10cab1f", "yellow plastic tub-85234776d45178d0", "inner part of plastic colander-62700200aa5bcb6d", "outer part of plastic colander-ae784b59e34414f5", "pink tub-62da6e3ddb49e8a4", "can opener-6798748b46b4bf7b", "last green plastic tub-131c5e91f81ba466", "glass-7019e116a8c3433f", "large glass lid-ef5d507a0d964784", "new track 2-d9cdd18e54fae096", "three plastic tubs-1941ddf33f716a85", "dishwasher brush-fef6398cb4354466", "v60 stand-ce24c492374d4d73", "v60 metal filter-6bb6a298e601a4c7", "coffee jug-99955b3e64ae51c0", "plastic food bag-710840db9ad365ed", "bread knife-13cf2382a2dcefbb", "black coffee spoon-33f368ccebba8f3d", "red plate-9a0a9378ec1991a1", "orange cloth-9550ea1395d8b999", "bit of plastic-e574dbaf80217f33", "cutlery stand-c4b5cf177f57dd97", "sink strainer-b4180c4533ecd01a", "rice box-89d873889fbc8de0", "long leaf plant-dc0cd2596d8d5c98", "other bowl-77ee35879fe06a66", "bread-923d61d123f0e8ec", "larger part of bread loaf-658ce2fc8929474d", "bread wrapping-4079edb10a4a41be", "ladle-968795cabd8dc591", "other spoon-9c25ab6fcf4127d4", "one fork-a08bbd8820f753a6", "third spoon-cf952ecf5957cc4f" ], "reasoning": "1. I watched the video and found the red pack of lentils (pack of lentils-f62cf03618da91cd) at 00:24.\n2. At 01:38, I saw the user open the pack of lentils (pack of lentils-f62cf03618da91cd) with scissors (scissors-7b21ea63ea07beb1).\n3. I saw the user pour the pack of lentils (pack of lentils-f62cf03618da91cd) into the plastic colander (plastic colander-789c0c8f1379eed4) from approximately 01:44-02:10.\n4. From 02:10-02:25, I saw the user seal and put away the pack of lentils (pack of lentils-f62cf03618da91cd). I counted this as the 1st action after pouring the pack of lentils (pack of lentils-f62cf03618da91cd).\n5. Then, from 02:26-02:50, I saw the user pick up the plastic colander (plastic colander-789c0c8f1379eed4) with lentils and then place it under the faucet, where they washed and drained it. I counted this as the 2nd action after pouring the pack of lentils (pack of lentils-f62cf03618da91cd), and thus, the final answer.", "referenced_object_masks": [ "plastic colander-789c0c8f1379eed4", "pack of lentils-f62cf03618da91cd", "scissors-7b21ea63ea07beb1" ] }, { "key": "P03-20240217-131219:c40b19e8d8f9d6ffd1fa458e36443ccb7984f008", "video_id": "P03-20240217-131219", "question": "Where was the recipe card in relation to me when I washed the lentils in the colander?", "answer_choice_0": "Directly overhead", "answer_choice_1": "Behind and to the right", "answer_choice_2": "In front and to the right", "answer_choice_3": "Behind and to the left", "answer_choice_4": "In front and to the left", "answer_id": 3, "answer": "Behind and to the left", "question_type": [ "Object Recognition", "Action Recognition", "Egocentric & Spatial Perception", "Temporal Reasoning", "Situational Awareness" ], "objects_of_interest": [ "right fork-9e719de8129aabab", "left fork-787a051fdb7b62b0", "recipe card-9d6cea18cbad9538", "folded white cloth-6456acd187f9a628", "pack of lentils-f62cf03618da91cd", "small knife-2ef4cd3b6b62247a", "scale-40685c5361cef4b5", "plastic colander-789c0c8f1379eed4", "scissors-7b21ea63ea07beb1", "seal tap-bc05cc4f3c7fb8c9", "cup-b560db3403d371bb", "cloth-7882b5a9a2352ec9", "chopping board-cb59981e62d4365d", "squash-6982230d527110b3", "big knife-d2a0f8a627e9277f", "one fourth left side squash-ce98c9ca9f000394", "right side squash-b9022d18f19c8310", "squash fourth piece-5d7f073b3c81689d", "sponge-1e546335a9c90260", "squash fifth piece-f24765fb6495a8a2", "squash sixth piece-c2371a92b3413242", "squash seventh piece-3d117a5ad3b43588", "squash eighth piece-8de19e02846c4548", "squash ninth piece-37e42be4b32e5985", "quash tenth piece-5d5d80e1cc678ffc", "large bowl-d4a9d64b9a7935bd", "peeled chunks of squash-f8f5e0a8c2fca8e8", "squash eleventh piece-f8d483d24adc468d", "squash twelfth piece-6a525c58c3f1cd55", "squash thirteenth piece-649c7f35a92ab614", "squash fourteenth piece-72a65f7315e1e6b8", "squash fifteenth piece-c17b673e6cc5358f", "squash sixteenth piece-e078956436211ba8", "squash seventeenth piece-652bae6e1c3273f7", "squash eighteenth piece-23a69ab374b090a5", "squash nineteenth piece-7a55f7263deb348b", "squash twentieth piece-e1937bf0abd1b8a4", "squash twenty-first piece-a80dc0c5c7692223", "few chunks of squash-2a980da60ae1f48d", "more chunks of squash-b55258824c020135", "last chunks of squash-6d0adf45129486b8", "remaining squash-8ba189bfb842a210", "squash last piece-0fbdf69f3bb5121c", "squash label-b275e3e17336d694", "squash twenty-second piece-d7fdafa680deb28a", "squash twenth-third piece-65cc8582de3dd261", "squash twenty-fourth piece-74e443355339d754", "squash twenty-fifth piece-1e571e990857ed6f", "squash twenty-sixth piece-0219dd2912f3f647", "squash twenty-seventh piece-b5bd69b93489bdeb", "squash twenty-eighth piece-a8d23b7f67f87c4c", "peeled chunk of squash-b9123a6432d32df4", "peeled chunks of squash 2-57ae00949b9234a7", "few more peeled squash chunks-e58bcdce444d7de4", "few more peeled squash chunks 2-6a0922ed69a6eff0", "more peeled squash chunks-2e21fe2f58a3c7dd", "more peeled squash chunks 3-fef2c998c96d623c", "last bits of squash-cc7d20b0488f5bb9", "bag of lettuce-481dd3dd502eb163", "bunch of spring greens-6ef90806487cc132", "leaf of spring greens-82c25823cb6df600", "second leaf of spring green-6157b96202ddc216", "third leaf of spring green-a32cc06c7c69ec31", "fourth leaf of spring greens-ce46765de085652e", "fifth leaf of spring greens-6458b0ab9d019f25", "spring green leaves-8ba878ac39101f61", "sixth leaf of spring green-308f01ab292133ad", "strips of spring green leaves-efc517aed4e95109", "second large bowl-ea9b6dc08e9a7257", "can of coconut milk-16f046dadbc13f81", "box of cling film-40089125571e5dcf", "partly wrapped squash-a01241c053751ad6", "cling film-32dc53be2190afc1", "plastic bag of onions-5e6cdb503c76d4e3", "onion-6d70efcb6a560beb", "peeled half of onion-54aaecb23e7b5487", "second half of peeled onion-a2e290d4d04c1bbb", "stack of small frying pan-543c1020a503d69a", "frying pan grey-cacd8936c048acfb", "pan padding-4e551b32f278fe2f", "sunflower oil bottle-4141adf1f5c3e0ea", "partly chopped onion first half-bc0ddf57c5d4d86e", "plastic spatula-3b5bc3812ee99ade", "food waste bin-e04a4431ef2ec134", "peels-1ec6b4ae40d55983", "other vegetable peels-87e445b5e4a2b889", "new recipe card-2b32f55b2b537f5c", "recipe kit-84ae2ca3af7bf898", "recipe kit 2-0cf8da91d218cc58", "recipe kit 3-1b5b09077ccd79b1", "recipe kit box-c17f989efac06973", "teaspoon-3002a5fee10cab1f", "yellow plastic tub-85234776d45178d0", "inner part of plastic colander-62700200aa5bcb6d", "outer part of plastic colander-ae784b59e34414f5", "pink tub-62da6e3ddb49e8a4", "can opener-6798748b46b4bf7b", "last green plastic tub-131c5e91f81ba466", "glass-7019e116a8c3433f", "large glass lid-ef5d507a0d964784", "new track 2-d9cdd18e54fae096", "three plastic tubs-1941ddf33f716a85", "dishwasher brush-fef6398cb4354466", "v60 stand-ce24c492374d4d73", "v60 metal filter-6bb6a298e601a4c7", "coffee jug-99955b3e64ae51c0", "plastic food bag-710840db9ad365ed", "bread knife-13cf2382a2dcefbb", "black coffee spoon-33f368ccebba8f3d", "red plate-9a0a9378ec1991a1", "orange cloth-9550ea1395d8b999", "bit of plastic-e574dbaf80217f33", "cutlery stand-c4b5cf177f57dd97", "sink strainer-b4180c4533ecd01a", "rice box-89d873889fbc8de0", "long leaf plant-dc0cd2596d8d5c98", "other bowl-77ee35879fe06a66", "bread-923d61d123f0e8ec", "larger part of bread loaf-658ce2fc8929474d", "bread wrapping-4079edb10a4a41be", "ladle-968795cabd8dc591", "other spoon-9c25ab6fcf4127d4", "one fork-a08bbd8820f753a6", "third spoon-cf952ecf5957cc4f" ], "reasoning": "1. I watched the video and found the recipe card (recipe card-9d6cea18cbad9538) at 00:08.\n2. I found the red pack of lentils (pack of lentils-f62cf03618da91cd) at 00:24.\n3. I found the plastic colander (plastic colander-789c0c8f1379eed4) at 00:58.\n4. At 01:44, I saw the user pour the pack of lentils (pack of lentils-f62cf03618da91cd) into the plastic colander (plastic colander-789c0c8f1379eed4).\n5. I saw the user wash the lentils in the colander (plastic colander-789c0c8f1379eed4) at 02:30, but the recipe card (recipe card-9d6cea18cbad9538) is not visible.\n6. Before the user washed the lentils in the colander (plastic colander-789c0c8f1379eed4), at 02:27, I saw the recipe card (recipe card-9d6cea18cbad9538) as the user turned to the right, away from the recipe card (recipe card-9d6cea18cbad9538) and towards the sink, about to wash the lentils in the colander (plastic colander-789c0c8f1379eed4).\n7. Therefore, the recipe card (recipe card-9d6cea18cbad9538) was behind and to the left of the user when they washed the lentils in the colander (plastic colander-789c0c8f1379eed4).", "referenced_object_masks": [ "recipe card-9d6cea18cbad9538", "pack of lentils-f62cf03618da91cd", "plastic colander-789c0c8f1379eed4" ] }, { "key": "P03-20240217-131219:cc1ce4a049be83058e944d734997a4401aef0d1b", "video_id": "P03-20240217-131219", "question": "Which direction did I move after opening the red pack of lentils?", "answer_choice_0": "To the left", "answer_choice_1": "To the right", "answer_choice_2": "Upwards", "answer_choice_3": "Forwards", "answer_choice_4": "Backwards", "answer_id": 0, "answer": "To the left", "question_type": [ "Object Recognition", "Action Recognition", "State Changes", "Temporal Reasoning", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "right fork-9e719de8129aabab", "left fork-787a051fdb7b62b0", "recipe card-9d6cea18cbad9538", "folded white cloth-6456acd187f9a628", "pack of lentils-f62cf03618da91cd", "small knife-2ef4cd3b6b62247a", "scale-40685c5361cef4b5", "plastic colander-789c0c8f1379eed4", "scissors-7b21ea63ea07beb1", "seal tap-bc05cc4f3c7fb8c9", "cup-b560db3403d371bb", "cloth-7882b5a9a2352ec9", "chopping board-cb59981e62d4365d", "squash-6982230d527110b3", "big knife-d2a0f8a627e9277f", "one fourth left side squash-ce98c9ca9f000394", "right side squash-b9022d18f19c8310", "squash fourth piece-5d7f073b3c81689d", "sponge-1e546335a9c90260", "squash fifth piece-f24765fb6495a8a2", "squash sixth piece-c2371a92b3413242", "squash seventh piece-3d117a5ad3b43588", "squash eighth piece-8de19e02846c4548", "squash ninth piece-37e42be4b32e5985", "quash tenth piece-5d5d80e1cc678ffc", "large bowl-d4a9d64b9a7935bd", "peeled chunks of squash-f8f5e0a8c2fca8e8", "squash eleventh piece-f8d483d24adc468d", "squash twelfth piece-6a525c58c3f1cd55", "squash thirteenth piece-649c7f35a92ab614", "squash fourteenth piece-72a65f7315e1e6b8", "squash fifteenth piece-c17b673e6cc5358f", "squash sixteenth piece-e078956436211ba8", "squash seventeenth piece-652bae6e1c3273f7", "squash eighteenth piece-23a69ab374b090a5", "squash nineteenth piece-7a55f7263deb348b", "squash twentieth piece-e1937bf0abd1b8a4", "squash twenty-first piece-a80dc0c5c7692223", "few chunks of squash-2a980da60ae1f48d", "more chunks of squash-b55258824c020135", "last chunks of squash-6d0adf45129486b8", "remaining squash-8ba189bfb842a210", "squash last piece-0fbdf69f3bb5121c", "squash label-b275e3e17336d694", "squash twenty-second piece-d7fdafa680deb28a", "squash twenth-third piece-65cc8582de3dd261", "squash twenty-fourth piece-74e443355339d754", "squash twenty-fifth piece-1e571e990857ed6f", "squash twenty-sixth piece-0219dd2912f3f647", "squash twenty-seventh piece-b5bd69b93489bdeb", "squash twenty-eighth piece-a8d23b7f67f87c4c", "peeled chunk of squash-b9123a6432d32df4", "peeled chunks of squash 2-57ae00949b9234a7", "few more peeled squash chunks-e58bcdce444d7de4", "few more peeled squash chunks 2-6a0922ed69a6eff0", "more peeled squash chunks-2e21fe2f58a3c7dd", "more peeled squash chunks 3-fef2c998c96d623c", "last bits of squash-cc7d20b0488f5bb9", "bag of lettuce-481dd3dd502eb163", "bunch of spring greens-6ef90806487cc132", "leaf of spring greens-82c25823cb6df600", "second leaf of spring green-6157b96202ddc216", "third leaf of spring green-a32cc06c7c69ec31", "fourth leaf of spring greens-ce46765de085652e", "fifth leaf of spring greens-6458b0ab9d019f25", "spring green leaves-8ba878ac39101f61", "sixth leaf of spring green-308f01ab292133ad", "strips of spring green leaves-efc517aed4e95109", "second large bowl-ea9b6dc08e9a7257", "can of coconut milk-16f046dadbc13f81", "box of cling film-40089125571e5dcf", "partly wrapped squash-a01241c053751ad6", "cling film-32dc53be2190afc1", "plastic bag of onions-5e6cdb503c76d4e3", "onion-6d70efcb6a560beb", "peeled half of onion-54aaecb23e7b5487", "second half of peeled onion-a2e290d4d04c1bbb", "stack of small frying pan-543c1020a503d69a", "frying pan grey-cacd8936c048acfb", "pan padding-4e551b32f278fe2f", "sunflower oil bottle-4141adf1f5c3e0ea", "partly chopped onion first half-bc0ddf57c5d4d86e", "plastic spatula-3b5bc3812ee99ade", "food waste bin-e04a4431ef2ec134", "peels-1ec6b4ae40d55983", "other vegetable peels-87e445b5e4a2b889", "new recipe card-2b32f55b2b537f5c", "recipe kit-84ae2ca3af7bf898", "recipe kit 2-0cf8da91d218cc58", "recipe kit 3-1b5b09077ccd79b1", "recipe kit box-c17f989efac06973", "teaspoon-3002a5fee10cab1f", "yellow plastic tub-85234776d45178d0", "inner part of plastic colander-62700200aa5bcb6d", "outer part of plastic colander-ae784b59e34414f5", "pink tub-62da6e3ddb49e8a4", "can opener-6798748b46b4bf7b", "last green plastic tub-131c5e91f81ba466", "glass-7019e116a8c3433f", "large glass lid-ef5d507a0d964784", "new track 2-d9cdd18e54fae096", "three plastic tubs-1941ddf33f716a85", "dishwasher brush-fef6398cb4354466", "v60 stand-ce24c492374d4d73", "v60 metal filter-6bb6a298e601a4c7", "coffee jug-99955b3e64ae51c0", "plastic food bag-710840db9ad365ed", "bread knife-13cf2382a2dcefbb", "black coffee spoon-33f368ccebba8f3d", "red plate-9a0a9378ec1991a1", "orange cloth-9550ea1395d8b999", "bit of plastic-e574dbaf80217f33", "cutlery stand-c4b5cf177f57dd97", "sink strainer-b4180c4533ecd01a", "rice box-89d873889fbc8de0", "long leaf plant-dc0cd2596d8d5c98", "other bowl-77ee35879fe06a66", "bread-923d61d123f0e8ec", "larger part of bread loaf-658ce2fc8929474d", "bread wrapping-4079edb10a4a41be", "ladle-968795cabd8dc591", "other spoon-9c25ab6fcf4127d4", "one fork-a08bbd8820f753a6", "third spoon-cf952ecf5957cc4f" ], "reasoning": "1. I watched the video and found the red pack of lentils (pack of lentils-f62cf03618da91cd) at 00:24.\n2. At 00:35, I saw the user unwrap the packaging of the pack of lentils (pack of lentils-f62cf03618da91cd), but it was still not open yet.\n3. At 01:38, I saw the user open the pack of lentils (pack of lentils-f62cf03618da91cd) with scissors (scissors-7b21ea63ea07beb1).\n4. After 01:38, I saw the user's surroundings move to the right in relation to their movement at 01:40.\n5. Therefore, the user moved to the left after opening the red pack of lentils (pack of lentils-f62cf03618da91cd).", "referenced_object_masks": [ "pack of lentils-f62cf03618da91cd", "scissors-7b21ea63ea07beb1" ] }, { "key": "P03-20240217-150310:1fb3850ccd974c28d3ae8b6f4349d50d15890779", "video_id": "P03-20240217-150310", "question": "Where did I place the two bowls with the spoon after using them at the wooden table and before starting to put them in the dishwasher?", "answer_choice_0": "On the table", "answer_choice_1": "In the dishwasher", "answer_choice_2": "In the sink", "answer_choice_3": "To the right of the sink", "answer_choice_4": "To the left of the sink", "answer_id": 4, "answer": "To the left of the sink", "question_type": [ "Object Recognition", "Action Recognition", "Situational Awareness", "Egocentric & Spatial Perception", "Temporal Reasoning", "State Changes" ], "objects_of_interest": [ "bowl1-a23a1f547e6edf1a", "bowl2-c95fee71a468aa87", "green box-b98564fa3cd586bc", "plastic box-9f03123a92d70210", "plastic box lid-3c81d0b0cf8bcea5", "mug-161099ca6bbdc3ab", "white mug-7c25d7cbbbd904b3", "spoon-3ef203bf850de129", "kitchen brush-296549c92f90b7ef", "frying pan-eec6314e85e08e12", "ladle-feb621822f0fba07", "dirty spoons-f580c6c28b57b67f", "black spoon-220528d9d174fbf0", "small dirty bowl-55c85492a99a13e2", "stack of two dirty small plates-78fece5da16cac67", "bowl3-3f0fe528117a9ae9", "food bin-69e3a96ac1f43668", "pan lid-aa60e65f633d898f", "knife-a0ff533fe7cbc1b1", "spatula-2fbe078744f9f79a", "rubbish-387bc101e0fcd08a", "plastic pots-549f7ab994bcc57f", "plastic pot2-3b57212b77d7daaa", "coffee filter-61c33ca20327d473", "chopping board-629b63a7c8736032", "v60 coffee filter stand-23ae3bdfcc748e65", "sponge-86113aa65db98b0d", "bottle-8cf04d19dc77fe31", "small plate one-7ca51b3bcbb8fc40", "coffee jug-15841daaaef48c53", "cloth-5885fef75ad4126a", "zip bag-c6ad994a90e246b9", "green cup-cd57de8c972478b5", "red cup-f07c76b7692d75a0", "cling film box-9642f761b19f391f", "piece of bread-bef932af82108ec3", "left chair-dba1a0d2879e0921", "left tea towel-f9ff9db5e5394e06", "other tea towel-396ca32c9be54693", "orange cloth-c7bdf22494a4b168", "cleaning product-112be7e82209f7ca", "plastic water bottle-3d965663b586e235", "small flower pot-9c28c53dbfc0a51a", "small cup-b842a3eb4de85073", "hand wash bottle-1d1cec5633974385", "cuttlery holder-50be7cd9ef94b273", "sink plug-2cc4c9a04e201e84" ], "reasoning": "1. I watched the video and found the two bowls (bowl1-a23a1f547e6edf1a, bowl2-c95fee71a468aa87) at 00:04.\n2. I found the spoon (spoon-3ef203bf850de129) at 00:26.\n3. At 00:27-01:11, I saw the user use the bowls (bowl1-a23a1f547e6edf1a, bowl2-c95fee71a468aa87) and spoon (spoon-3ef203bf850de129) at the wooden table.\n4. After 01:11, I saw the user start to put the bowls (bowl1-a23a1f547e6edf1a, bowl2-c95fee71a468aa87) and spoon (spoon-3ef203bf850de129) in the dishwasher, starting with the spoon (spoon-3ef203bf850de129) first at 01:25.\n5. After 01:11 and before 01:25, I saw the user place the bowls (bowl1-a23a1f547e6edf1a, bowl2-c95fee71a468aa87) with the spoon (spoon-3ef203bf850de129) on the countertop to the left of the sink at 01:15.\n6. Therefore, the user placed the bowls (bowl1-a23a1f547e6edf1a, bowl2-c95fee71a468aa87) with the spoon (spoon-3ef203bf850de129) to the left of the sink after using them at the wooden table and before starting to put them in the dishwasher.", "referenced_object_masks": [ "spoon-3ef203bf850de129", "bowl1-a23a1f547e6edf1a", "bowl2-c95fee71a468aa87" ] }, { "key": "P03-20240217-150310:262c6ee0fd8f9301b3f36260ad903de259681b9e", "video_id": "P03-20240217-150310", "question": "How many electrical sockets immediately to the left of the microwave are occupied?", "answer_choice_0": "6", "answer_choice_1": "5", "answer_choice_2": "3", "answer_choice_3": "2", "answer_choice_4": "4", "answer_id": 3, "answer": "2", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Counting", "Temporal Reasoning", "Numerical Reasoning" ], "objects_of_interest": [ "bowl1-a23a1f547e6edf1a", "bowl2-c95fee71a468aa87", "green box-b98564fa3cd586bc", "plastic box-9f03123a92d70210", "plastic box lid-3c81d0b0cf8bcea5", "mug-161099ca6bbdc3ab", "white mug-7c25d7cbbbd904b3", "spoon-3ef203bf850de129", "kitchen brush-296549c92f90b7ef", "frying pan-eec6314e85e08e12", "ladle-feb621822f0fba07", "dirty spoons-f580c6c28b57b67f", "black spoon-220528d9d174fbf0", "small dirty bowl-55c85492a99a13e2", "stack of two dirty small plates-78fece5da16cac67", "bowl3-3f0fe528117a9ae9", "food bin-69e3a96ac1f43668", "pan lid-aa60e65f633d898f", "knife-a0ff533fe7cbc1b1", "spatula-2fbe078744f9f79a", "rubbish-387bc101e0fcd08a", "plastic pots-549f7ab994bcc57f", "plastic pot2-3b57212b77d7daaa", "coffee filter-61c33ca20327d473", "chopping board-629b63a7c8736032", "v60 coffee filter stand-23ae3bdfcc748e65", "sponge-86113aa65db98b0d", "bottle-8cf04d19dc77fe31", "small plate one-7ca51b3bcbb8fc40", "coffee jug-15841daaaef48c53", "cloth-5885fef75ad4126a", "zip bag-c6ad994a90e246b9", "green cup-cd57de8c972478b5", "red cup-f07c76b7692d75a0", "cling film box-9642f761b19f391f", "piece of bread-bef932af82108ec3", "left chair-dba1a0d2879e0921", "left tea towel-f9ff9db5e5394e06", "other tea towel-396ca32c9be54693", "orange cloth-c7bdf22494a4b168", "cleaning product-112be7e82209f7ca", "plastic water bottle-3d965663b586e235", "small flower pot-9c28c53dbfc0a51a", "small cup-b842a3eb4de85073", "hand wash bottle-1d1cec5633974385", "cuttlery holder-50be7cd9ef94b273", "sink plug-2cc4c9a04e201e84" ], "reasoning": "1. I locate the microwave across the room in the left corner at 00:00.\n2. I observe the user approach the microwave at 01:14, which reveals three electrical sockets to the left of the microwave .\n3. I determine that only one of the sockets is empty.\n4. I subtract 1 from 3 to get 2.", "referenced_object_masks": [] }, { "key": "P03-20240217-150310:38307811fc477711dd7d6fdbc67db71f1f00f62c", "video_id": "P03-20240217-150310", "question": "What direction would I have turned to look at the dishwasher if it were in the opposite direction while I was at the sink?", "answer_choice_0": "135 degrees to the left", "answer_choice_1": "45 degrees to the right", "answer_choice_2": "45 degrees to the left", "answer_choice_3": "180 degrees behind", "answer_choice_4": "135 degrees to the right", "answer_id": 1, "answer": "45 degrees to the right", "question_type": [ "Situational Awareness", "Action Recognition", "Egocentric & Spatial Perception", "Temporal Reasoning", "Counterfactual" ], "objects_of_interest": [ "bowl1-a23a1f547e6edf1a", "bowl2-c95fee71a468aa87", "green box-b98564fa3cd586bc", "plastic box-9f03123a92d70210", "plastic box lid-3c81d0b0cf8bcea5", "mug-161099ca6bbdc3ab", "white mug-7c25d7cbbbd904b3", "spoon-3ef203bf850de129", "kitchen brush-296549c92f90b7ef", "frying pan-eec6314e85e08e12", "ladle-feb621822f0fba07", "dirty spoons-f580c6c28b57b67f", "black spoon-220528d9d174fbf0", "small dirty bowl-55c85492a99a13e2", "stack of two dirty small plates-78fece5da16cac67", "bowl3-3f0fe528117a9ae9", "food bin-69e3a96ac1f43668", "pan lid-aa60e65f633d898f", "knife-a0ff533fe7cbc1b1", "spatula-2fbe078744f9f79a", "rubbish-387bc101e0fcd08a", "plastic pots-549f7ab994bcc57f", "plastic pot2-3b57212b77d7daaa", "coffee filter-61c33ca20327d473", "chopping board-629b63a7c8736032", "v60 coffee filter stand-23ae3bdfcc748e65", "sponge-86113aa65db98b0d", "bottle-8cf04d19dc77fe31", "small plate one-7ca51b3bcbb8fc40", "coffee jug-15841daaaef48c53", "cloth-5885fef75ad4126a", "zip bag-c6ad994a90e246b9", "green cup-cd57de8c972478b5", "red cup-f07c76b7692d75a0", "cling film box-9642f761b19f391f", "piece of bread-bef932af82108ec3", "left chair-dba1a0d2879e0921", "left tea towel-f9ff9db5e5394e06", "other tea towel-396ca32c9be54693", "orange cloth-c7bdf22494a4b168", "cleaning product-112be7e82209f7ca", "plastic water bottle-3d965663b586e235", "small flower pot-9c28c53dbfc0a51a", "small cup-b842a3eb4de85073", "hand wash bottle-1d1cec5633974385", "cuttlery holder-50be7cd9ef94b273", "sink plug-2cc4c9a04e201e84" ], "reasoning": "1. I watched the video and found the sink at 01:15.\n2. At 01:20, I saw the dishwasher when the user turned 135 degrees to the left to look from the sink and towards the dishwasher.\n3. Since the user turned 135 degrees to the left to view the dishwasher from the sink, the dishwasher was consequently behind and to the left of the user while they were at the sink.\n4. Since the opposite of \"behind and to the left\" is \"in front and to the right\", the opposite of \"135 degrees to the left\" is consequently \"45 degrees to the right\".\n5. Therefore, the user would have turned 45 degrees to the right to look at the dishwasher if it were in the opposite direction while they were at the sink.", "referenced_object_masks": [] }, { "key": "P03-20240217-150310:3aed4505666c984e42c9feb1c816464bfde21262", "video_id": "P03-20240217-150310", "question": "How many times do I brush the outside of the third bowl that enters the dishwasher?", "answer_choice_0": "4", "answer_choice_1": "2", "answer_choice_2": "6", "answer_choice_3": "3", "answer_choice_4": "5", "answer_id": 4, "answer": "5", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Counting", "Temporal Reasoning" ], "objects_of_interest": [ "bowl1-a23a1f547e6edf1a", "bowl2-c95fee71a468aa87", "green box-b98564fa3cd586bc", "plastic box-9f03123a92d70210", "plastic box lid-3c81d0b0cf8bcea5", "mug-161099ca6bbdc3ab", "white mug-7c25d7cbbbd904b3", "spoon-3ef203bf850de129", "kitchen brush-296549c92f90b7ef", "frying pan-eec6314e85e08e12", "ladle-feb621822f0fba07", "dirty spoons-f580c6c28b57b67f", "black spoon-220528d9d174fbf0", "small dirty bowl-55c85492a99a13e2", "stack of two dirty small plates-78fece5da16cac67", "bowl3-3f0fe528117a9ae9", "food bin-69e3a96ac1f43668", "pan lid-aa60e65f633d898f", "knife-a0ff533fe7cbc1b1", "spatula-2fbe078744f9f79a", "rubbish-387bc101e0fcd08a", "plastic pots-549f7ab994bcc57f", "plastic pot2-3b57212b77d7daaa", "coffee filter-61c33ca20327d473", "chopping board-629b63a7c8736032", "v60 coffee filter stand-23ae3bdfcc748e65", "sponge-86113aa65db98b0d", "bottle-8cf04d19dc77fe31", "small plate one-7ca51b3bcbb8fc40", "coffee jug-15841daaaef48c53", "cloth-5885fef75ad4126a", "zip bag-c6ad994a90e246b9", "green cup-cd57de8c972478b5", "red cup-f07c76b7692d75a0", "cling film box-9642f761b19f391f", "piece of bread-bef932af82108ec3", "left chair-dba1a0d2879e0921", "left tea towel-f9ff9db5e5394e06", "other tea towel-396ca32c9be54693", "orange cloth-c7bdf22494a4b168", "cleaning product-112be7e82209f7ca", "plastic water bottle-3d965663b586e235", "small flower pot-9c28c53dbfc0a51a", "small cup-b842a3eb4de85073", "hand wash bottle-1d1cec5633974385", "cuttlery holder-50be7cd9ef94b273", "sink plug-2cc4c9a04e201e84" ], "reasoning": "1. I locate the sink and dishwasher directly ahead of the user in the kitchen at 00:00.\n2. I locate the 2 bowls (bowl1-a23a1f547e6edf1a, bowl2-c95fee71a468aa87) at 00:02 on the kitchen table to the right of the user.\n3. I observe the user open the dishwasher and add a second bowl (bowl2-c95fee71a468aa87) to the top rack from 01:40 to 01:45.\n4. I observe the user add a third bowl (bowl1-a23a1f547e6edf1a) to the dishwasher from 01:58 to 02:02.\n5. I rewind and count the number of times the user brushes the outside of the third bowl (bowl1-a23a1f547e6edf1a). I conclude this action is repeated 5 times.", "referenced_object_masks": [ "bowl1-a23a1f547e6edf1a", "bowl2-c95fee71a468aa87" ] }, { "key": "P03-20240217-150310:53a6bac5a547f5bb89de89ad7b3ab97642b6d215", "video_id": "P03-20240217-150310", "question": "When did I finish washing the third dish after filling the dishwasher?", "answer_choice_0": "07:25", "answer_choice_1": "08:25", "answer_choice_2": "08:00", "answer_choice_3": "07:00", "answer_choice_4": "06:25", "answer_id": 0, "answer": "07:25", "question_type": [ "Situational Awareness", "State Changes", "Action Recognition", "Temporal Reasoning", "Counting", "Object Recognition" ], "objects_of_interest": [ "bowl1-a23a1f547e6edf1a", "bowl2-c95fee71a468aa87", "green box-b98564fa3cd586bc", "plastic box-9f03123a92d70210", "plastic box lid-3c81d0b0cf8bcea5", "mug-161099ca6bbdc3ab", "white mug-7c25d7cbbbd904b3", "spoon-3ef203bf850de129", "kitchen brush-296549c92f90b7ef", "frying pan-eec6314e85e08e12", "ladle-feb621822f0fba07", "dirty spoons-f580c6c28b57b67f", "black spoon-220528d9d174fbf0", "small dirty bowl-55c85492a99a13e2", "stack of two dirty small plates-78fece5da16cac67", "bowl3-3f0fe528117a9ae9", "food bin-69e3a96ac1f43668", "pan lid-aa60e65f633d898f", "knife-a0ff533fe7cbc1b1", "spatula-2fbe078744f9f79a", "rubbish-387bc101e0fcd08a", "plastic pots-549f7ab994bcc57f", "plastic pot2-3b57212b77d7daaa", "coffee filter-61c33ca20327d473", "chopping board-629b63a7c8736032", "v60 coffee filter stand-23ae3bdfcc748e65", "sponge-86113aa65db98b0d", "bottle-8cf04d19dc77fe31", "small plate one-7ca51b3bcbb8fc40", "coffee jug-15841daaaef48c53", "cloth-5885fef75ad4126a", "zip bag-c6ad994a90e246b9", "green cup-cd57de8c972478b5", "red cup-f07c76b7692d75a0", "cling film box-9642f761b19f391f", "piece of bread-bef932af82108ec3", "left chair-dba1a0d2879e0921", "left tea towel-f9ff9db5e5394e06", "other tea towel-396ca32c9be54693", "orange cloth-c7bdf22494a4b168", "cleaning product-112be7e82209f7ca", "plastic water bottle-3d965663b586e235", "small flower pot-9c28c53dbfc0a51a", "small cup-b842a3eb4de85073", "hand wash bottle-1d1cec5633974385", "cuttlery holder-50be7cd9ef94b273", "sink plug-2cc4c9a04e201e84" ], "reasoning": "1. I watched the video and found the dishwasher at 01:20.\n2. I saw the user finish filling the dishwasher at 06:25-06:28.\n3. After 06:28, I counted each dish the user washed.\n4. At 06:40, I saw the user wash the first dish, which was a small plate (small plate one-7ca51b3bcbb8fc40).\n5. At 07:05, I saw the user wash the second dish, which was another small plate .\n6. At 07:15, I saw the user start washing the third dish, which was a coffee filter stand (v60 coffee filter stand-23ae3bdfcc748e65).\n7. At 07:25, I saw the user finish washing the coffee filter stand (v60 coffee filter stand-23ae3bdfcc748e65).\n8. Therefore, the user finished washing the third dish after filling the dishwasher at 07:25.", "referenced_object_masks": [ "small plate one-7ca51b3bcbb8fc40", "v60 coffee filter stand-23ae3bdfcc748e65" ] }, { "key": "P03-20240217-150310:7b8e2292c0493bdd1e7d5c7180dff76ad2a01cde", "video_id": "P03-20240217-150310", "question": "What is the first thing I pick up with my right hand after clearing the first two bowls from the table?", "answer_choice_0": "Bowl", "answer_choice_1": "Faucet", "answer_choice_2": "Table", "answer_choice_3": "Kitchen brush", "answer_choice_4": "Container", "answer_id": 3, "answer": "Kitchen brush", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Counting", "Temporal Reasoning" ], "objects_of_interest": [ "bowl1-a23a1f547e6edf1a", "bowl2-c95fee71a468aa87", "green box-b98564fa3cd586bc", "plastic box-9f03123a92d70210", "plastic box lid-3c81d0b0cf8bcea5", "mug-161099ca6bbdc3ab", "white mug-7c25d7cbbbd904b3", "spoon-3ef203bf850de129", "kitchen brush-296549c92f90b7ef", "frying pan-eec6314e85e08e12", "ladle-feb621822f0fba07", "dirty spoons-f580c6c28b57b67f", "black spoon-220528d9d174fbf0", "small dirty bowl-55c85492a99a13e2", "stack of two dirty small plates-78fece5da16cac67", "bowl3-3f0fe528117a9ae9", "food bin-69e3a96ac1f43668", "pan lid-aa60e65f633d898f", "knife-a0ff533fe7cbc1b1", "spatula-2fbe078744f9f79a", "rubbish-387bc101e0fcd08a", "plastic pots-549f7ab994bcc57f", "plastic pot2-3b57212b77d7daaa", "coffee filter-61c33ca20327d473", "chopping board-629b63a7c8736032", "v60 coffee filter stand-23ae3bdfcc748e65", "sponge-86113aa65db98b0d", "bottle-8cf04d19dc77fe31", "small plate one-7ca51b3bcbb8fc40", "coffee jug-15841daaaef48c53", "cloth-5885fef75ad4126a", "zip bag-c6ad994a90e246b9", "green cup-cd57de8c972478b5", "red cup-f07c76b7692d75a0", "cling film box-9642f761b19f391f", "piece of bread-bef932af82108ec3", "left chair-dba1a0d2879e0921", "left tea towel-f9ff9db5e5394e06", "other tea towel-396ca32c9be54693", "orange cloth-c7bdf22494a4b168", "cleaning product-112be7e82209f7ca", "plastic water bottle-3d965663b586e235", "small flower pot-9c28c53dbfc0a51a", "small cup-b842a3eb4de85073", "hand wash bottle-1d1cec5633974385", "cuttlery holder-50be7cd9ef94b273", "sink plug-2cc4c9a04e201e84" ], "reasoning": "1. I locate the first 2 bowls (bowl1-a23a1f547e6edf1a, bowl2-c95fee71a468aa87) at 00:02 on the kitchen table to the right.\n2. I determine that the user empties out the 2 bowls (bowl1-a23a1f547e6edf1a, bowl2-c95fee71a468aa87) into the tupperware (plastic box-9f03123a92d70210) from 00:27 to 01:09.\n3. I determine that the user stacks the bowls (bowl1-a23a1f547e6edf1a, bowl2-c95fee71a468aa87) and clears them from the table at 01:11.\n4. I observe the user reach out to grab the kitchen brush (kitchen brush-296549c92f90b7ef) at 01:15 with their right hand.", "referenced_object_masks": [ "plastic box-9f03123a92d70210", "bowl1-a23a1f547e6edf1a", "bowl2-c95fee71a468aa87", "kitchen brush-296549c92f90b7ef" ] }, { "key": "P03-20240217-150310:94252b616ac650accc56b616b243e219d1971bdf", "video_id": "P03-20240217-150310", "question": "What number do I get if I add the number of tea towels I remove from the table to the number of potted plants above the kitchen sink?", "answer_choice_0": "5", "answer_choice_1": "6", "answer_choice_2": "2", "answer_choice_3": "3", "answer_choice_4": "4", "answer_id": 1, "answer": "6", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Counting", "Temporal Reasoning", "Numerical Reasoning" ], "objects_of_interest": [ "bowl1-a23a1f547e6edf1a", "bowl2-c95fee71a468aa87", "green box-b98564fa3cd586bc", "plastic box-9f03123a92d70210", "plastic box lid-3c81d0b0cf8bcea5", "mug-161099ca6bbdc3ab", "white mug-7c25d7cbbbd904b3", "spoon-3ef203bf850de129", "kitchen brush-296549c92f90b7ef", "frying pan-eec6314e85e08e12", "ladle-feb621822f0fba07", "dirty spoons-f580c6c28b57b67f", "black spoon-220528d9d174fbf0", "small dirty bowl-55c85492a99a13e2", "stack of two dirty small plates-78fece5da16cac67", "bowl3-3f0fe528117a9ae9", "food bin-69e3a96ac1f43668", "pan lid-aa60e65f633d898f", "knife-a0ff533fe7cbc1b1", "spatula-2fbe078744f9f79a", "rubbish-387bc101e0fcd08a", "plastic pots-549f7ab994bcc57f", "plastic pot2-3b57212b77d7daaa", "coffee filter-61c33ca20327d473", "chopping board-629b63a7c8736032", "v60 coffee filter stand-23ae3bdfcc748e65", "sponge-86113aa65db98b0d", "bottle-8cf04d19dc77fe31", "small plate one-7ca51b3bcbb8fc40", "coffee jug-15841daaaef48c53", "cloth-5885fef75ad4126a", "zip bag-c6ad994a90e246b9", "green cup-cd57de8c972478b5", "red cup-f07c76b7692d75a0", "cling film box-9642f761b19f391f", "piece of bread-bef932af82108ec3", "left chair-dba1a0d2879e0921", "left tea towel-f9ff9db5e5394e06", "other tea towel-396ca32c9be54693", "orange cloth-c7bdf22494a4b168", "cleaning product-112be7e82209f7ca", "plastic water bottle-3d965663b586e235", "small flower pot-9c28c53dbfc0a51a", "small cup-b842a3eb4de85073", "hand wash bottle-1d1cec5633974385", "cuttlery holder-50be7cd9ef94b273", "sink plug-2cc4c9a04e201e84" ], "reasoning": "1. I locate the tea towels (left tea towel-f9ff9db5e5394e06, other tea towel-396ca32c9be54693) on the table at 00:02.\n2. I locate a clear view of the plants above the sink at 01:14.\n3. I determine there are 4 potted plants above the sink at 01:14.\n4. I observe the user remove from the table the first tea towel (left tea towel-f9ff9db5e5394e06) and the second tea towel (other tea towel-396ca32c9be54693) from 09:35 to 09:55.\n5. I add 2 to 4 to get 6.", "referenced_object_masks": [ "left tea towel-f9ff9db5e5394e06", "other tea towel-396ca32c9be54693" ] }, { "key": "P03-20240217-150310:9c238ee4dc1660362f8a8c4949b03b90423b7769", "video_id": "P03-20240217-150310", "question": "Where is the sink located in relation to me when I place the fourth bowl into the top dishwasher rack?", "answer_choice_0": "To the left of me", "answer_choice_1": "Above me", "answer_choice_2": "In front of me", "answer_choice_3": "To the right of me", "answer_choice_4": "Behind me", "answer_id": 3, "answer": "To the right of me", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Counting", "Temporal Reasoning" ], "objects_of_interest": [ "bowl1-a23a1f547e6edf1a", "bowl2-c95fee71a468aa87", "green box-b98564fa3cd586bc", "plastic box-9f03123a92d70210", "plastic box lid-3c81d0b0cf8bcea5", "mug-161099ca6bbdc3ab", "white mug-7c25d7cbbbd904b3", "spoon-3ef203bf850de129", "kitchen brush-296549c92f90b7ef", "frying pan-eec6314e85e08e12", "ladle-feb621822f0fba07", "dirty spoons-f580c6c28b57b67f", "black spoon-220528d9d174fbf0", "small dirty bowl-55c85492a99a13e2", "stack of two dirty small plates-78fece5da16cac67", "bowl3-3f0fe528117a9ae9", "food bin-69e3a96ac1f43668", "pan lid-aa60e65f633d898f", "knife-a0ff533fe7cbc1b1", "spatula-2fbe078744f9f79a", "rubbish-387bc101e0fcd08a", "plastic pots-549f7ab994bcc57f", "plastic pot2-3b57212b77d7daaa", "coffee filter-61c33ca20327d473", "chopping board-629b63a7c8736032", "v60 coffee filter stand-23ae3bdfcc748e65", "sponge-86113aa65db98b0d", "bottle-8cf04d19dc77fe31", "small plate one-7ca51b3bcbb8fc40", "coffee jug-15841daaaef48c53", "cloth-5885fef75ad4126a", "zip bag-c6ad994a90e246b9", "green cup-cd57de8c972478b5", "red cup-f07c76b7692d75a0", "cling film box-9642f761b19f391f", "piece of bread-bef932af82108ec3", "left chair-dba1a0d2879e0921", "left tea towel-f9ff9db5e5394e06", "other tea towel-396ca32c9be54693", "orange cloth-c7bdf22494a4b168", "cleaning product-112be7e82209f7ca", "plastic water bottle-3d965663b586e235", "small flower pot-9c28c53dbfc0a51a", "small cup-b842a3eb4de85073", "hand wash bottle-1d1cec5633974385", "cuttlery holder-50be7cd9ef94b273", "sink plug-2cc4c9a04e201e84" ], "reasoning": "1. I locate the the sink and dish washer directly ahead of the user in the kitchen at 00:00.\n2. I observe that the user pulls out the top dish washer rack with one bowl in it at 01:39.\n3. I see the user place the second (bowl2-c95fee71a468aa87), third (bowl1-a23a1f547e6edf1a), and fourth (small dirty bowl-55c85492a99a13e2) bowls into the dish washer at 01:45, 02:02, and 03:38.\n4. I determine the location of the sink in relation to the user at 03:38.", "referenced_object_masks": [ "small dirty bowl-55c85492a99a13e2", "bowl1-a23a1f547e6edf1a", "bowl2-c95fee71a468aa87" ] }, { "key": "P03-20240217-150310:a311e04e9348bb203fb68ad7f35f35f21b4e622f", "video_id": "P03-20240217-150310", "question": "Where did I get the second item I put on the wooden table from?", "answer_choice_0": "The dishwasher", "answer_choice_1": "A cabinet", "answer_choice_2": "The sink", "answer_choice_3": "A countertop", "answer_choice_4": "The table", "answer_id": 1, "answer": "A cabinet", "question_type": [ "Situational Awareness", "Counting", "Action Recognition", "State Changes", "Egocentric & Spatial Perception", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "bowl1-a23a1f547e6edf1a", "bowl2-c95fee71a468aa87", "green box-b98564fa3cd586bc", "plastic box-9f03123a92d70210", "plastic box lid-3c81d0b0cf8bcea5", "mug-161099ca6bbdc3ab", "white mug-7c25d7cbbbd904b3", "spoon-3ef203bf850de129", "kitchen brush-296549c92f90b7ef", "frying pan-eec6314e85e08e12", "ladle-feb621822f0fba07", "dirty spoons-f580c6c28b57b67f", "black spoon-220528d9d174fbf0", "small dirty bowl-55c85492a99a13e2", "stack of two dirty small plates-78fece5da16cac67", "bowl3-3f0fe528117a9ae9", "food bin-69e3a96ac1f43668", "pan lid-aa60e65f633d898f", "knife-a0ff533fe7cbc1b1", "spatula-2fbe078744f9f79a", "rubbish-387bc101e0fcd08a", "plastic pots-549f7ab994bcc57f", "plastic pot2-3b57212b77d7daaa", "coffee filter-61c33ca20327d473", "chopping board-629b63a7c8736032", "v60 coffee filter stand-23ae3bdfcc748e65", "sponge-86113aa65db98b0d", "bottle-8cf04d19dc77fe31", "small plate one-7ca51b3bcbb8fc40", "coffee jug-15841daaaef48c53", "cloth-5885fef75ad4126a", "zip bag-c6ad994a90e246b9", "green cup-cd57de8c972478b5", "red cup-f07c76b7692d75a0", "cling film box-9642f761b19f391f", "piece of bread-bef932af82108ec3", "left chair-dba1a0d2879e0921", "left tea towel-f9ff9db5e5394e06", "other tea towel-396ca32c9be54693", "orange cloth-c7bdf22494a4b168", "cleaning product-112be7e82209f7ca", "plastic water bottle-3d965663b586e235", "small flower pot-9c28c53dbfc0a51a", "small cup-b842a3eb4de85073", "hand wash bottle-1d1cec5633974385", "cuttlery holder-50be7cd9ef94b273", "sink plug-2cc4c9a04e201e84" ], "reasoning": "1. I watched the video and found the wooden table at 00:02.\n2. I counted each item the user put on the wooden table.\n3. At 00:20, I saw the user put the first item on the wooden table, which was a tupperware lid (plastic box lid-3c81d0b0cf8bcea5).\n4. At 00:25, I saw the user put the second item on the wooden table, which was a tupperware box (plastic box-9f03123a92d70210).\n5. Before 00:25, I saw the user get the tupperware box (plastic box-9f03123a92d70210) from a cabinet.\n6. Therefore, the user got the second item they put on the wooden table from a cabinet.", "referenced_object_masks": [ "plastic box lid-3c81d0b0cf8bcea5", "plastic box-9f03123a92d70210" ] }, { "key": "P03-20240217-150310:a546c01baa3184d1ed90ebe04d248c0452466644", "video_id": "P03-20240217-150310", "question": "If I had scrubbed and put the frying pan in the dishwasher twice as fast, approximately how much time would have passed for me to do so?", "answer_choice_0": "11 seconds", "answer_choice_1": "22 seconds", "answer_choice_2": "45 seconds", "answer_choice_3": "60 seconds", "answer_choice_4": "90 seconds", "answer_id": 1, "answer": "22 seconds", "question_type": [ "Situational Awareness", "Object Recognition", "Action Recognition", "Egocentric & Spatial Perception", "State Changes", "Temporal Reasoning", "Numerical Reasoning", "Counterfactual" ], "objects_of_interest": [ "bowl1-a23a1f547e6edf1a", "bowl2-c95fee71a468aa87", "green box-b98564fa3cd586bc", "plastic box-9f03123a92d70210", "plastic box lid-3c81d0b0cf8bcea5", "mug-161099ca6bbdc3ab", "white mug-7c25d7cbbbd904b3", "spoon-3ef203bf850de129", "kitchen brush-296549c92f90b7ef", "frying pan-eec6314e85e08e12", "ladle-feb621822f0fba07", "dirty spoons-f580c6c28b57b67f", "black spoon-220528d9d174fbf0", "small dirty bowl-55c85492a99a13e2", "stack of two dirty small plates-78fece5da16cac67", "bowl3-3f0fe528117a9ae9", "food bin-69e3a96ac1f43668", "pan lid-aa60e65f633d898f", "knife-a0ff533fe7cbc1b1", "spatula-2fbe078744f9f79a", "rubbish-387bc101e0fcd08a", "plastic pots-549f7ab994bcc57f", "plastic pot2-3b57212b77d7daaa", "coffee filter-61c33ca20327d473", "chopping board-629b63a7c8736032", "v60 coffee filter stand-23ae3bdfcc748e65", "sponge-86113aa65db98b0d", "bottle-8cf04d19dc77fe31", "small plate one-7ca51b3bcbb8fc40", "coffee jug-15841daaaef48c53", "cloth-5885fef75ad4126a", "zip bag-c6ad994a90e246b9", "green cup-cd57de8c972478b5", "red cup-f07c76b7692d75a0", "cling film box-9642f761b19f391f", "piece of bread-bef932af82108ec3", "left chair-dba1a0d2879e0921", "left tea towel-f9ff9db5e5394e06", "other tea towel-396ca32c9be54693", "orange cloth-c7bdf22494a4b168", "cleaning product-112be7e82209f7ca", "plastic water bottle-3d965663b586e235", "small flower pot-9c28c53dbfc0a51a", "small cup-b842a3eb4de85073", "hand wash bottle-1d1cec5633974385", "cuttlery holder-50be7cd9ef94b273", "sink plug-2cc4c9a04e201e84" ], "reasoning": "1. I watched the video and found the dishwasher at 01:20.\n2. I found the frying pan (frying pan-eec6314e85e08e12) at 02:04.\n3. I saw the user start scrubbing the frying pan (frying pan-eec6314e85e08e12) at 02:07.\n4. I saw the user put the frying pan (frying pan-eec6314e85e08e12) in the dishwasher at 02:50.\n5. I calculated 02:50-02:07=00:43.\n6. I calculated 43/2=21.5, which approximates to 22.\n7. Therefore, if the user had scrubbed and put the frying pan (frying pan-eec6314e85e08e12) in the dishwasher twice as fast, approximately 22 seconds would have passed for them to do so.", "referenced_object_masks": [ "frying pan-eec6314e85e08e12" ] }, { "key": "P03-20240217-152152:17ed4d7f33e2cc01c504c4f66435f1bde446229f", "video_id": "P03-20240217-152152", "question": "How many instances would I have placed my hand on the washer knob and turned it if there were 3 additional instances?", "answer_choice_0": "10", "answer_choice_1": "12", "answer_choice_2": "8", "answer_choice_3": "7", "answer_choice_4": "11", "answer_id": 4, "answer": "11", "question_type": [ "Egocentric & Spatial Perception", "Action Recognition", "Counterfactual", "Counting", "Object Recognition" ], "objects_of_interest": [ "laundry bag-06f594a456c0a791", "cloth-e68654aa7a1faca0", "jeans-f876d0ba8c9c3c4e", "jumper-bbff514e3fff194b", "sock-6ffe1f080f2042b7", "clothes-f16d389970b4a622", "more clothes-4337c96055fba798", "sock2-43d9a7d673bf5b6c", "laundry detergent bottle-67e280f98e5cb274", "cap-91174ad22f1a7f16", "laundry conditioner-e5aa827dd1bb1a17", "laundry conditioner cap-e2a97ec7437be14a" ], "reasoning": "1. I watched the video and looked for when the user placed a hand on the washer knob and turned it. I found the first instance of this at 01:44.\n2. The additional instances of the user turning the washer knob were ambiguous at first glance and hard to detect, so I slowed the video down to capture additional instances at 01:51, 01:54, 01:55, 01:58, 02:00, 02:11 and 02:24. Counting these all together, I arrived at a total of 8.\n3. Since there were 8 total instances, I added 3, arriving at a total of 11.", "referenced_object_masks": [] }, { "key": "P03-20240217-152152:42f7a6b5207196dfc0f3f3c3cef0cf89c74ceb40", "video_id": "P03-20240217-152152", "question": "From the direction I faced while pulling laundry out of the basket, what would I be looking at if I turned 270 degrees counter-clockwise?", "answer_choice_0": "The dining room", "answer_choice_1": "The sink", "answer_choice_2": "The dishwasher", "answer_choice_3": "The cupboards", "answer_choice_4": "The oven", "answer_id": 0, "answer": "The dining room", "question_type": [ "Action Recognition", "Cause and Effect", "Counterfactual", "Egocentric & Spatial Perception", "Situational Awareness" ], "objects_of_interest": [ "laundry bag-06f594a456c0a791", "cloth-e68654aa7a1faca0", "jeans-f876d0ba8c9c3c4e", "jumper-bbff514e3fff194b", "sock-6ffe1f080f2042b7", "clothes-f16d389970b4a622", "more clothes-4337c96055fba798", "sock2-43d9a7d673bf5b6c", "laundry detergent bottle-67e280f98e5cb274", "cap-91174ad22f1a7f16", "laundry conditioner-e5aa827dd1bb1a17", "laundry conditioner cap-e2a97ec7437be14a" ], "reasoning": "1. I watched the video in order to find out when the user pulls laundry out of a basket. I found this for the first time at 00:10, when the user grabs jeans (jeans-f876d0ba8c9c3c4e) from the basket (laundry bag-06f594a456c0a791).\n2. I then took note of where the user was facing as they did this. I noticed that directly ahead of them is an oven .\n3. In order to understand where the user would be facing if they turned 270 degrees at 00:10, I went back to the start of the video, at 00:02. At that time, the user walks forward while holding the basket (laundry bag-06f594a456c0a791). Directly ahead of them on the right is the oven which comes into view at 00:03.\n4. Since this is on the right, I could assume that if the user turned 270 degrees to the left, nearly a complete circle, they would be facing the direction they just walked into the kitchen. This means they would be facing the dining room.", "referenced_object_masks": [ "laundry bag-06f594a456c0a791", "jeans-f876d0ba8c9c3c4e" ] }, { "key": "P03-20240217-152152:7a152f4936ced734429ec0eec29449951be4a683", "video_id": "P03-20240217-152152", "question": "From the moment I threw the first laundry item into the washer, how long would it have taken me to close the washer door if I took an additional 45 seconds to do so?", "answer_choice_0": "1 minute and 59 seconds", "answer_choice_1": "1 minute and 48 seconds", "answer_choice_2": "2 minutes and 5 seconds", "answer_choice_3": "1 minute and 35 seconds", "answer_choice_4": "2 minutes and 13 seconds", "answer_id": 1, "answer": "1 minute and 48 seconds", "question_type": [ "Action Recognition", "Counterfactual", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "laundry bag-06f594a456c0a791", "cloth-e68654aa7a1faca0", "jeans-f876d0ba8c9c3c4e", "jumper-bbff514e3fff194b", "sock-6ffe1f080f2042b7", "clothes-f16d389970b4a622", "more clothes-4337c96055fba798", "sock2-43d9a7d673bf5b6c", "laundry detergent bottle-67e280f98e5cb274", "cap-91174ad22f1a7f16", "laundry conditioner-e5aa827dd1bb1a17", "laundry conditioner cap-e2a97ec7437be14a" ], "reasoning": "1. I watched the video, looking for the time when the user first threw an item into the washer . I found this occurred at 00:09, when the user threw in a cloth (cloth-e68654aa7a1faca0).\n2. Then, I looked for when the user closed the washer door . I found this occurred at 01:12.\n3. I then subtracted 00:09 from 01:12, to get a total of 01:03.\n4. I added 45 additional seconds to 01:03, getting a total of 1 minute and 48 seconds as the final answer.", "referenced_object_masks": [ "cloth-e68654aa7a1faca0" ] }, { "key": "P03-20240217-152152:831baa95c75d69b845dd4a1ba8a5ef4501d3ea0d", "video_id": "P03-20240217-152152", "question": "What is the ratio of objects on the table near the kitchen to the number of times I put items into the washer?", "answer_choice_0": "1:3", "answer_choice_1": "2:3", "answer_choice_2": "1:2", "answer_choice_3": "1:4", "answer_choice_4": "2:5", "answer_id": 2, "answer": "1:2", "question_type": [ "Action Recognition", "Counting", "Numerical Reasoning", "Situational Awareness" ], "objects_of_interest": [ "laundry bag-06f594a456c0a791", "cloth-e68654aa7a1faca0", "jeans-f876d0ba8c9c3c4e", "jumper-bbff514e3fff194b", "sock-6ffe1f080f2042b7", "clothes-f16d389970b4a622", "more clothes-4337c96055fba798", "sock2-43d9a7d673bf5b6c", "laundry detergent bottle-67e280f98e5cb274", "cap-91174ad22f1a7f16", "laundry conditioner-e5aa827dd1bb1a17", "laundry conditioner cap-e2a97ec7437be14a" ], "reasoning": "1. I watched the video and looked first for a table near the kitchen with objects on it. I found this immediately at 00:00, when the table is visible on the left. On top of it, there is a water bottle , a green cup , a red cup , and a square object .\n2. Then, I counted the number of times the user put items into the washer. This occurred at the following times with the following objects: the cloth (cloth-e68654aa7a1faca0) at 00:09, the jeans (jeans-f876d0ba8c9c3c4e) at 00:12, the jumper (jumper-bbff514e3fff194b) at 00:16, more jeans at 00:19, a sock (sock-6ffe1f080f2042b7) at 00:22, clothes (clothes-f16d389970b4a622) at 00:26, more clothes (more clothes-4337c96055fba798) at 00:31, and one more sock (sock2-43d9a7d673bf5b6c) at 00:36. No other items were added.\n3. Since the user put items in 8 times, and there were 4 items on the kitchen table, I determined the ratio was 4:8. Simplifying this reaches a final answer of 1:2.", "referenced_object_masks": [ "more clothes-4337c96055fba798", "clothes-f16d389970b4a622", "sock2-43d9a7d673bf5b6c", "cloth-e68654aa7a1faca0", "jumper-bbff514e3fff194b", "sock-6ffe1f080f2042b7", "jeans-f876d0ba8c9c3c4e" ] }, { "key": "P03-20240217-152152:c8419431d76375d628acdf0c3159d2c07e9ff48c", "video_id": "P03-20240217-152152", "question": "Where is the fridge in relation to me as I pour laundry detergent into the cap?", "answer_choice_0": "Behind me and to my right", "answer_choice_1": "In front of me and to my left", "answer_choice_2": "Behind me and to my left", "answer_choice_3": "Directly in front of me", "answer_choice_4": "In front of me and to my right", "answer_id": 2, "answer": "Behind me and to my left", "question_type": [ "Action Recognition", "Object Recognition", "Egocentric & Spatial Perception", "Situational Awareness" ], "objects_of_interest": [ "laundry bag-06f594a456c0a791", "cloth-e68654aa7a1faca0", "jeans-f876d0ba8c9c3c4e", "jumper-bbff514e3fff194b", "sock-6ffe1f080f2042b7", "clothes-f16d389970b4a622", "more clothes-4337c96055fba798", "sock2-43d9a7d673bf5b6c", "laundry detergent bottle-67e280f98e5cb274", "cap-91174ad22f1a7f16", "laundry conditioner-e5aa827dd1bb1a17", "laundry conditioner cap-e2a97ec7437be14a" ], "reasoning": "1. I watched the video, looking for when the user poured laundry detergent from its bottle (laundry detergent bottle-67e280f98e5cb274) into the cap (cap-91174ad22f1a7f16). I found this moment from 00:50-00:57.\n2. Then, I looked for the fridge . I didn't see it immediately, so I went back in the video to locate it. I found the fridge at 00:02, where it is positioned on the left just before the user enters the kitchen.\n3. I noticed from this same angle at 00:02, the washer is visible straight ahead and on the right, in front of the sink. The sink was also visible from 00:50-00:57, and the user at that time was standing to the left of it, oriented toward the upper left corner of the kitchen.\n4. From this information, I gleaned that the fridge was behind the user and to their left when they were pouring laundry detergent (laundry detergent bottle-67e280f98e5cb274) into the cap (cap-91174ad22f1a7f16).", "referenced_object_masks": [ "cap-91174ad22f1a7f16", "laundry detergent bottle-67e280f98e5cb274" ] }, { "key": "P03-20240217-192543:08faa5fad78526043cec9eecfffe2772b982b6f7", "video_id": "P03-20240217-192543", "question": "Considering the amount of times I blotted grease from each pizza, which one contains the most grease?", "answer_choice_0": "Eggplant and eggs pizza", "answer_choice_1": "Mushroom pizza", "answer_choice_2": "Eggplant pizza", "answer_choice_3": "Egg pizza", "answer_choice_4": "Cheese pizza", "answer_id": 0, "answer": "Eggplant and eggs pizza", "question_type": [ "Object Recognition", "Action Recognition", "Event Occurence", "Situational Awareness", "Counting" ], "objects_of_interest": [ "chair-ae03caabc008cafb", "table-a33ac46cdf21b10d", "chair2-0db6f6a3fc22c641", "aubergine-20c9c618fa8582c0", "broccoli-1046800e5ac9e9f6", "box of mushrooms-c8ed0de7c276fb90", "cauliflower-60cbba9c43361b16", "box of tomatoes-1e9bc904a15f4ddc", "bag of potatoes-3c2a8c3a953c896e", "bowl-f1e14dbc55594915", "bag of vegetables-5f21f12e8ede6d7e", "glass-2d8b5870be4d17ca", "coffee jug-7d07b54a26894e4a", "pink tea saucer-01fdf27ae6b4c2f4", "green tea saucer-58949162c8c7e96b", "v60 filter stand-bebc25cf16165a0a", "large bowl-76e1bb44fe1ff059", "plastic colander-01d1e453d8330cff", "red knife-7cd78e9152bfda9f", "saucepan-1ae923a3e236c771", "plastic padding-6af5261a2508f789", "small frying pan-53bba2b7ee816a80", "chopping board-7a330dc9feae0b4d", "white cloth-db4c999c952809af", "green glass-7d4b3c5d4d6c5022", "plastic water bottle-6fe607511c700e20", "cloth2-06a0860a291859d7", "knife-711b76a54098c634", "sponge-8a508859b42e89c2", "sunflower oil bottle-afec528692163e0b", "bottle of tomato sauce-ad4056643625219f", "small saucepan-b2583f8fb59800a8", "aubergine chunks1-dd76fed8cdbeb1f9", "aubergine chunks2-1262d276474e9376", "aubergine chunks3-53680550e6c37e87", "wooden spoon-8c3e202e4069e242", "dough scraper-9ca4d6e4d99afe45", "salt container-ff11afda402bd7eb", "herb container-caa3e734af663fb3", "black pepper grinder-8580a8a725cf289c", "mushroom1-392b1258da597bb3", "mushroom2-6ad27c0ea25e3a7e", "mushroom3-a2cb332725831efa", "mushroom4-2cca5f7434198cf1", "mushroom5-82f0c56cc5190e9e", "small saucepan2-5f26776f068a0c3a", "box of eggs-045b2504626151a4", "egg-0ddf9965588cb69c", "small cup-7305591d1f2d3c4e", "one garlic clove-bfc3b756d4c09592", "garlic mesh-95309fe3c60b3ca5", "garlic clove2-7e99f37e0c3473a1", "plate-2c8a296efdb1ab1b", "kitchen roll holder-1416823f3b2c2df6", "kitchen towel piece-be8a51519a505f1a", "ground garlic container-4f195d3f54e9ef4c", "peeled garlic clove-58d26742b7ba74ca", "slice of mushroom-76bd5fda55baa253", "garlic clove3-7d79cdc205528bc5", "ground garlic container lid-6c81336872669aad", "small bowl-4681bc924aa25206", "ladle-668b0e4edd9cd84b", "cone-5202d776756c5d1f", "large cup-8ce7a207ef526787", "small bowl2-fe8fbdb509fd2136", "olive oil tin cap-64d3f95de92b5455", "spoon-8d8f09014071bb0e", "tomato sauce cap-eb9f0d69cc49f849", "bag-adb3ca53674df30d", "food plastic bag-ab50eb05d089d7cf", "coffee grinder-1a49e057a10065ea", "glass olive dispencer-7ffffcdb088c428f", "orange cloth-a00209deb0226c52", "plastic water bottle2-cca5ffeec29524a3", "plastic cup-569d1cdc613949ba", "rice box-9631cf6a085c0a8a", "plastic measuring cup-e437978f66299245", "pizza oven-d082997d1bd13f8e", "box-db8fabef7497b2c4", "bag of semolina flour-338f5c2467d4cb04", "box of dough-192cccdb37d7b19c", "cheese grater-dd600872585677c5", "pizza peel-02628224f1c1d3fe", "two pizza spatulas-277428e76cac81ac", "box lid-b023925590e46846", "clip-98c90e797564ea14", "mozzarella bag-6f9c6f09d13ab016", "scissors-d37396785ab1d8f7", "mozzarella cheese-aff7c515cbf174e7", "chunks of mozzarella-38dafdec0ec9590c", "one slice of egg-8f15829ac5dfbd3f", "another slice of egg-2db503542d981640", "third slice of egg-5178fe2d5160b344", "fourth slice of egg-446e5c1a8127ff28", "red plate-32d950c2aa0ea8b0", "kitchen towel piece2-af33ff05ac2e60f4", "cutlery-35692999d3cbd20e", "tea towel-6df77a314affe872", "tea towel2-23b33810122204da", "pizza spatula2-8008bb1230333497", "fork-05d0abe7a5551fed", "knife2-eda0fb4051f7eb69", "fork2-ca3e03f919ec2342", "knife3-ee85eb04e657509b", "plate2-017fd5b7141d3c63", "large glass lid-d789692dfd9d2ffe", "washing up liquid bottle-064eabf0b77aa7c0", "kitchen towel piece3-b9f72292f67a7762", "pizza spatula1-6461eaf6cb9de906" ], "reasoning": "1. I first looked at the amount of pizzas the user made. The first pizza, which contained eggplant (aubergine chunks1-dd76fed8cdbeb1f9, aubergine chunks2-1262d276474e9376, aubergine chunks3-53680550e6c37e87) and boiled eggs (egg-0ddf9965588cb69c), was put in the pizza oven (pizza oven-d082997d1bd13f8e) at 43:03. \n2. At 48:07, the user blots some grease from the first pizza for the first time, at 49:17 for the second time, and at 50:09 for the third time. \n3. At 50:30, the user places the second pizza in the pizza oven pizza (oven-d082997d1bd13f8e). This pizza contained mushrooms (mushroom1-392b1258da597bb3, mushroom2-6ad27c0ea25e3a7e, mushroom3-a2cb332725831efa, mushroom4-2cca5f7434198cf1, mushroom5-82f0c56cc5190e9e) at 10:41. The user has 5 mushrooms (mushroom1-392b1258da597bb3, mushroom2-6ad27c0ea25e3a7e, mushroom3-a2cb332725831efa, mushroom4-2cca5f7434198cf1, mushroom5-82f0c56cc5190e9e). \n4. At 54:32, the user blots grease from the second pizza for the first time, then at 56:03 he does it for the second and last time. Therefore, the pizza that contained the most grease was the first pizza, the eggplant (aubergine chunks1-dd76fed8cdbeb1f9, aubergine chunks2-1262d276474e9376, aubergine chunks3-53680550e6c37e87) and boiled eggs (egg-0ddf9965588cb69c) pizza.", "referenced_object_masks": [ "pizza oven-d082997d1bd13f8e", "aubergine chunks1-dd76fed8cdbeb1f9", "aubergine chunks2-1262d276474e9376", "aubergine chunks3-53680550e6c37e87", "egg-0ddf9965588cb69c", "mushroom1-392b1258da597bb3", "mushroom2-6ad27c0ea25e3a7e", "mushroom3-a2cb332725831efa", "mushroom4-2cca5f7434198cf1", "mushroom5-82f0c56cc5190e9e", "oven-d082997d1bd13f8e" ] }, { "key": "P03-20240217-192543:49b81f6e0580ef921f6c11ec57d87452dd8309fb", "video_id": "P03-20240217-192543", "question": "What is the ratio of eggplant slices to mushroom slices in the user\u2019s recipe?", "answer_choice_0": "20:9", "answer_choice_1": "1:5", "answer_choice_2": "9:20", "answer_choice_3": "5:1", "answer_choice_4": "9:19", "answer_id": 2, "answer": "9:20", "question_type": [ "Object Recognition", "Action Recognition", "Counting", "Numerical Reasoning" ], "objects_of_interest": [ "chair-ae03caabc008cafb", "table-a33ac46cdf21b10d", "chair2-0db6f6a3fc22c641", "aubergine-20c9c618fa8582c0", "broccoli-1046800e5ac9e9f6", "box of mushrooms-c8ed0de7c276fb90", "cauliflower-60cbba9c43361b16", "box of tomatoes-1e9bc904a15f4ddc", "bag of potatoes-3c2a8c3a953c896e", "bowl-f1e14dbc55594915", "bag of vegetables-5f21f12e8ede6d7e", "glass-2d8b5870be4d17ca", "coffee jug-7d07b54a26894e4a", "pink tea saucer-01fdf27ae6b4c2f4", "green tea saucer-58949162c8c7e96b", "v60 filter stand-bebc25cf16165a0a", "large bowl-76e1bb44fe1ff059", "plastic colander-01d1e453d8330cff", "red knife-7cd78e9152bfda9f", "saucepan-1ae923a3e236c771", "plastic padding-6af5261a2508f789", "small frying pan-53bba2b7ee816a80", "chopping board-7a330dc9feae0b4d", "white cloth-db4c999c952809af", "green glass-7d4b3c5d4d6c5022", "plastic water bottle-6fe607511c700e20", "cloth2-06a0860a291859d7", "knife-711b76a54098c634", "sponge-8a508859b42e89c2", "sunflower oil bottle-afec528692163e0b", "bottle of tomato sauce-ad4056643625219f", "small saucepan-b2583f8fb59800a8", "aubergine chunks1-dd76fed8cdbeb1f9", "aubergine chunks2-1262d276474e9376", "aubergine chunks3-53680550e6c37e87", "wooden spoon-8c3e202e4069e242", "dough scraper-9ca4d6e4d99afe45", "salt container-ff11afda402bd7eb", "herb container-caa3e734af663fb3", "black pepper grinder-8580a8a725cf289c", "mushroom1-392b1258da597bb3", "mushroom2-6ad27c0ea25e3a7e", "mushroom3-a2cb332725831efa", "mushroom4-2cca5f7434198cf1", "mushroom5-82f0c56cc5190e9e", "small saucepan2-5f26776f068a0c3a", "box of eggs-045b2504626151a4", "egg-0ddf9965588cb69c", "small cup-7305591d1f2d3c4e", "one garlic clove-bfc3b756d4c09592", "garlic mesh-95309fe3c60b3ca5", "garlic clove2-7e99f37e0c3473a1", "plate-2c8a296efdb1ab1b", "kitchen roll holder-1416823f3b2c2df6", "kitchen towel piece-be8a51519a505f1a", "ground garlic container-4f195d3f54e9ef4c", "peeled garlic clove-58d26742b7ba74ca", "slice of mushroom-76bd5fda55baa253", "garlic clove3-7d79cdc205528bc5", "ground garlic container lid-6c81336872669aad", "small bowl-4681bc924aa25206", "ladle-668b0e4edd9cd84b", "cone-5202d776756c5d1f", "large cup-8ce7a207ef526787", "small bowl2-fe8fbdb509fd2136", "olive oil tin cap-64d3f95de92b5455", "spoon-8d8f09014071bb0e", "tomato sauce cap-eb9f0d69cc49f849", "bag-adb3ca53674df30d", "food plastic bag-ab50eb05d089d7cf", "coffee grinder-1a49e057a10065ea", "glass olive dispencer-7ffffcdb088c428f", "orange cloth-a00209deb0226c52", "plastic water bottle2-cca5ffeec29524a3", "plastic cup-569d1cdc613949ba", "rice box-9631cf6a085c0a8a", "plastic measuring cup-e437978f66299245", "pizza oven-d082997d1bd13f8e", "box-db8fabef7497b2c4", "bag of semolina flour-338f5c2467d4cb04", "box of dough-192cccdb37d7b19c", "cheese grater-dd600872585677c5", "pizza peel-02628224f1c1d3fe", "two pizza spatulas-277428e76cac81ac", "box lid-b023925590e46846", "clip-98c90e797564ea14", "mozzarella bag-6f9c6f09d13ab016", "scissors-d37396785ab1d8f7", "mozzarella cheese-aff7c515cbf174e7", "chunks of mozzarella-38dafdec0ec9590c", "one slice of egg-8f15829ac5dfbd3f", "another slice of egg-2db503542d981640", "third slice of egg-5178fe2d5160b344", "fourth slice of egg-446e5c1a8127ff28", "red plate-32d950c2aa0ea8b0", "kitchen towel piece2-af33ff05ac2e60f4", "cutlery-35692999d3cbd20e", "tea towel-6df77a314affe872", "tea towel2-23b33810122204da", "pizza spatula2-8008bb1230333497", "fork-05d0abe7a5551fed", "knife2-eda0fb4051f7eb69", "fork2-ca3e03f919ec2342", "knife3-ee85eb04e657509b", "plate2-017fd5b7141d3c63", "large glass lid-d789692dfd9d2ffe", "washing up liquid bottle-064eabf0b77aa7c0", "kitchen towel piece3-b9f72292f67a7762", "pizza spatula1-6461eaf6cb9de906" ], "reasoning": "1. I located the moment the user starts cutting the eggplant at 03:23. At 03:24, the user cuts to the tip of the eggplant and discards it to the side. From 03:34 to 03:49 the user makes 8 cuts with the knife, for a total of 9 slices of eggplant . \n2. I then located the moment the user starts to cut the mushrooms (mushroom1-392b1258da597bb3, mushroom2-6ad27c0ea25e3a7e, mushroom3-a2cb332725831efa, mushroom4-2cca5f7434198cf1, mushroom5-82f0c56cc5190e9e) at 10:41. The user has 5 mushrooms (mushroom1-392b1258da597bb3, mushroom2-6ad27c0ea25e3a7e, mushroom3-a2cb332725831efa, mushroom4-2cca5f7434198cf1, mushroom5-82f0c56cc5190e9e) on a plate and makes 4 cuts with the knife (knife-711b76a54098c634) to each one from 10:41 to 11:13. I calculated the user cut 20 slices of mushrooms (5 x 4 = 20). \n3. The ratio is 9:20.", "referenced_object_masks": [ "mushroom1-392b1258da597bb3", "mushroom2-6ad27c0ea25e3a7e", "mushroom3-a2cb332725831efa", "mushroom4-2cca5f7434198cf1", "mushroom5-82f0c56cc5190e9e", "knife-711b76a54098c634" ] }, { "key": "P03-20240217-192543:5cada089e9eef1a85d0c37d47a9c7902c7e2e0d1", "video_id": "P03-20240217-192543", "question": "What is the second object I move from the fridge to the table?", "answer_choice_0": "Tomatoes", "answer_choice_1": "Cauliflower", "answer_choice_2": "Mushrooms", "answer_choice_3": "Broccoli", "answer_choice_4": "Eggplant", "answer_id": 2, "answer": "Mushrooms", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "chair-ae03caabc008cafb", "table-a33ac46cdf21b10d", "chair2-0db6f6a3fc22c641", "aubergine-20c9c618fa8582c0", "broccoli-1046800e5ac9e9f6", "box of mushrooms-c8ed0de7c276fb90", "cauliflower-60cbba9c43361b16", "box of tomatoes-1e9bc904a15f4ddc", "bag of potatoes-3c2a8c3a953c896e", "bowl-f1e14dbc55594915", "bag of vegetables-5f21f12e8ede6d7e", "glass-2d8b5870be4d17ca", "coffee jug-7d07b54a26894e4a", "pink tea saucer-01fdf27ae6b4c2f4", "green tea saucer-58949162c8c7e96b", "v60 filter stand-bebc25cf16165a0a", "large bowl-76e1bb44fe1ff059", "plastic colander-01d1e453d8330cff", "red knife-7cd78e9152bfda9f", "saucepan-1ae923a3e236c771", "plastic padding-6af5261a2508f789", "small frying pan-53bba2b7ee816a80", "chopping board-7a330dc9feae0b4d", "white cloth-db4c999c952809af", "green glass-7d4b3c5d4d6c5022", "plastic water bottle-6fe607511c700e20", "cloth2-06a0860a291859d7", "knife-711b76a54098c634", "sponge-8a508859b42e89c2", "sunflower oil bottle-afec528692163e0b", "bottle of tomato sauce-ad4056643625219f", "small saucepan-b2583f8fb59800a8", "aubergine chunks1-dd76fed8cdbeb1f9", "aubergine chunks2-1262d276474e9376", "aubergine chunks3-53680550e6c37e87", "wooden spoon-8c3e202e4069e242", "dough scraper-9ca4d6e4d99afe45", "salt container-ff11afda402bd7eb", "herb container-caa3e734af663fb3", "black pepper grinder-8580a8a725cf289c", "mushroom1-392b1258da597bb3", "mushroom2-6ad27c0ea25e3a7e", "mushroom3-a2cb332725831efa", "mushroom4-2cca5f7434198cf1", "mushroom5-82f0c56cc5190e9e", "small saucepan2-5f26776f068a0c3a", "box of eggs-045b2504626151a4", "egg-0ddf9965588cb69c", "small cup-7305591d1f2d3c4e", "one garlic clove-bfc3b756d4c09592", "garlic mesh-95309fe3c60b3ca5", "garlic clove2-7e99f37e0c3473a1", "plate-2c8a296efdb1ab1b", "kitchen roll holder-1416823f3b2c2df6", "kitchen towel piece-be8a51519a505f1a", "ground garlic container-4f195d3f54e9ef4c", "peeled garlic clove-58d26742b7ba74ca", "slice of mushroom-76bd5fda55baa253", "garlic clove3-7d79cdc205528bc5", "ground garlic container lid-6c81336872669aad", "small bowl-4681bc924aa25206", "ladle-668b0e4edd9cd84b", "cone-5202d776756c5d1f", "large cup-8ce7a207ef526787", "small bowl2-fe8fbdb509fd2136", "olive oil tin cap-64d3f95de92b5455", "spoon-8d8f09014071bb0e", "tomato sauce cap-eb9f0d69cc49f849", "bag-adb3ca53674df30d", "food plastic bag-ab50eb05d089d7cf", "coffee grinder-1a49e057a10065ea", "glass olive dispencer-7ffffcdb088c428f", "orange cloth-a00209deb0226c52", "plastic water bottle2-cca5ffeec29524a3", "plastic cup-569d1cdc613949ba", "rice box-9631cf6a085c0a8a", "plastic measuring cup-e437978f66299245", "pizza oven-d082997d1bd13f8e", "box-db8fabef7497b2c4", "bag of semolina flour-338f5c2467d4cb04", "box of dough-192cccdb37d7b19c", "cheese grater-dd600872585677c5", "pizza peel-02628224f1c1d3fe", "two pizza spatulas-277428e76cac81ac", "box lid-b023925590e46846", "clip-98c90e797564ea14", "mozzarella bag-6f9c6f09d13ab016", "scissors-d37396785ab1d8f7", "mozzarella cheese-aff7c515cbf174e7", "chunks of mozzarella-38dafdec0ec9590c", "one slice of egg-8f15829ac5dfbd3f", "another slice of egg-2db503542d981640", "third slice of egg-5178fe2d5160b344", "fourth slice of egg-446e5c1a8127ff28", "red plate-32d950c2aa0ea8b0", "kitchen towel piece2-af33ff05ac2e60f4", "cutlery-35692999d3cbd20e", "tea towel-6df77a314affe872", "tea towel2-23b33810122204da", "pizza spatula2-8008bb1230333497", "fork-05d0abe7a5551fed", "knife2-eda0fb4051f7eb69", "fork2-ca3e03f919ec2342", "knife3-ee85eb04e657509b", "plate2-017fd5b7141d3c63", "large glass lid-d789692dfd9d2ffe", "washing up liquid bottle-064eabf0b77aa7c0", "kitchen towel piece3-b9f72292f67a7762", "pizza spatula1-6461eaf6cb9de906" ], "reasoning": "1. I locate the fridge and the table (table-a33ac46cdf21b10d) at 00:00 to the left and right of the user respectively.\n2. I determine that the user opens the fridge at 00:14.\n3. I determine that the user takes the eggplant (aubergine-20c9c618fa8582c0) from the fridge and places it on the table (table-a33ac46cdf21b10d) from 00:19 to 00:23.\n4. I determine that the user takes the mushrooms (box of mushrooms-c8ed0de7c276fb90) from the fridge and places them on the table (table-a33ac46cdf21b10d) from 00:26 to 00:29.", "referenced_object_masks": [ "box of mushrooms-c8ed0de7c276fb90", "aubergine-20c9c618fa8582c0", "table-a33ac46cdf21b10d" ] }, { "key": "P03-20240217-192543:6823a4b90b70c915547ae56af130b87071578c05", "video_id": "P03-20240217-192543", "question": "How many times do I transfer eggplant from the cutting board to the pan?", "answer_choice_0": "2", "answer_choice_1": "3", "answer_choice_2": "4", "answer_choice_3": "5", "answer_choice_4": "1", "answer_id": 2, "answer": "4", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Counting", "Temporal Reasoning" ], "objects_of_interest": [ "chair-ae03caabc008cafb", "table-a33ac46cdf21b10d", "chair2-0db6f6a3fc22c641", "aubergine-20c9c618fa8582c0", "broccoli-1046800e5ac9e9f6", "box of mushrooms-c8ed0de7c276fb90", "cauliflower-60cbba9c43361b16", "box of tomatoes-1e9bc904a15f4ddc", "bag of potatoes-3c2a8c3a953c896e", "bowl-f1e14dbc55594915", "bag of vegetables-5f21f12e8ede6d7e", "glass-2d8b5870be4d17ca", "coffee jug-7d07b54a26894e4a", "pink tea saucer-01fdf27ae6b4c2f4", "green tea saucer-58949162c8c7e96b", "v60 filter stand-bebc25cf16165a0a", "large bowl-76e1bb44fe1ff059", "plastic colander-01d1e453d8330cff", "red knife-7cd78e9152bfda9f", "saucepan-1ae923a3e236c771", "plastic padding-6af5261a2508f789", "small frying pan-53bba2b7ee816a80", "chopping board-7a330dc9feae0b4d", "white cloth-db4c999c952809af", "green glass-7d4b3c5d4d6c5022", "plastic water bottle-6fe607511c700e20", "cloth2-06a0860a291859d7", "knife-711b76a54098c634", "sponge-8a508859b42e89c2", "sunflower oil bottle-afec528692163e0b", "bottle of tomato sauce-ad4056643625219f", "small saucepan-b2583f8fb59800a8", "aubergine chunks1-dd76fed8cdbeb1f9", "aubergine chunks2-1262d276474e9376", "aubergine chunks3-53680550e6c37e87", "wooden spoon-8c3e202e4069e242", "dough scraper-9ca4d6e4d99afe45", "salt container-ff11afda402bd7eb", "herb container-caa3e734af663fb3", "black pepper grinder-8580a8a725cf289c", "mushroom1-392b1258da597bb3", "mushroom2-6ad27c0ea25e3a7e", "mushroom3-a2cb332725831efa", "mushroom4-2cca5f7434198cf1", "mushroom5-82f0c56cc5190e9e", "small saucepan2-5f26776f068a0c3a", "box of eggs-045b2504626151a4", "egg-0ddf9965588cb69c", "small cup-7305591d1f2d3c4e", "one garlic clove-bfc3b756d4c09592", "garlic mesh-95309fe3c60b3ca5", "garlic clove2-7e99f37e0c3473a1", "plate-2c8a296efdb1ab1b", "kitchen roll holder-1416823f3b2c2df6", "kitchen towel piece-be8a51519a505f1a", "ground garlic container-4f195d3f54e9ef4c", "peeled garlic clove-58d26742b7ba74ca", "slice of mushroom-76bd5fda55baa253", "garlic clove3-7d79cdc205528bc5", "ground garlic container lid-6c81336872669aad", "small bowl-4681bc924aa25206", "ladle-668b0e4edd9cd84b", "cone-5202d776756c5d1f", "large cup-8ce7a207ef526787", "small bowl2-fe8fbdb509fd2136", "olive oil tin cap-64d3f95de92b5455", "spoon-8d8f09014071bb0e", "tomato sauce cap-eb9f0d69cc49f849", "bag-adb3ca53674df30d", "food plastic bag-ab50eb05d089d7cf", "coffee grinder-1a49e057a10065ea", "glass olive dispencer-7ffffcdb088c428f", "orange cloth-a00209deb0226c52", "plastic water bottle2-cca5ffeec29524a3", "plastic cup-569d1cdc613949ba", "rice box-9631cf6a085c0a8a", "plastic measuring cup-e437978f66299245", "pizza oven-d082997d1bd13f8e", "box-db8fabef7497b2c4", "bag of semolina flour-338f5c2467d4cb04", "box of dough-192cccdb37d7b19c", "cheese grater-dd600872585677c5", "pizza peel-02628224f1c1d3fe", "two pizza spatulas-277428e76cac81ac", "box lid-b023925590e46846", "clip-98c90e797564ea14", "mozzarella bag-6f9c6f09d13ab016", "scissors-d37396785ab1d8f7", "mozzarella cheese-aff7c515cbf174e7", "chunks of mozzarella-38dafdec0ec9590c", "one slice of egg-8f15829ac5dfbd3f", "another slice of egg-2db503542d981640", "third slice of egg-5178fe2d5160b344", "fourth slice of egg-446e5c1a8127ff28", "red plate-32d950c2aa0ea8b0", "kitchen towel piece2-af33ff05ac2e60f4", "cutlery-35692999d3cbd20e", "tea towel-6df77a314affe872", "tea towel2-23b33810122204da", "pizza spatula2-8008bb1230333497", "fork-05d0abe7a5551fed", "knife2-eda0fb4051f7eb69", "fork2-ca3e03f919ec2342", "knife3-ee85eb04e657509b", "plate2-017fd5b7141d3c63", "large glass lid-d789692dfd9d2ffe", "washing up liquid bottle-064eabf0b77aa7c0", "kitchen towel piece3-b9f72292f67a7762", "pizza spatula1-6461eaf6cb9de906" ], "reasoning": "1. I locate the eggplant (aubergine-20c9c618fa8582c0) at 00:19 as the user removes it from the fridge . \n2. At 06:00, I locate the pan (small frying pan-53bba2b7ee816a80) directly in front of the user as they hold a cutting board full of cut-up eggplant (aubergine chunks1-dd76fed8cdbeb1f9, aubergine chunks2-1262d276474e9376, aubergine chunks3-53680550e6c37e87).\n3. Between 06:00 and 06:08, I observe the user transfer the cut-up eggplant (aubergine chunks1-dd76fed8cdbeb1f9, aubergine chunks2-1262d276474e9376, aubergine chunks3-53680550e6c37e87) to the pan (small frying pan-53bba2b7ee816a80) 4 times.", "referenced_object_masks": [ "small frying pan-53bba2b7ee816a80", "aubergine-20c9c618fa8582c0", "aubergine chunks1-dd76fed8cdbeb1f9", "aubergine chunks2-1262d276474e9376", "aubergine chunks3-53680550e6c37e87" ] }, { "key": "P03-20240217-192543:6a52706322910160db8b8ec132cc08851a76e7e3", "video_id": "P03-20240217-192543", "question": "What is the first thing I touch with my right hand after turning off the fire alarm?", "answer_choice_0": "Table", "answer_choice_1": "Keys", "answer_choice_2": "Door handle", "answer_choice_3": "Pizza spatula", "answer_choice_4": "Chair", "answer_id": 1, "answer": "Keys", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "chair-ae03caabc008cafb", "table-a33ac46cdf21b10d", "chair2-0db6f6a3fc22c641", "aubergine-20c9c618fa8582c0", "broccoli-1046800e5ac9e9f6", "box of mushrooms-c8ed0de7c276fb90", "cauliflower-60cbba9c43361b16", "box of tomatoes-1e9bc904a15f4ddc", "bag of potatoes-3c2a8c3a953c896e", "bowl-f1e14dbc55594915", "bag of vegetables-5f21f12e8ede6d7e", "glass-2d8b5870be4d17ca", "coffee jug-7d07b54a26894e4a", "pink tea saucer-01fdf27ae6b4c2f4", "green tea saucer-58949162c8c7e96b", "v60 filter stand-bebc25cf16165a0a", "large bowl-76e1bb44fe1ff059", "plastic colander-01d1e453d8330cff", "red knife-7cd78e9152bfda9f", "saucepan-1ae923a3e236c771", "plastic padding-6af5261a2508f789", "small frying pan-53bba2b7ee816a80", "chopping board-7a330dc9feae0b4d", "white cloth-db4c999c952809af", "green glass-7d4b3c5d4d6c5022", "plastic water bottle-6fe607511c700e20", "cloth2-06a0860a291859d7", "knife-711b76a54098c634", "sponge-8a508859b42e89c2", "sunflower oil bottle-afec528692163e0b", "bottle of tomato sauce-ad4056643625219f", "small saucepan-b2583f8fb59800a8", "aubergine chunks1-dd76fed8cdbeb1f9", "aubergine chunks2-1262d276474e9376", "aubergine chunks3-53680550e6c37e87", "wooden spoon-8c3e202e4069e242", "dough scraper-9ca4d6e4d99afe45", "salt container-ff11afda402bd7eb", "herb container-caa3e734af663fb3", "black pepper grinder-8580a8a725cf289c", "mushroom1-392b1258da597bb3", "mushroom2-6ad27c0ea25e3a7e", "mushroom3-a2cb332725831efa", "mushroom4-2cca5f7434198cf1", "mushroom5-82f0c56cc5190e9e", "small saucepan2-5f26776f068a0c3a", "box of eggs-045b2504626151a4", "egg-0ddf9965588cb69c", "small cup-7305591d1f2d3c4e", "one garlic clove-bfc3b756d4c09592", "garlic mesh-95309fe3c60b3ca5", "garlic clove2-7e99f37e0c3473a1", "plate-2c8a296efdb1ab1b", "kitchen roll holder-1416823f3b2c2df6", "kitchen towel piece-be8a51519a505f1a", "ground garlic container-4f195d3f54e9ef4c", "peeled garlic clove-58d26742b7ba74ca", "slice of mushroom-76bd5fda55baa253", "garlic clove3-7d79cdc205528bc5", "ground garlic container lid-6c81336872669aad", "small bowl-4681bc924aa25206", "ladle-668b0e4edd9cd84b", "cone-5202d776756c5d1f", "large cup-8ce7a207ef526787", "small bowl2-fe8fbdb509fd2136", "olive oil tin cap-64d3f95de92b5455", "spoon-8d8f09014071bb0e", "tomato sauce cap-eb9f0d69cc49f849", "bag-adb3ca53674df30d", "food plastic bag-ab50eb05d089d7cf", "coffee grinder-1a49e057a10065ea", "glass olive dispencer-7ffffcdb088c428f", "orange cloth-a00209deb0226c52", "plastic water bottle2-cca5ffeec29524a3", "plastic cup-569d1cdc613949ba", "rice box-9631cf6a085c0a8a", "plastic measuring cup-e437978f66299245", "pizza oven-d082997d1bd13f8e", "box-db8fabef7497b2c4", "bag of semolina flour-338f5c2467d4cb04", "box of dough-192cccdb37d7b19c", "cheese grater-dd600872585677c5", "pizza peel-02628224f1c1d3fe", "two pizza spatulas-277428e76cac81ac", "box lid-b023925590e46846", "clip-98c90e797564ea14", "mozzarella bag-6f9c6f09d13ab016", "scissors-d37396785ab1d8f7", "mozzarella cheese-aff7c515cbf174e7", "chunks of mozzarella-38dafdec0ec9590c", "one slice of egg-8f15829ac5dfbd3f", "another slice of egg-2db503542d981640", "third slice of egg-5178fe2d5160b344", "fourth slice of egg-446e5c1a8127ff28", "red plate-32d950c2aa0ea8b0", "kitchen towel piece2-af33ff05ac2e60f4", "cutlery-35692999d3cbd20e", "tea towel-6df77a314affe872", "tea towel2-23b33810122204da", "pizza spatula2-8008bb1230333497", "fork-05d0abe7a5551fed", "knife2-eda0fb4051f7eb69", "fork2-ca3e03f919ec2342", "knife3-ee85eb04e657509b", "plate2-017fd5b7141d3c63", "large glass lid-d789692dfd9d2ffe", "washing up liquid bottle-064eabf0b77aa7c0", "kitchen towel piece3-b9f72292f67a7762", "pizza spatula1-6461eaf6cb9de906" ], "reasoning": "1. I locate the fire alarm at 55:32 on the ceiling above the user.\n2. I determine that the user turns off the fire alarm at 55:41.\n3. At 55:45, I observe the user approach the door to the left of the counter.\n4. I observe the user place their right hand on a set of keys in the door lock .", "referenced_object_masks": [] }, { "key": "P03-20240217-192543:6a5a246e7c391d431b6981a3a035b4e661fe27c9", "video_id": "P03-20240217-192543", "question": "What is the 3rd non-food item I use in front of the water bottle that I move to another area?", "answer_choice_0": "Knife", "answer_choice_1": "Cutting board", "answer_choice_2": "Can opener", "answer_choice_3": "Rag", "answer_choice_4": "Spoon", "answer_id": 3, "answer": "Rag", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Counting", "Situational Awareness" ], "objects_of_interest": [ "chair-ae03caabc008cafb", "table-a33ac46cdf21b10d", "chair2-0db6f6a3fc22c641", "aubergine-20c9c618fa8582c0", "broccoli-1046800e5ac9e9f6", "box of mushrooms-c8ed0de7c276fb90", "cauliflower-60cbba9c43361b16", "box of tomatoes-1e9bc904a15f4ddc", "bag of potatoes-3c2a8c3a953c896e", "bowl-f1e14dbc55594915", "bag of vegetables-5f21f12e8ede6d7e", "glass-2d8b5870be4d17ca", "coffee jug-7d07b54a26894e4a", "pink tea saucer-01fdf27ae6b4c2f4", "green tea saucer-58949162c8c7e96b", "v60 filter stand-bebc25cf16165a0a", "large bowl-76e1bb44fe1ff059", "plastic colander-01d1e453d8330cff", "red knife-7cd78e9152bfda9f", "saucepan-1ae923a3e236c771", "plastic padding-6af5261a2508f789", "small frying pan-53bba2b7ee816a80", "chopping board-7a330dc9feae0b4d", "white cloth-db4c999c952809af", "green glass-7d4b3c5d4d6c5022", "plastic water bottle-6fe607511c700e20", "cloth2-06a0860a291859d7", "knife-711b76a54098c634", "sponge-8a508859b42e89c2", "sunflower oil bottle-afec528692163e0b", "bottle of tomato sauce-ad4056643625219f", "small saucepan-b2583f8fb59800a8", "aubergine chunks1-dd76fed8cdbeb1f9", "aubergine chunks2-1262d276474e9376", "aubergine chunks3-53680550e6c37e87", "wooden spoon-8c3e202e4069e242", "dough scraper-9ca4d6e4d99afe45", "salt container-ff11afda402bd7eb", "herb container-caa3e734af663fb3", "black pepper grinder-8580a8a725cf289c", "mushroom1-392b1258da597bb3", "mushroom2-6ad27c0ea25e3a7e", "mushroom3-a2cb332725831efa", "mushroom4-2cca5f7434198cf1", "mushroom5-82f0c56cc5190e9e", "small saucepan2-5f26776f068a0c3a", "box of eggs-045b2504626151a4", "egg-0ddf9965588cb69c", "small cup-7305591d1f2d3c4e", "one garlic clove-bfc3b756d4c09592", "garlic mesh-95309fe3c60b3ca5", "garlic clove2-7e99f37e0c3473a1", "plate-2c8a296efdb1ab1b", "kitchen roll holder-1416823f3b2c2df6", "kitchen towel piece-be8a51519a505f1a", "ground garlic container-4f195d3f54e9ef4c", "peeled garlic clove-58d26742b7ba74ca", "slice of mushroom-76bd5fda55baa253", "garlic clove3-7d79cdc205528bc5", "ground garlic container lid-6c81336872669aad", "small bowl-4681bc924aa25206", "ladle-668b0e4edd9cd84b", "cone-5202d776756c5d1f", "large cup-8ce7a207ef526787", "small bowl2-fe8fbdb509fd2136", "olive oil tin cap-64d3f95de92b5455", "spoon-8d8f09014071bb0e", "tomato sauce cap-eb9f0d69cc49f849", "bag-adb3ca53674df30d", "food plastic bag-ab50eb05d089d7cf", "coffee grinder-1a49e057a10065ea", "glass olive dispencer-7ffffcdb088c428f", "orange cloth-a00209deb0226c52", "plastic water bottle2-cca5ffeec29524a3", "plastic cup-569d1cdc613949ba", "rice box-9631cf6a085c0a8a", "plastic measuring cup-e437978f66299245", "pizza oven-d082997d1bd13f8e", "box-db8fabef7497b2c4", "bag of semolina flour-338f5c2467d4cb04", "box of dough-192cccdb37d7b19c", "cheese grater-dd600872585677c5", "pizza peel-02628224f1c1d3fe", "two pizza spatulas-277428e76cac81ac", "box lid-b023925590e46846", "clip-98c90e797564ea14", "mozzarella bag-6f9c6f09d13ab016", "scissors-d37396785ab1d8f7", "mozzarella cheese-aff7c515cbf174e7", "chunks of mozzarella-38dafdec0ec9590c", "one slice of egg-8f15829ac5dfbd3f", "another slice of egg-2db503542d981640", "third slice of egg-5178fe2d5160b344", "fourth slice of egg-446e5c1a8127ff28", "red plate-32d950c2aa0ea8b0", "kitchen towel piece2-af33ff05ac2e60f4", "cutlery-35692999d3cbd20e", "tea towel-6df77a314affe872", "tea towel2-23b33810122204da", "pizza spatula2-8008bb1230333497", "fork-05d0abe7a5551fed", "knife2-eda0fb4051f7eb69", "fork2-ca3e03f919ec2342", "knife3-ee85eb04e657509b", "plate2-017fd5b7141d3c63", "large glass lid-d789692dfd9d2ffe", "washing up liquid bottle-064eabf0b77aa7c0", "kitchen towel piece3-b9f72292f67a7762", "pizza spatula1-6461eaf6cb9de906" ], "reasoning": "1. I located the segment where the user stands in front of the counter where the water bottle (plastic water bottle-6fe607511c700e20) is for the first time, which happens at 02:02. The user places a cutting board (chopping board-7a330dc9feae0b4d) in front of the water bottle (plastic water bottle-6fe607511c700e20). A white rag (white cloth-db4c999c952809af) is on the counter to the right of the water bottle (plastic water bottle-6fe607511c700e20) as well. \n2. From 03:24, the user employs a knife (knife-711b76a54098c634) in conjunction with the cutting board (chopping board-7a330dc9feae0b4d).\n3. At 05:18, the user grabs the cutting board (chopping board-7a330dc9feae0b4d) with the pieces of eggplant (aubergine chunks1-dd76fed8cdbeb1f9, aubergine chunks2-1262d276474e9376, aubergine chunks3-53680550e6c37e87) and moves it to the stove along with the knife (knife-711b76a54098c634), making them the first and second items moved. \n4. At 26:16, the user brings back the empty cutting board (chopping board-7a330dc9feae0b4d) and places it in front of the water bottle again (plastic water bottle-6fe607511c700e20). The user starts peeling a boiled egg (egg-0ddf9965588cb69c) at 26:17. At 27:41, the user grabs the white cloth (white cloth-db4c999c952809af) from the counter and moves it to the front of the microwave, making this the third item moved to another surface from this area.", "referenced_object_masks": [ "knife-711b76a54098c634", "aubergine chunks1-dd76fed8cdbeb1f9", "aubergine chunks2-1262d276474e9376", "aubergine chunks3-53680550e6c37e87", "egg-0ddf9965588cb69c", "chopping board-7a330dc9feae0b4d", "plastic water bottle-6fe607511c700e20", "white cloth-db4c999c952809af" ] }, { "key": "P03-20240217-192543:93fac129a82d375613a7822eb8f750a9f29cc20c", "video_id": "P03-20240217-192543", "question": "After opening the door, what is the first type of object that I hold in each hand?", "answer_choice_0": "Plate", "answer_choice_1": "Chair", "answer_choice_2": "Keys", "answer_choice_3": "Pizza spatula", "answer_choice_4": "Paper towel", "answer_id": 3, "answer": "Pizza spatula", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Temporal Reasoning", "Action Recognition" ], "objects_of_interest": [ "chair-ae03caabc008cafb", "table-a33ac46cdf21b10d", "chair2-0db6f6a3fc22c641", "aubergine-20c9c618fa8582c0", "broccoli-1046800e5ac9e9f6", "box of mushrooms-c8ed0de7c276fb90", "cauliflower-60cbba9c43361b16", "box of tomatoes-1e9bc904a15f4ddc", "bag of potatoes-3c2a8c3a953c896e", "bowl-f1e14dbc55594915", "bag of vegetables-5f21f12e8ede6d7e", "glass-2d8b5870be4d17ca", "coffee jug-7d07b54a26894e4a", "pink tea saucer-01fdf27ae6b4c2f4", "green tea saucer-58949162c8c7e96b", "v60 filter stand-bebc25cf16165a0a", "large bowl-76e1bb44fe1ff059", "plastic colander-01d1e453d8330cff", "red knife-7cd78e9152bfda9f", "saucepan-1ae923a3e236c771", "plastic padding-6af5261a2508f789", "small frying pan-53bba2b7ee816a80", "chopping board-7a330dc9feae0b4d", "white cloth-db4c999c952809af", "green glass-7d4b3c5d4d6c5022", "plastic water bottle-6fe607511c700e20", "cloth2-06a0860a291859d7", "knife-711b76a54098c634", "sponge-8a508859b42e89c2", "sunflower oil bottle-afec528692163e0b", "bottle of tomato sauce-ad4056643625219f", "small saucepan-b2583f8fb59800a8", "aubergine chunks1-dd76fed8cdbeb1f9", "aubergine chunks2-1262d276474e9376", "aubergine chunks3-53680550e6c37e87", "wooden spoon-8c3e202e4069e242", "dough scraper-9ca4d6e4d99afe45", "salt container-ff11afda402bd7eb", "herb container-caa3e734af663fb3", "black pepper grinder-8580a8a725cf289c", "mushroom1-392b1258da597bb3", "mushroom2-6ad27c0ea25e3a7e", "mushroom3-a2cb332725831efa", "mushroom4-2cca5f7434198cf1", "mushroom5-82f0c56cc5190e9e", "small saucepan2-5f26776f068a0c3a", "box of eggs-045b2504626151a4", "egg-0ddf9965588cb69c", "small cup-7305591d1f2d3c4e", "one garlic clove-bfc3b756d4c09592", "garlic mesh-95309fe3c60b3ca5", "garlic clove2-7e99f37e0c3473a1", "plate-2c8a296efdb1ab1b", "kitchen roll holder-1416823f3b2c2df6", "kitchen towel piece-be8a51519a505f1a", "ground garlic container-4f195d3f54e9ef4c", "peeled garlic clove-58d26742b7ba74ca", "slice of mushroom-76bd5fda55baa253", "garlic clove3-7d79cdc205528bc5", "ground garlic container lid-6c81336872669aad", "small bowl-4681bc924aa25206", "ladle-668b0e4edd9cd84b", "cone-5202d776756c5d1f", "large cup-8ce7a207ef526787", "small bowl2-fe8fbdb509fd2136", "olive oil tin cap-64d3f95de92b5455", "spoon-8d8f09014071bb0e", "tomato sauce cap-eb9f0d69cc49f849", "bag-adb3ca53674df30d", "food plastic bag-ab50eb05d089d7cf", "coffee grinder-1a49e057a10065ea", "glass olive dispencer-7ffffcdb088c428f", "orange cloth-a00209deb0226c52", "plastic water bottle2-cca5ffeec29524a3", "plastic cup-569d1cdc613949ba", "rice box-9631cf6a085c0a8a", "plastic measuring cup-e437978f66299245", "pizza oven-d082997d1bd13f8e", "box-db8fabef7497b2c4", "bag of semolina flour-338f5c2467d4cb04", "box of dough-192cccdb37d7b19c", "cheese grater-dd600872585677c5", "pizza peel-02628224f1c1d3fe", "two pizza spatulas-277428e76cac81ac", "box lid-b023925590e46846", "clip-98c90e797564ea14", "mozzarella bag-6f9c6f09d13ab016", "scissors-d37396785ab1d8f7", "mozzarella cheese-aff7c515cbf174e7", "chunks of mozzarella-38dafdec0ec9590c", "one slice of egg-8f15829ac5dfbd3f", "another slice of egg-2db503542d981640", "third slice of egg-5178fe2d5160b344", "fourth slice of egg-446e5c1a8127ff28", "red plate-32d950c2aa0ea8b0", "kitchen towel piece2-af33ff05ac2e60f4", "cutlery-35692999d3cbd20e", "tea towel-6df77a314affe872", "tea towel2-23b33810122204da", "pizza spatula2-8008bb1230333497", "fork-05d0abe7a5551fed", "knife2-eda0fb4051f7eb69", "fork2-ca3e03f919ec2342", "knife3-ee85eb04e657509b", "plate2-017fd5b7141d3c63", "large glass lid-d789692dfd9d2ffe", "washing up liquid bottle-064eabf0b77aa7c0", "kitchen towel piece3-b9f72292f67a7762", "pizza spatula1-6461eaf6cb9de906" ], "reasoning": "1. I locate the door at 01:04 to the left of the user.\n2. I observe the user open the door at 55:47.\n3. I observe the user pick up the pizza spatulas (pizza spatula2-8008bb1230333497, pizza spatula1-6461eaf6cb9de906) with their right hand.\n4. I observe the user transfer one of the pizza spatulas (pizza spatula2-8008bb1230333497) to their left hand.", "referenced_object_masks": [ "pizza spatula2-8008bb1230333497", "pizza spatula1-6461eaf6cb9de906" ] }, { "key": "P03-20240217-192543:accbde7571a5268164a3e2f71a226e70bdfcadd0", "video_id": "P03-20240217-192543", "question": "During my first time at the fridge, how many more items do I take out of the fridge than there are seats at the table?", "answer_choice_0": "5", "answer_choice_1": "4", "answer_choice_2": "2", "answer_choice_3": "1", "answer_choice_4": "3", "answer_id": 3, "answer": "1", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Counting", "Temporal Reasoning", "Numerical Reasoning" ], "objects_of_interest": [ "chair-ae03caabc008cafb", "table-a33ac46cdf21b10d", "chair2-0db6f6a3fc22c641", "aubergine-20c9c618fa8582c0", "broccoli-1046800e5ac9e9f6", "box of mushrooms-c8ed0de7c276fb90", "cauliflower-60cbba9c43361b16", "box of tomatoes-1e9bc904a15f4ddc", "bag of potatoes-3c2a8c3a953c896e", "bowl-f1e14dbc55594915", "bag of vegetables-5f21f12e8ede6d7e", "glass-2d8b5870be4d17ca", "coffee jug-7d07b54a26894e4a", "pink tea saucer-01fdf27ae6b4c2f4", "green tea saucer-58949162c8c7e96b", "v60 filter stand-bebc25cf16165a0a", "large bowl-76e1bb44fe1ff059", "plastic colander-01d1e453d8330cff", "red knife-7cd78e9152bfda9f", "saucepan-1ae923a3e236c771", "plastic padding-6af5261a2508f789", "small frying pan-53bba2b7ee816a80", "chopping board-7a330dc9feae0b4d", "white cloth-db4c999c952809af", "green glass-7d4b3c5d4d6c5022", "plastic water bottle-6fe607511c700e20", "cloth2-06a0860a291859d7", "knife-711b76a54098c634", "sponge-8a508859b42e89c2", "sunflower oil bottle-afec528692163e0b", "bottle of tomato sauce-ad4056643625219f", "small saucepan-b2583f8fb59800a8", "aubergine chunks1-dd76fed8cdbeb1f9", "aubergine chunks2-1262d276474e9376", "aubergine chunks3-53680550e6c37e87", "wooden spoon-8c3e202e4069e242", "dough scraper-9ca4d6e4d99afe45", "salt container-ff11afda402bd7eb", "herb container-caa3e734af663fb3", "black pepper grinder-8580a8a725cf289c", "mushroom1-392b1258da597bb3", "mushroom2-6ad27c0ea25e3a7e", "mushroom3-a2cb332725831efa", "mushroom4-2cca5f7434198cf1", "mushroom5-82f0c56cc5190e9e", "small saucepan2-5f26776f068a0c3a", "box of eggs-045b2504626151a4", "egg-0ddf9965588cb69c", "small cup-7305591d1f2d3c4e", "one garlic clove-bfc3b756d4c09592", "garlic mesh-95309fe3c60b3ca5", "garlic clove2-7e99f37e0c3473a1", "plate-2c8a296efdb1ab1b", "kitchen roll holder-1416823f3b2c2df6", "kitchen towel piece-be8a51519a505f1a", "ground garlic container-4f195d3f54e9ef4c", "peeled garlic clove-58d26742b7ba74ca", "slice of mushroom-76bd5fda55baa253", "garlic clove3-7d79cdc205528bc5", "ground garlic container lid-6c81336872669aad", "small bowl-4681bc924aa25206", "ladle-668b0e4edd9cd84b", "cone-5202d776756c5d1f", "large cup-8ce7a207ef526787", "small bowl2-fe8fbdb509fd2136", "olive oil tin cap-64d3f95de92b5455", "spoon-8d8f09014071bb0e", "tomato sauce cap-eb9f0d69cc49f849", "bag-adb3ca53674df30d", "food plastic bag-ab50eb05d089d7cf", "coffee grinder-1a49e057a10065ea", "glass olive dispencer-7ffffcdb088c428f", "orange cloth-a00209deb0226c52", "plastic water bottle2-cca5ffeec29524a3", "plastic cup-569d1cdc613949ba", "rice box-9631cf6a085c0a8a", "plastic measuring cup-e437978f66299245", "pizza oven-d082997d1bd13f8e", "box-db8fabef7497b2c4", "bag of semolina flour-338f5c2467d4cb04", "box of dough-192cccdb37d7b19c", "cheese grater-dd600872585677c5", "pizza peel-02628224f1c1d3fe", "two pizza spatulas-277428e76cac81ac", "box lid-b023925590e46846", "clip-98c90e797564ea14", "mozzarella bag-6f9c6f09d13ab016", "scissors-d37396785ab1d8f7", "mozzarella cheese-aff7c515cbf174e7", "chunks of mozzarella-38dafdec0ec9590c", "one slice of egg-8f15829ac5dfbd3f", "another slice of egg-2db503542d981640", "third slice of egg-5178fe2d5160b344", "fourth slice of egg-446e5c1a8127ff28", "red plate-32d950c2aa0ea8b0", "kitchen towel piece2-af33ff05ac2e60f4", "cutlery-35692999d3cbd20e", "tea towel-6df77a314affe872", "tea towel2-23b33810122204da", "pizza spatula2-8008bb1230333497", "fork-05d0abe7a5551fed", "knife2-eda0fb4051f7eb69", "fork2-ca3e03f919ec2342", "knife3-ee85eb04e657509b", "plate2-017fd5b7141d3c63", "large glass lid-d789692dfd9d2ffe", "washing up liquid bottle-064eabf0b77aa7c0", "kitchen towel piece3-b9f72292f67a7762", "pizza spatula1-6461eaf6cb9de906" ], "reasoning": "1. I locate the fridge and the table (table-a33ac46cdf21b10d) at 00:00 to the left and right of the user respectively.\n2. I identify 2 chairs (chair-ae03caabc008cafb, chair2-0db6f6a3fc22c641) at the table (table-a33ac46cdf21b10d) at 00:12.\n2. I observe the user open the fridge at 00:14.\n3. I observe the user close the fridge at 00:51.\n4. Between 00:14 and 00:51, I count 3 objects (aubergine-20c9c618fa8582c0, box of mushrooms-c8ed0de7c276fb90, bowl-f1e14dbc55594915) removed from the fridge by the user.\n5. I subtract 2 from 3 to get 1.", "referenced_object_masks": [ "aubergine-20c9c618fa8582c0", "box of mushrooms-c8ed0de7c276fb90", "bowl-f1e14dbc55594915", "chair-ae03caabc008cafb", "chair2-0db6f6a3fc22c641", "table-a33ac46cdf21b10d" ] }, { "key": "P03-20240217-192543:d4bd53cde70a6a889562455587d0732dc3442b2a", "video_id": "P03-20240217-192543", "question": "During the segment when I am putting eggplant cubes to drain on paper towels, how many more times do I use the wooden spoon to create an audible tap on the edge of the pan compared to the number of audible taps on the plate?", "answer_choice_0": "7", "answer_choice_1": "4", "answer_choice_2": "6", "answer_choice_3": "8", "answer_choice_4": "13", "answer_id": 3, "answer": "8", "question_type": [ "Object Recognition", "Action Recognition", "Counting", "Numerical Reasoning" ], "objects_of_interest": [ "chair-ae03caabc008cafb", "table-a33ac46cdf21b10d", "chair2-0db6f6a3fc22c641", "aubergine-20c9c618fa8582c0", "broccoli-1046800e5ac9e9f6", "box of mushrooms-c8ed0de7c276fb90", "cauliflower-60cbba9c43361b16", "box of tomatoes-1e9bc904a15f4ddc", "bag of potatoes-3c2a8c3a953c896e", "bowl-f1e14dbc55594915", "bag of vegetables-5f21f12e8ede6d7e", "glass-2d8b5870be4d17ca", "coffee jug-7d07b54a26894e4a", "pink tea saucer-01fdf27ae6b4c2f4", "green tea saucer-58949162c8c7e96b", "v60 filter stand-bebc25cf16165a0a", "large bowl-76e1bb44fe1ff059", "plastic colander-01d1e453d8330cff", "red knife-7cd78e9152bfda9f", "saucepan-1ae923a3e236c771", "plastic padding-6af5261a2508f789", "small frying pan-53bba2b7ee816a80", "chopping board-7a330dc9feae0b4d", "white cloth-db4c999c952809af", "green glass-7d4b3c5d4d6c5022", "plastic water bottle-6fe607511c700e20", "cloth2-06a0860a291859d7", "knife-711b76a54098c634", "sponge-8a508859b42e89c2", "sunflower oil bottle-afec528692163e0b", "bottle of tomato sauce-ad4056643625219f", "small saucepan-b2583f8fb59800a8", "aubergine chunks1-dd76fed8cdbeb1f9", "aubergine chunks2-1262d276474e9376", "aubergine chunks3-53680550e6c37e87", "wooden spoon-8c3e202e4069e242", "dough scraper-9ca4d6e4d99afe45", "salt container-ff11afda402bd7eb", "herb container-caa3e734af663fb3", "black pepper grinder-8580a8a725cf289c", "mushroom1-392b1258da597bb3", "mushroom2-6ad27c0ea25e3a7e", "mushroom3-a2cb332725831efa", "mushroom4-2cca5f7434198cf1", "mushroom5-82f0c56cc5190e9e", "small saucepan2-5f26776f068a0c3a", "box of eggs-045b2504626151a4", "egg-0ddf9965588cb69c", "small cup-7305591d1f2d3c4e", "one garlic clove-bfc3b756d4c09592", "garlic mesh-95309fe3c60b3ca5", "garlic clove2-7e99f37e0c3473a1", "plate-2c8a296efdb1ab1b", "kitchen roll holder-1416823f3b2c2df6", "kitchen towel piece-be8a51519a505f1a", "ground garlic container-4f195d3f54e9ef4c", "peeled garlic clove-58d26742b7ba74ca", "slice of mushroom-76bd5fda55baa253", "garlic clove3-7d79cdc205528bc5", "ground garlic container lid-6c81336872669aad", "small bowl-4681bc924aa25206", "ladle-668b0e4edd9cd84b", "cone-5202d776756c5d1f", "large cup-8ce7a207ef526787", "small bowl2-fe8fbdb509fd2136", "olive oil tin cap-64d3f95de92b5455", "spoon-8d8f09014071bb0e", "tomato sauce cap-eb9f0d69cc49f849", "bag-adb3ca53674df30d", "food plastic bag-ab50eb05d089d7cf", "coffee grinder-1a49e057a10065ea", "glass olive dispencer-7ffffcdb088c428f", "orange cloth-a00209deb0226c52", "plastic water bottle2-cca5ffeec29524a3", "plastic cup-569d1cdc613949ba", "rice box-9631cf6a085c0a8a", "plastic measuring cup-e437978f66299245", "pizza oven-d082997d1bd13f8e", "box-db8fabef7497b2c4", "bag of semolina flour-338f5c2467d4cb04", "box of dough-192cccdb37d7b19c", "cheese grater-dd600872585677c5", "pizza peel-02628224f1c1d3fe", "two pizza spatulas-277428e76cac81ac", "box lid-b023925590e46846", "clip-98c90e797564ea14", "mozzarella bag-6f9c6f09d13ab016", "scissors-d37396785ab1d8f7", "mozzarella cheese-aff7c515cbf174e7", "chunks of mozzarella-38dafdec0ec9590c", "one slice of egg-8f15829ac5dfbd3f", "another slice of egg-2db503542d981640", "third slice of egg-5178fe2d5160b344", "fourth slice of egg-446e5c1a8127ff28", "red plate-32d950c2aa0ea8b0", "kitchen towel piece2-af33ff05ac2e60f4", "cutlery-35692999d3cbd20e", "tea towel-6df77a314affe872", "tea towel2-23b33810122204da", "pizza spatula2-8008bb1230333497", "fork-05d0abe7a5551fed", "knife2-eda0fb4051f7eb69", "fork2-ca3e03f919ec2342", "knife3-ee85eb04e657509b", "plate2-017fd5b7141d3c63", "large glass lid-d789692dfd9d2ffe", "washing up liquid bottle-064eabf0b77aa7c0", "kitchen towel piece3-b9f72292f67a7762", "pizza spatula1-6461eaf6cb9de906" ], "reasoning": "1. I watched the whole video and saw the user putting the eggplant cubes (aubergine chunks1-dd76fed8cdbeb1f9, aubergine chunks2-1262d276474e9376, aubergine chunks3-53680550e6c37e87) out of the frying pan (small frying pan-53bba2b7ee816a80) and onto the plate (plate-2c8a296efdb1ab1b) with the paper towels starting at 15:53.\n2. At 16:11, the user tapped the wooden spoon (wooden spoon-8c3e202e4069e242) on the side of the frying pan (small frying pan-53bba2b7ee816a80) 8 times, and all of these taps were audibly heard.\n3. At 16:14, the user tapped the wooden spoon (wooden spoon-8c3e202e4069e242) on the side of the frying pan (small frying pan-53bba2b7ee816a80) another 4 times, and all of these taps were audibly heard.\n4. 8 plus 4 is 12.\n5. At 16:19, the user tapped the wooden spoon (wooden spoon-8c3e202e4069e242) against the plate (plate-2c8a296efdb1ab1b) 4 times, and all of these taps were audibly heard. \n6. 12 - 4 is 8. So the user audibly tapped the wooden spoon (wooden spoon-8c3e202e4069e242) against the side of the frying pan (small frying pan-53bba2b7ee816a80) 8 more times than against the plate (plate-2c8a296efdb1ab1b).", "referenced_object_masks": [ "small frying pan-53bba2b7ee816a80", "plate-2c8a296efdb1ab1b", "wooden spoon-8c3e202e4069e242", "aubergine chunks1-dd76fed8cdbeb1f9", "aubergine chunks2-1262d276474e9376", "aubergine chunks3-53680550e6c37e87" ] }, { "key": "P03-20240217-192543:f76fd47d85cdc76cde391def659b22b0b47f515f", "video_id": "P03-20240217-192543", "question": "What is the third item I hold with my left hand after I put the eggplant to fry in the pan?", "answer_choice_0": "Dough scraper", "answer_choice_1": "Wooden spoon", "answer_choice_2": "Spatula", "answer_choice_3": "Frying pan", "answer_choice_4": "Sauce bottle", "answer_id": 0, "answer": "Dough scraper", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "chair-ae03caabc008cafb", "table-a33ac46cdf21b10d", "chair2-0db6f6a3fc22c641", "aubergine-20c9c618fa8582c0", "broccoli-1046800e5ac9e9f6", "box of mushrooms-c8ed0de7c276fb90", "cauliflower-60cbba9c43361b16", "box of tomatoes-1e9bc904a15f4ddc", "bag of potatoes-3c2a8c3a953c896e", "bowl-f1e14dbc55594915", "bag of vegetables-5f21f12e8ede6d7e", "glass-2d8b5870be4d17ca", "coffee jug-7d07b54a26894e4a", "pink tea saucer-01fdf27ae6b4c2f4", "green tea saucer-58949162c8c7e96b", "v60 filter stand-bebc25cf16165a0a", "large bowl-76e1bb44fe1ff059", "plastic colander-01d1e453d8330cff", "red knife-7cd78e9152bfda9f", "saucepan-1ae923a3e236c771", "plastic padding-6af5261a2508f789", "small frying pan-53bba2b7ee816a80", "chopping board-7a330dc9feae0b4d", "white cloth-db4c999c952809af", "green glass-7d4b3c5d4d6c5022", "plastic water bottle-6fe607511c700e20", "cloth2-06a0860a291859d7", "knife-711b76a54098c634", "sponge-8a508859b42e89c2", "sunflower oil bottle-afec528692163e0b", "bottle of tomato sauce-ad4056643625219f", "small saucepan-b2583f8fb59800a8", "aubergine chunks1-dd76fed8cdbeb1f9", "aubergine chunks2-1262d276474e9376", "aubergine chunks3-53680550e6c37e87", "wooden spoon-8c3e202e4069e242", "dough scraper-9ca4d6e4d99afe45", "salt container-ff11afda402bd7eb", "herb container-caa3e734af663fb3", "black pepper grinder-8580a8a725cf289c", "mushroom1-392b1258da597bb3", "mushroom2-6ad27c0ea25e3a7e", "mushroom3-a2cb332725831efa", "mushroom4-2cca5f7434198cf1", "mushroom5-82f0c56cc5190e9e", "small saucepan2-5f26776f068a0c3a", "box of eggs-045b2504626151a4", "egg-0ddf9965588cb69c", "small cup-7305591d1f2d3c4e", "one garlic clove-bfc3b756d4c09592", "garlic mesh-95309fe3c60b3ca5", "garlic clove2-7e99f37e0c3473a1", "plate-2c8a296efdb1ab1b", "kitchen roll holder-1416823f3b2c2df6", "kitchen towel piece-be8a51519a505f1a", "ground garlic container-4f195d3f54e9ef4c", "peeled garlic clove-58d26742b7ba74ca", "slice of mushroom-76bd5fda55baa253", "garlic clove3-7d79cdc205528bc5", "ground garlic container lid-6c81336872669aad", "small bowl-4681bc924aa25206", "ladle-668b0e4edd9cd84b", "cone-5202d776756c5d1f", "large cup-8ce7a207ef526787", "small bowl2-fe8fbdb509fd2136", "olive oil tin cap-64d3f95de92b5455", "spoon-8d8f09014071bb0e", "tomato sauce cap-eb9f0d69cc49f849", "bag-adb3ca53674df30d", "food plastic bag-ab50eb05d089d7cf", "coffee grinder-1a49e057a10065ea", "glass olive dispencer-7ffffcdb088c428f", "orange cloth-a00209deb0226c52", "plastic water bottle2-cca5ffeec29524a3", "plastic cup-569d1cdc613949ba", "rice box-9631cf6a085c0a8a", "plastic measuring cup-e437978f66299245", "pizza oven-d082997d1bd13f8e", "box-db8fabef7497b2c4", "bag of semolina flour-338f5c2467d4cb04", "box of dough-192cccdb37d7b19c", "cheese grater-dd600872585677c5", "pizza peel-02628224f1c1d3fe", "two pizza spatulas-277428e76cac81ac", "box lid-b023925590e46846", "clip-98c90e797564ea14", "mozzarella bag-6f9c6f09d13ab016", "scissors-d37396785ab1d8f7", "mozzarella cheese-aff7c515cbf174e7", "chunks of mozzarella-38dafdec0ec9590c", "one slice of egg-8f15829ac5dfbd3f", "another slice of egg-2db503542d981640", "third slice of egg-5178fe2d5160b344", "fourth slice of egg-446e5c1a8127ff28", "red plate-32d950c2aa0ea8b0", "kitchen towel piece2-af33ff05ac2e60f4", "cutlery-35692999d3cbd20e", "tea towel-6df77a314affe872", "tea towel2-23b33810122204da", "pizza spatula2-8008bb1230333497", "fork-05d0abe7a5551fed", "knife2-eda0fb4051f7eb69", "fork2-ca3e03f919ec2342", "knife3-ee85eb04e657509b", "plate2-017fd5b7141d3c63", "large glass lid-d789692dfd9d2ffe", "washing up liquid bottle-064eabf0b77aa7c0", "kitchen towel piece3-b9f72292f67a7762", "pizza spatula1-6461eaf6cb9de906" ], "reasoning": "1. I located the moment the user started to stir fry the eggplant chunks (aubergine chunks1-dd76fed8cdbeb1f9, aubergine chunks2-1262d276474e9376, aubergine chunks3-53680550e6c37e87) in the pan (small frying pan-53bba2b7ee816a80) at 06:01. \n2. At 06:15, the user opens a cabinet door with the right hand and grabs a spatula with the left hand at 06:16. The user releases the spatula handle at 06:16. This is the first item the user grabs with the left hand. \n3. From 06:17 to 06:19, the user moves left and opens and closes the top drawer to the left with the left hand. \n4. I watched as the user opened the second drawer with the left hand at 06:19 and grabbed a wooden spoon (wooden spoon-8c3e202e4069e242) at 06:22. 5. The user pulls the wooden spoon from the drawer with the left hand at 06:23 and a dough scraper (dough scraper-9ca4d6e4d99afe45) with the right hand. The wooden spoon (wooden spoon-8c3e202e4069e242) is the second item the user grabs with the left hand. \n6. I watched as the user closed the drawer at 06:24 and walked towards the left side of the stove. At 06:25, as the user walks toward the stove, the user switches the items he\u2019s holding in his hands, grabbing the dough scraper (dough scraper-9ca4d6e4d99afe45) with the left hand and the wooden spoon (wooden spoon-8c3e202e4069e242) with the right hand. \n7. The user then places the dough scraper on the cabinet at 06:26 with the left hand. Therefore, the third item the user grabs with the left hand after putting the eggplant chunks (aubergine chunks1-dd76fed8cdbeb1f9, aubergine chunks2-1262d276474e9376, aubergine chunks3-53680550e6c37e87) to fry is the dough scraper (dough scraper-9ca4d6e4d99afe45).", "referenced_object_masks": [ "wooden spoon-8c3e202e4069e242", "aubergine chunks1-dd76fed8cdbeb1f9", "aubergine chunks2-1262d276474e9376", "aubergine chunks3-53680550e6c37e87", "small frying pan-53bba2b7ee816a80", "dough scraper-9ca4d6e4d99afe45" ] }, { "key": "P03-20240217-210126:1d9bc607ae876135a2c11bbb640d0333173e7587", "video_id": "P03-20240217-210126", "question": "What is the product of the number of pieces of garbage I threw away times the number of plates I cleared from the table times the number of bowls I cleared from the table?", "answer_choice_0": "4", "answer_choice_1": "2", "answer_choice_2": "6", "answer_choice_3": "12", "answer_choice_4": "16", "answer_id": 3, "answer": "12", "question_type": [ "Numerical Reasoning", "Action Recognition", "Counting", "Object Recognition" ], "objects_of_interest": [ "chocolate wrapper-7c40e7a661fe0537", "tissue-b728288e7954e2e3", "small bit-5f01aaf8d98d1dd6", "dirty cutlery1-56af77f12281e096", "dirty cutlery2-3ca42bbb3c1a7ea3", "dirty cutlery3-a1bd8cb46f45887f", "dirty cutlery4-4e990f44be54ee95", "dishwasher brush-f18cad13c7c16701", "dirty plates-c34c467c514ced60", "dirty plates2-511970007ddc97d6", "lid-96da12d6669c2cee", "small bowl-67bec94b1b69f939", "pan-0a276a9f802ad488", "saucepan-d6164b5a60749848", "bowl-8f7d15c1a65c5f89", "saucepan2-bdb6ba6ccc79a622", "other bowl-92a1d64ade1e689e", "plate-7cac809986acca21", "other plate-2e5e6163ec957459" ], "reasoning": "1. I watched the video and noted each time an object was thrown out. From 00:02-00:06, the user picks up 3 bits of garbage: a chocolate wrapper (chocolate wrapper-7c40e7a661fe0537), a tissue (tissue-b728288e7954e2e3) that the user folds inside the wrapper (chocolate wrapper-7c40e7a661fe0537), and another piece of garbage (small bit-5f01aaf8d98d1dd6). The user throws these away at 00:09.\n2. I counted each plate that the user cleared from the table. From 00:54-01:01 the user picks up and brings 2 plates (dirty plates-c34c467c514ced60, dirty plates2-511970007ddc97d6) into the kitchen.\n5. I counted each bowl that the user cleared from the table. From 00:54-01:01, the user picks up a plate (dirty plates2-511970007ddc97d6) with a bowl (small bowl-67bec94b1b69f939) on it and brings it to the kitchen. From 02:05-02:09, the user picks up another bowl (bowl-8f7d15c1a65c5f89) and brings it to the kitchen. \n6. I multiplied the number of pieces of garbage the user threw out (3) times the number of plates cleared from the table (2) times the number of bowls cleared from the table (2): 3*2*2=12.", "referenced_object_masks": [ "bowl-8f7d15c1a65c5f89", "tissue-b728288e7954e2e3", "dirty plates2-511970007ddc97d6", "dirty plates-c34c467c514ced60", "small bowl-67bec94b1b69f939", "chocolate wrapper-7c40e7a661fe0537", "small bit-5f01aaf8d98d1dd6" ] }, { "key": "P03-20240217-210126:a39facbff025b262389acaa8269da08b8a300068", "video_id": "P03-20240217-210126", "question": "How many plates did I scrub with the scrubber?", "answer_choice_0": "3", "answer_choice_1": "1", "answer_choice_2": "0", "answer_choice_3": "4", "answer_choice_4": "2", "answer_id": 4, "answer": "2", "question_type": [ "Counting", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "chocolate wrapper-7c40e7a661fe0537", "tissue-b728288e7954e2e3", "small bit-5f01aaf8d98d1dd6", "dirty cutlery1-56af77f12281e096", "dirty cutlery2-3ca42bbb3c1a7ea3", "dirty cutlery3-a1bd8cb46f45887f", "dirty cutlery4-4e990f44be54ee95", "dishwasher brush-f18cad13c7c16701", "dirty plates-c34c467c514ced60", "dirty plates2-511970007ddc97d6", "lid-96da12d6669c2cee", "small bowl-67bec94b1b69f939", "pan-0a276a9f802ad488", "saucepan-d6164b5a60749848", "bowl-8f7d15c1a65c5f89", "saucepan2-bdb6ba6ccc79a622", "other bowl-92a1d64ade1e689e", "plate-7cac809986acca21", "other plate-2e5e6163ec957459" ], "reasoning": "1. I watched the video and counted each plate that was scrubbed with the scrubber (dishwasher brush-f18cad13c7c16701). \n2. I saw that the user scrubbed the first plate (dirty plates-c34c467c514ced60) with the scrubber (dishwasher brush-f18cad13c7c16701) from 01:02-01:07. \n3. I saw that the user scrubbed the second plate (dirty plates2-511970007ddc97d6) with the scrubber (dishwasher brush-f18cad13c7c16701) from 01:38-01:41. \n4. I did not see any more plates scrubbed with the scrubber (dishwasher brush-f18cad13c7c16701), therefore I counted a total of 2.", "referenced_object_masks": [ "dirty plates2-511970007ddc97d6", "dirty plates-c34c467c514ced60", "dishwasher brush-f18cad13c7c16701" ] }, { "key": "P03-20240217-210126:d0dbaca84aa632883e20ff2b270f6236a8e5bbb1", "video_id": "P03-20240217-210126", "question": "How much time passed between when I first picked up the scrubber and when I started scrubbing the pot with tomato sauce?", "answer_choice_0": "01:30", "answer_choice_1": "02:10", "answer_choice_2": "02:50", "answer_choice_3": "1:57", "answer_choice_4": "02:30", "answer_id": 3, "answer": "1:57", "question_type": [ "Object Recognition", "Temporal Reasoning", "Numerical Reasoning" ], "objects_of_interest": [ "chocolate wrapper-7c40e7a661fe0537", "tissue-b728288e7954e2e3", "small bit-5f01aaf8d98d1dd6", "dirty cutlery1-56af77f12281e096", "dirty cutlery2-3ca42bbb3c1a7ea3", "dirty cutlery3-a1bd8cb46f45887f", "dirty cutlery4-4e990f44be54ee95", "dishwasher brush-f18cad13c7c16701", "dirty plates-c34c467c514ced60", "dirty plates2-511970007ddc97d6", "lid-96da12d6669c2cee", "small bowl-67bec94b1b69f939", "pan-0a276a9f802ad488", "saucepan-d6164b5a60749848", "bowl-8f7d15c1a65c5f89", "saucepan2-bdb6ba6ccc79a622", "other bowl-92a1d64ade1e689e", "plate-7cac809986acca21", "other plate-2e5e6163ec957459" ], "reasoning": "1. I watched the video and observed the user pick up the scrubber (dishwasher brush-f18cad13c7c16701) for the first time at 00:21. \n2. I saw the user first pick up the pot with tomato sauce (saucepan2-bdb6ba6ccc79a622) at 02:11. \n3. At 02:18, I saw the user start to scrub the pot with tomato sauce (saucepan2-bdb6ba6ccc79a622) with the scrubber (dishwasher brush-f18cad13c7c16701). \n4. I subtracted 02:18-00:21=01:57.", "referenced_object_masks": [ "saucepan2-bdb6ba6ccc79a622", "dishwasher brush-f18cad13c7c16701" ] }, { "key": "P03-20240217-210126:de3f52fcfb93c31d91340aad64d58b12de9edb8d", "video_id": "P03-20240217-210126", "question": "When I first opened the dishwasher, where was the pot that I used to cook tomato sauce?", "answer_choice_0": "Above the dishwasher", "answer_choice_1": "Opposite the dishwasher", "answer_choice_2": "Right of the dishwasher", "answer_choice_3": "Left of the dishwasher", "answer_choice_4": "In the dishwasher", "answer_id": 1, "answer": "Opposite the dishwasher", "question_type": [ "Egocentric & Spatial Perception", "Situational Awareness", "Object Recognition", "Action Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "chocolate wrapper-7c40e7a661fe0537", "tissue-b728288e7954e2e3", "small bit-5f01aaf8d98d1dd6", "dirty cutlery1-56af77f12281e096", "dirty cutlery2-3ca42bbb3c1a7ea3", "dirty cutlery3-a1bd8cb46f45887f", "dirty cutlery4-4e990f44be54ee95", "dishwasher brush-f18cad13c7c16701", "dirty plates-c34c467c514ced60", "dirty plates2-511970007ddc97d6", "lid-96da12d6669c2cee", "small bowl-67bec94b1b69f939", "pan-0a276a9f802ad488", "saucepan-d6164b5a60749848", "bowl-8f7d15c1a65c5f89", "saucepan2-bdb6ba6ccc79a622", "other bowl-92a1d64ade1e689e", "plate-7cac809986acca21", "other plate-2e5e6163ec957459" ], "reasoning": "1. I watched the video and looked for when the user opened the dishwasher . This occurred from 00:16-00:18. \n2. I looked for when the user first picked up the pot with tomato sauce (saucepan2-bdb6ba6ccc79a622). This occurred at 02:11. \n3. I observed the surroundings from 02:11-02:14. I noted that at 02:11, the pot with tomato sauce (saucepan2-bdb6ba6ccc79a622) was on the stove to the left of the user. As the user turned, I noted that at 02:13 they were facing the dishwasher . And at 02:14 I noted the user was facing the sink, in between the dishwasher , now on their left, and the stove, now on their right. \n4. Since 02:11 was the first time the pot with tomato sauce (saucepan2-bdb6ba6ccc79a622) was touched, I determined that when the user first opened the dishwasher from 00:16-00:18, the pot with tomato sauce (saucepan2-bdb6ba6ccc79a622) was behind the user, and therefore opposite the dishwasher .", "referenced_object_masks": [ "saucepan2-bdb6ba6ccc79a622" ] }, { "key": "P03-20240217-210126:e7ccebe33beefa1b7ea83657d5a7d0374f45e05e", "video_id": "P03-20240217-210126", "question": "When I threw away the chocolate wrapper, how many degrees counterclockwise would I turn if I wanted to face the sink?", "answer_choice_0": "270", "answer_choice_1": "0", "answer_choice_2": "45", "answer_choice_3": "90", "answer_choice_4": "315", "answer_id": 0, "answer": "270", "question_type": [ "Egocentric & Spatial Perception", "Numerical Reasoning", "Object Recognition", "Situational Awareness", "Action Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "chocolate wrapper-7c40e7a661fe0537", "tissue-b728288e7954e2e3", "small bit-5f01aaf8d98d1dd6", "dirty cutlery1-56af77f12281e096", "dirty cutlery2-3ca42bbb3c1a7ea3", "dirty cutlery3-a1bd8cb46f45887f", "dirty cutlery4-4e990f44be54ee95", "dishwasher brush-f18cad13c7c16701", "dirty plates-c34c467c514ced60", "dirty plates2-511970007ddc97d6", "lid-96da12d6669c2cee", "small bowl-67bec94b1b69f939", "pan-0a276a9f802ad488", "saucepan-d6164b5a60749848", "bowl-8f7d15c1a65c5f89", "saucepan2-bdb6ba6ccc79a622", "other bowl-92a1d64ade1e689e", "plate-7cac809986acca21", "other plate-2e5e6163ec957459" ], "reasoning": "1. I watched the video and looked for when the user picks up the chocolate wrapper (chocolate wrapper-7c40e7a661fe0537). This occurred at 00:02. From 00:02-00:06, I saw the user put a tissue (tissue-b728288e7954e2e3) on the chocolate wrapper (chocolate wrapper-7c40e7a661fe0537) and then wrap and crumple the chocolate wrapper (chocolate wrapper-7c40e7a661fe0537) around the tissue (tissue-b728288e7954e2e3). \n2. From 00:06-00:07, I saw the user, holding the crumpled garbage , turn about 180 degrees left and crouch toward a trash can . Then, at 00:09, I saw the user throw away the crumpled garbage , including the chocolate wrapper (chocolate wrapper-7c40e7a661fe0537), into the trash can . \n3. From 00:09-00:10, I saw the user stand up and turn about 90 degrees to their right. I observed the sink straight ahead. \n4. Since the user saw the sink when turning 90 degrees clockwise, I subtracted 360-90 to find out how far they would have to turn counterclockwise to see the sink. 360-90=270.", "referenced_object_masks": [ "chocolate wrapper-7c40e7a661fe0537", "tissue-b728288e7954e2e3" ] }, { "key": "P03-20240217-210958:00c977aaed83d84d072cfe97b11318a3420b598b", "video_id": "P03-20240217-210958", "question": "After unjamming the dishwasher, what is the fourth unique item I handle other than the dishwasher and the object that was jamming it?", "answer_choice_0": "Large Bowl", "answer_choice_1": "Small Bowl", "answer_choice_2": "Small silver fork", "answer_choice_3": "Small silver knife", "answer_choice_4": "Dish brush", "answer_id": 4, "answer": "Dish brush", "question_type": [ "Numerical Reasoning", "Reading", "Counterfactual" ], "objects_of_interest": [ "knife-184d4fcac806f1f6", "small bowl-d7506f6043427514", "white bowl-63a9bfda1bf3f34b", "knife2-0ef68b39845dd1f8", "dishwasher brush-5dcf79fa622b3e10", "large bowl-301f1353cc4016fd", "small saucepan-8aaf0ab59d275137", "used kitchen roll-677cc83e279799d1", "spoon-6bde2ce481ceed16", "plate-3ad3d3f8372bc06b", "sponge-c17425c6f342f8aa", "white cloth-07a3dc1ff5f05466", "dishwasher tablet-4dc20cb87089e5d3", "small frying pan-cd5dd79cc2402496", "pizza peel-bcedfda3e55ac0d6", "washing up liquid bottle-50648ef22a7062b3", "pizza spatula1-b6bf5382ef08beb0", "pizza spatula2-cc2bec2a4e44efe3", "plastic box-d5b35a191777d1f2", "plastic box lid-fe97377b799386ad", "sink strainer-3202c768a4e2f37f", "cloth2-ba957fe9957f969e", "dough scraper-c22465f46d5e8eff", "pizza oven-8860315eea5698e5", "orange cloth-4e3125ae759cfc6c", "coffee grinder-da26efd8d9dbb285", "cleaner-0e92cdf383e33bc1", "kettle-a77eb92372c0788d", "microwave oven-247d931a640d9fd9", "red box-20aa93b87a0f4845", "small cup-d894577c7bc187e6", "soap tray-18a3b6de3fd43d98", "red glass-b97e64636f1eeb66", "green glass-c59a2df00897bd92", "tea towel1-0e18f9d4e9342683", "chair1-e178a45feabacf8f", "cloth bag-282f770ae1c40b2b", "stack of two glass ramekins-718e87cf4ef03ed9", "scale-7dbecec7b90b4920", "tea towel2-6c81551755ce0456" ], "reasoning": "1. First, I found the moment when the user unjammed the dishwasher by rearranging a knife (knife-184d4fcac806f1f6) that had fallen through the cutlery bin at 00:16. \n2. I then looked for the user to handle other unique objects and noticed the user grabbed a bowl (small bowl-d7506f6043427514) from the sink at 00:20.\n3. I noticed the user grabbed a second bowl (white bowl-63a9bfda1bf3f34b)at 00:22.\n4. I noticed the user grabbed the same knife (knife-184d4fcac806f1f6) that they originally unjammed from the dishwasher at 00:33, which I did not count.\n5. At 00:36, I saw the user grab a knife with a green handle (knife2-0ef68b39845dd1f8). I counted this as the third object. \n6. Finally, at 00:39, I saw the user adjust a dish brush (dishwasher brush-5dcf79fa622b3e10), which I counted as the fourth object.", "referenced_object_masks": [ "small bowl-d7506f6043427514", "dishwasher brush-5dcf79fa622b3e10", "white bowl-63a9bfda1bf3f34b", "knife-184d4fcac806f1f6", "knife2-0ef68b39845dd1f8" ] }, { "key": "P03-20240217-210958:21c5a951280ae21863f275bf0a3aeccccde34b3b", "video_id": "P03-20240217-210958", "question": "If I multiplied the number of bowls on the bottom shelf of the dishwasher by the time the user finishes loading it by 3, how many bowls would I have?", "answer_choice_0": "12", "answer_choice_1": "6", "answer_choice_2": "15", "answer_choice_3": "9", "answer_choice_4": "3", "answer_id": 3, "answer": "9", "question_type": [ "Numerical Reasoning", "Object Recognition", "Counterfactual" ], "objects_of_interest": [ "knife-184d4fcac806f1f6", "small bowl-d7506f6043427514", "white bowl-63a9bfda1bf3f34b", "knife2-0ef68b39845dd1f8", "dishwasher brush-5dcf79fa622b3e10", "large bowl-301f1353cc4016fd", "small saucepan-8aaf0ab59d275137", "used kitchen roll-677cc83e279799d1", "spoon-6bde2ce481ceed16", "plate-3ad3d3f8372bc06b", "sponge-c17425c6f342f8aa", "white cloth-07a3dc1ff5f05466", "dishwasher tablet-4dc20cb87089e5d3", "small frying pan-cd5dd79cc2402496", "pizza peel-bcedfda3e55ac0d6", "washing up liquid bottle-50648ef22a7062b3", "pizza spatula1-b6bf5382ef08beb0", "pizza spatula2-cc2bec2a4e44efe3", "plastic box-d5b35a191777d1f2", "plastic box lid-fe97377b799386ad", "sink strainer-3202c768a4e2f37f", "cloth2-ba957fe9957f969e", "dough scraper-c22465f46d5e8eff", "pizza oven-8860315eea5698e5", "orange cloth-4e3125ae759cfc6c", "coffee grinder-da26efd8d9dbb285", "cleaner-0e92cdf383e33bc1", "kettle-a77eb92372c0788d", "microwave oven-247d931a640d9fd9", "red box-20aa93b87a0f4845", "small cup-d894577c7bc187e6", "soap tray-18a3b6de3fd43d98", "red glass-b97e64636f1eeb66", "green glass-c59a2df00897bd92", "tea towel1-0e18f9d4e9342683", "chair1-e178a45feabacf8f", "cloth bag-282f770ae1c40b2b", "stack of two glass ramekins-718e87cf4ef03ed9", "scale-7dbecec7b90b4920", "tea towel2-6c81551755ce0456" ], "reasoning": "1. At 00:29, the user places a white bowl (white bowl-63a9bfda1bf3f34b) amongst 2 other white bowls on the bottom shelf of the dishwasher .\n2. This means that there are 3 bowls total in the bottom shelf of the dishwasher .\n3. 3x3=9.", "referenced_object_masks": [ "white bowl-63a9bfda1bf3f34b" ] }, { "key": "P03-20240217-210958:3767b112bc7d9d1c74fb6d5b4dbf503ef9f396a0", "video_id": "P03-20240217-210958", "question": "What is the difference in the time shown on the microwave when I put the pink spray away and when I'm scraping the right side of the pizza oven?", "answer_choice_0": "6 minutes", "answer_choice_1": "4 minutes", "answer_choice_2": "3 minutes", "answer_choice_3": "5 minutes", "answer_choice_4": "2 minutes", "answer_id": 3, "answer": "5 minutes", "question_type": [ "Action Recognition", "Numerical Reasoning", "State Changes", "Object Recognition", "Reading" ], "objects_of_interest": [ "knife-184d4fcac806f1f6", "small bowl-d7506f6043427514", "white bowl-63a9bfda1bf3f34b", "knife2-0ef68b39845dd1f8", "dishwasher brush-5dcf79fa622b3e10", "large bowl-301f1353cc4016fd", "small saucepan-8aaf0ab59d275137", "used kitchen roll-677cc83e279799d1", "spoon-6bde2ce481ceed16", "plate-3ad3d3f8372bc06b", "sponge-c17425c6f342f8aa", "white cloth-07a3dc1ff5f05466", "dishwasher tablet-4dc20cb87089e5d3", "small frying pan-cd5dd79cc2402496", "pizza peel-bcedfda3e55ac0d6", "washing up liquid bottle-50648ef22a7062b3", "pizza spatula1-b6bf5382ef08beb0", "pizza spatula2-cc2bec2a4e44efe3", "plastic box-d5b35a191777d1f2", "plastic box lid-fe97377b799386ad", "sink strainer-3202c768a4e2f37f", "cloth2-ba957fe9957f969e", "dough scraper-c22465f46d5e8eff", "pizza oven-8860315eea5698e5", "orange cloth-4e3125ae759cfc6c", "coffee grinder-da26efd8d9dbb285", "cleaner-0e92cdf383e33bc1", "kettle-a77eb92372c0788d", "microwave oven-247d931a640d9fd9", "red box-20aa93b87a0f4845", "small cup-d894577c7bc187e6", "soap tray-18a3b6de3fd43d98", "red glass-b97e64636f1eeb66", "green glass-c59a2df00897bd92", "tea towel1-0e18f9d4e9342683", "chair1-e178a45feabacf8f", "cloth bag-282f770ae1c40b2b", "stack of two glass ramekins-718e87cf4ef03ed9", "scale-7dbecec7b90b4920", "tea towel2-6c81551755ce0456" ], "reasoning": "1. I locate the timestamp of when the user is putting the pink spray (cleaner-0e92cdf383e33bc1) away. This occurs at 14:43. At that moment, I can see the time displayed on the microwave (microwave oven-247d931a640d9fd9) which reads, \"21:21\".\n2. Next, I locate the timestamp of when the user is scraping the right side of the pizza oven (pizza oven-8860315eea5698e5). This occurs between 09:20-09:29. At 09:29, I can see the time displayed on the microwave (microwave oven-247d931a640d9fd9) which reads, \"21:16\".\n3. Then, I subtract 21:21-21:16=5.\n4. Therefore, I determine the difference between the 2 times displayed on the microwave during the 2 occurrences is 5 minutes.", "referenced_object_masks": [ "pizza oven-8860315eea5698e5", "microwave oven-247d931a640d9fd9", "cleaner-0e92cdf383e33bc1" ] }, { "key": "P03-20240217-210958:4a23597aa46c6f1ca36c9c29fac0e241ada4c1f0", "video_id": "P03-20240217-210958", "question": "If I were facing the left cabinet underneath the sink and the bottom shelf was divided into 4 quadrants (like a graph), in what quadrant do I store my dishwasher tablets?", "answer_choice_0": "The dishwasher tablet isn't in the left cabinet", "answer_choice_1": "Lower right quadrant", "answer_choice_2": "Upper right quadrant", "answer_choice_3": "Lower left quadrant", "answer_choice_4": "Upper left quadrant", "answer_id": 3, "answer": "Lower left quadrant", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "knife-184d4fcac806f1f6", "small bowl-d7506f6043427514", "white bowl-63a9bfda1bf3f34b", "knife2-0ef68b39845dd1f8", "dishwasher brush-5dcf79fa622b3e10", "large bowl-301f1353cc4016fd", "small saucepan-8aaf0ab59d275137", "used kitchen roll-677cc83e279799d1", "spoon-6bde2ce481ceed16", "plate-3ad3d3f8372bc06b", "sponge-c17425c6f342f8aa", "white cloth-07a3dc1ff5f05466", "dishwasher tablet-4dc20cb87089e5d3", "small frying pan-cd5dd79cc2402496", "pizza peel-bcedfda3e55ac0d6", "washing up liquid bottle-50648ef22a7062b3", "pizza spatula1-b6bf5382ef08beb0", "pizza spatula2-cc2bec2a4e44efe3", "plastic box-d5b35a191777d1f2", "plastic box lid-fe97377b799386ad", "sink strainer-3202c768a4e2f37f", "cloth2-ba957fe9957f969e", "dough scraper-c22465f46d5e8eff", "pizza oven-8860315eea5698e5", "orange cloth-4e3125ae759cfc6c", "coffee grinder-da26efd8d9dbb285", "cleaner-0e92cdf383e33bc1", "kettle-a77eb92372c0788d", "microwave oven-247d931a640d9fd9", "red box-20aa93b87a0f4845", "small cup-d894577c7bc187e6", "soap tray-18a3b6de3fd43d98", "red glass-b97e64636f1eeb66", "green glass-c59a2df00897bd92", "tea towel1-0e18f9d4e9342683", "chair1-e178a45feabacf8f", "cloth bag-282f770ae1c40b2b", "stack of two glass ramekins-718e87cf4ef03ed9", "scale-7dbecec7b90b4920", "tea towel2-6c81551755ce0456" ], "reasoning": "1. At 01:56, the user can be seen taking out a dishwasher tablet (dishwasher tablet-4dc20cb87089e5d3) from the lower shelf in the left cabinet under the kitchen sink.\n2. If the user was facing this cabinet directly, I would describe where he takes the dishwasher tablet (dishwasher tablet-4dc20cb87089e5d3) as the lower left side of the cabinet.\n3. Therefore the user stores the dishwasher tablet (dishwasher tablet-4dc20cb87089e5d3) on the lower left quadrant of the bottom shelf.", "referenced_object_masks": [ "dishwasher tablet-4dc20cb87089e5d3" ] }, { "key": "P03-20240217-210958:8f065e03280279a8a324951932a58327f37dec04", "video_id": "P03-20240217-210958", "question": "In relation to where I'm standing when I'm wiping off my pizza oven's lid while it's closed, in what direction is my pizza oven's storage place?", "answer_choice_0": "Below me", "answer_choice_1": "On my right", "answer_choice_2": "Behind me", "answer_choice_3": "Above me", "answer_choice_4": "On my left", "answer_id": 2, "answer": "Behind me", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "knife-184d4fcac806f1f6", "small bowl-d7506f6043427514", "white bowl-63a9bfda1bf3f34b", "knife2-0ef68b39845dd1f8", "dishwasher brush-5dcf79fa622b3e10", "large bowl-301f1353cc4016fd", "small saucepan-8aaf0ab59d275137", "used kitchen roll-677cc83e279799d1", "spoon-6bde2ce481ceed16", "plate-3ad3d3f8372bc06b", "sponge-c17425c6f342f8aa", "white cloth-07a3dc1ff5f05466", "dishwasher tablet-4dc20cb87089e5d3", "small frying pan-cd5dd79cc2402496", "pizza peel-bcedfda3e55ac0d6", "washing up liquid bottle-50648ef22a7062b3", "pizza spatula1-b6bf5382ef08beb0", "pizza spatula2-cc2bec2a4e44efe3", "plastic box-d5b35a191777d1f2", "plastic box lid-fe97377b799386ad", "sink strainer-3202c768a4e2f37f", "cloth2-ba957fe9957f969e", "dough scraper-c22465f46d5e8eff", "pizza oven-8860315eea5698e5", "orange cloth-4e3125ae759cfc6c", "coffee grinder-da26efd8d9dbb285", "cleaner-0e92cdf383e33bc1", "kettle-a77eb92372c0788d", "microwave oven-247d931a640d9fd9", "red box-20aa93b87a0f4845", "small cup-d894577c7bc187e6", "soap tray-18a3b6de3fd43d98", "red glass-b97e64636f1eeb66", "green glass-c59a2df00897bd92", "tea towel1-0e18f9d4e9342683", "chair1-e178a45feabacf8f", "cloth bag-282f770ae1c40b2b", "stack of two glass ramekins-718e87cf4ef03ed9", "scale-7dbecec7b90b4920", "tea towel2-6c81551755ce0456" ], "reasoning": "1. First, I determine when the user is wiping off the top of their pizza oven (pizza oven-8860315eea5698e5) while it is closed. I located this section at 10:45. \n2. I keep watching and when they're done, the user turns left at 10:48, before turning 180 degrees and walking to the cabinet behind them. At 10:52, the user puts the pizza oven away (pizza oven-8860315eea5698e5) inside this cabinet.\n3. This cabinet/storage place is positioned directly behind the user.", "referenced_object_masks": [ "pizza oven-8860315eea5698e5" ] }, { "key": "P03-20240217-210958:aab30c63490a046122213e8b08d4c72051a51e4d", "video_id": "P03-20240217-210958", "question": "While I'm cleaning the pizza oven with a cloth, in which direction do I wipe just before moving to the inside of the lid?", "answer_choice_0": "Once clockwise", "answer_choice_1": "Once backwards", "answer_choice_2": "3 times clockwise", "answer_choice_3": "Twice up and down", "answer_choice_4": "Twice down and up", "answer_id": 0, "answer": "Once clockwise", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Counting", "Object Recognition" ], "objects_of_interest": [ "knife-184d4fcac806f1f6", "small bowl-d7506f6043427514", "white bowl-63a9bfda1bf3f34b", "knife2-0ef68b39845dd1f8", "dishwasher brush-5dcf79fa622b3e10", "large bowl-301f1353cc4016fd", "small saucepan-8aaf0ab59d275137", "used kitchen roll-677cc83e279799d1", "spoon-6bde2ce481ceed16", "plate-3ad3d3f8372bc06b", "sponge-c17425c6f342f8aa", "white cloth-07a3dc1ff5f05466", "dishwasher tablet-4dc20cb87089e5d3", "small frying pan-cd5dd79cc2402496", "pizza peel-bcedfda3e55ac0d6", "washing up liquid bottle-50648ef22a7062b3", "pizza spatula1-b6bf5382ef08beb0", "pizza spatula2-cc2bec2a4e44efe3", "plastic box-d5b35a191777d1f2", "plastic box lid-fe97377b799386ad", "sink strainer-3202c768a4e2f37f", "cloth2-ba957fe9957f969e", "dough scraper-c22465f46d5e8eff", "pizza oven-8860315eea5698e5", "orange cloth-4e3125ae759cfc6c", "coffee grinder-da26efd8d9dbb285", "cleaner-0e92cdf383e33bc1", "kettle-a77eb92372c0788d", "microwave oven-247d931a640d9fd9", "red box-20aa93b87a0f4845", "small cup-d894577c7bc187e6", "soap tray-18a3b6de3fd43d98", "red glass-b97e64636f1eeb66", "green glass-c59a2df00897bd92", "tea towel1-0e18f9d4e9342683", "chair1-e178a45feabacf8f", "cloth bag-282f770ae1c40b2b", "stack of two glass ramekins-718e87cf4ef03ed9", "scale-7dbecec7b90b4920", "tea towel2-6c81551755ce0456" ], "reasoning": "1. First, I locate the time stamp when the user is cleaning the pizza oven (pizza oven-8860315eea5698e5) with a cloth (white cloth-07a3dc1ff5f05466). This occurs at 09:53-10:37. \n2. At 10:38, the user moved on to wipe the inside of the lid (pizza oven-8860315eea5698e5).\n3. I rewind the video back to determine in what direction the user wipes just before this. At 10:36, the user wipes once using a clockwise motion.", "referenced_object_masks": [ "white cloth-07a3dc1ff5f05466", "pizza oven-8860315eea5698e5" ] }, { "key": "P03-20240217-210958:c3211fc9a6d6e7591c93931ff7a15fc41edac6ad", "video_id": "P03-20240217-210958", "question": "If I had a computer password with the hint \"the living things on my kitchen windowsill, placed in reverse order, with 0 being green things and 1 being red things,\" what would my password be?", "answer_choice_0": "10001", "answer_choice_1": "0001", "answer_choice_2": "0100", "answer_choice_3": "1000", "answer_choice_4": "0010", "answer_id": 4, "answer": "0010", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Counterfactual" ], "objects_of_interest": [ "knife-184d4fcac806f1f6", "small bowl-d7506f6043427514", "white bowl-63a9bfda1bf3f34b", "knife2-0ef68b39845dd1f8", "dishwasher brush-5dcf79fa622b3e10", "large bowl-301f1353cc4016fd", "small saucepan-8aaf0ab59d275137", "used kitchen roll-677cc83e279799d1", "spoon-6bde2ce481ceed16", "plate-3ad3d3f8372bc06b", "sponge-c17425c6f342f8aa", "white cloth-07a3dc1ff5f05466", "dishwasher tablet-4dc20cb87089e5d3", "small frying pan-cd5dd79cc2402496", "pizza peel-bcedfda3e55ac0d6", "washing up liquid bottle-50648ef22a7062b3", "pizza spatula1-b6bf5382ef08beb0", "pizza spatula2-cc2bec2a4e44efe3", "plastic box-d5b35a191777d1f2", "plastic box lid-fe97377b799386ad", "sink strainer-3202c768a4e2f37f", "cloth2-ba957fe9957f969e", "dough scraper-c22465f46d5e8eff", "pizza oven-8860315eea5698e5", "orange cloth-4e3125ae759cfc6c", "coffee grinder-da26efd8d9dbb285", "cleaner-0e92cdf383e33bc1", "kettle-a77eb92372c0788d", "microwave oven-247d931a640d9fd9", "red box-20aa93b87a0f4845", "small cup-d894577c7bc187e6", "soap tray-18a3b6de3fd43d98", "red glass-b97e64636f1eeb66", "green glass-c59a2df00897bd92", "tea towel1-0e18f9d4e9342683", "chair1-e178a45feabacf8f", "cloth bag-282f770ae1c40b2b", "stack of two glass ramekins-718e87cf4ef03ed9", "scale-7dbecec7b90b4920", "tea towel2-6c81551755ce0456" ], "reasoning": "1. I watched the video to observe what the user might mean by \"the living things on my kitchen windowsill\" and found 4 plants on the kitchen windowsill at 03:25.\n2. 3 of the plants are green and 1 of them is red.\n3. Using the password hint provided in the question, the plant colors in reverse (right to left) order are green (0), green (0), red (1), green (0).\n4. Therefore, the password is 0010.", "referenced_object_masks": [] }, { "key": "P03-20240217-210958:cd49ce06354cc5f9e9d61ca872662449ffe37d32", "video_id": "P03-20240217-210958", "question": "Which specific row on an item, positioned on the same side of the room as I am, has an object that matches the color of the towel visible on my right when I'm scrubbing the pizza oven the first time?", "answer_choice_0": "First row", "answer_choice_1": "Second row", "answer_choice_2": "Fourth row", "answer_choice_3": "Third row", "answer_choice_4": "Fifth row", "answer_id": 1, "answer": "Second row", "question_type": [ "Object Recognition", "Counting", "Action Recognition", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "knife-184d4fcac806f1f6", "small bowl-d7506f6043427514", "white bowl-63a9bfda1bf3f34b", "knife2-0ef68b39845dd1f8", "dishwasher brush-5dcf79fa622b3e10", "large bowl-301f1353cc4016fd", "small saucepan-8aaf0ab59d275137", "used kitchen roll-677cc83e279799d1", "spoon-6bde2ce481ceed16", "plate-3ad3d3f8372bc06b", "sponge-c17425c6f342f8aa", "white cloth-07a3dc1ff5f05466", "dishwasher tablet-4dc20cb87089e5d3", "small frying pan-cd5dd79cc2402496", "pizza peel-bcedfda3e55ac0d6", "washing up liquid bottle-50648ef22a7062b3", "pizza spatula1-b6bf5382ef08beb0", "pizza spatula2-cc2bec2a4e44efe3", "plastic box-d5b35a191777d1f2", "plastic box lid-fe97377b799386ad", "sink strainer-3202c768a4e2f37f", "cloth2-ba957fe9957f969e", "dough scraper-c22465f46d5e8eff", "pizza oven-8860315eea5698e5", "orange cloth-4e3125ae759cfc6c", "coffee grinder-da26efd8d9dbb285", "cleaner-0e92cdf383e33bc1", "kettle-a77eb92372c0788d", "microwave oven-247d931a640d9fd9", "red box-20aa93b87a0f4845", "small cup-d894577c7bc187e6", "soap tray-18a3b6de3fd43d98", "red glass-b97e64636f1eeb66", "green glass-c59a2df00897bd92", "tea towel1-0e18f9d4e9342683", "chair1-e178a45feabacf8f", "cloth bag-282f770ae1c40b2b", "stack of two glass ramekins-718e87cf4ef03ed9", "scale-7dbecec7b90b4920", "tea towel2-6c81551755ce0456" ], "reasoning": "1. First, I locate the time stamp when the user is scrubbing the pizza oven (pizza oven-8860315eea5698e5). This occurs at 08:06-08:42. \n2. When the user stops scrubbing the pizza oven (pizza oven-8860315eea5698e5) and looks to their right, I spot an orange towel (orange cloth-4e3125ae759cfc6c) folded on top of the microwave (microwave oven-247d931a640d9fd9). I determine this is the color of the object I am looking for. \n3. Next, I rewind the video to when the user is scrubbing the oven (pizza oven-8860315eea5698e5) to locate an object with rows near the user. I spot a white shelf positioned on the user's left with several rows. \n4. Between 08:30-08:34, I located a bottle with an amber-colored liquid and an orange top situated behind a bottle with a green liquid and a white top on the second row.", "referenced_object_masks": [ "pizza oven-8860315eea5698e5", "microwave oven-247d931a640d9fd9", "orange cloth-4e3125ae759cfc6c" ] }, { "key": "P03-20240217-210958:d16a759657cda3c3f2e6336355b222e0fcda91c6", "video_id": "P03-20240217-210958", "question": "If I were to describe the movement of my thumb when I wiped it on my dishwasher just before washing my fingers, did I move it along an x-axis or a y-axis (or both) and was the shift positive or negative?", "answer_choice_0": "Negative y-axis", "answer_choice_1": "Positive y-axis", "answer_choice_2": "Positive x-axis", "answer_choice_3": "Negative and along both axes", "answer_choice_4": "Negative x-axis", "answer_id": 1, "answer": "Positive y-axis", "question_type": [ "Egocentric & Spatial Perception", "Counterfactual", "Action Recognition" ], "objects_of_interest": [ "knife-184d4fcac806f1f6", "small bowl-d7506f6043427514", "white bowl-63a9bfda1bf3f34b", "knife2-0ef68b39845dd1f8", "dishwasher brush-5dcf79fa622b3e10", "large bowl-301f1353cc4016fd", "small saucepan-8aaf0ab59d275137", "used kitchen roll-677cc83e279799d1", "spoon-6bde2ce481ceed16", "plate-3ad3d3f8372bc06b", "sponge-c17425c6f342f8aa", "white cloth-07a3dc1ff5f05466", "dishwasher tablet-4dc20cb87089e5d3", "small frying pan-cd5dd79cc2402496", "pizza peel-bcedfda3e55ac0d6", "washing up liquid bottle-50648ef22a7062b3", "pizza spatula1-b6bf5382ef08beb0", "pizza spatula2-cc2bec2a4e44efe3", "plastic box-d5b35a191777d1f2", "plastic box lid-fe97377b799386ad", "sink strainer-3202c768a4e2f37f", "cloth2-ba957fe9957f969e", "dough scraper-c22465f46d5e8eff", "pizza oven-8860315eea5698e5", "orange cloth-4e3125ae759cfc6c", "coffee grinder-da26efd8d9dbb285", "cleaner-0e92cdf383e33bc1", "kettle-a77eb92372c0788d", "microwave oven-247d931a640d9fd9", "red box-20aa93b87a0f4845", "small cup-d894577c7bc187e6", "soap tray-18a3b6de3fd43d98", "red glass-b97e64636f1eeb66", "green glass-c59a2df00897bd92", "tea towel1-0e18f9d4e9342683", "chair1-e178a45feabacf8f", "cloth bag-282f770ae1c40b2b", "stack of two glass ramekins-718e87cf4ef03ed9", "scale-7dbecec7b90b4920", "tea towel2-6c81551755ce0456" ], "reasoning": "1. I watched the video to find where the user wiped his thumb on the dishwasher and saw it happen at 01:45.\n2. The user wipes his thumb in one motion, starting from one point and going straight up.\n3. Using the parameters set in the question, the movement of their thumb indicates a positive y-axis shift.", "referenced_object_masks": [] }, { "key": "P03-20240217-210958:f99210cb87d2577ba0d3c9471e661e96d10e5af2", "video_id": "P03-20240217-210958", "question": "Can you pinpoint the direction of my gaze in all 3 instances: when I make the last fold in the first yellow towel, when I face the seating area for the first time, and when the light blue rolling pin comes into view?", "answer_choice_0": "Straight ahead; straight ahead; down", "answer_choice_1": "Down; down; straight ahead", "answer_choice_2": "Up, straight ahead; down", "answer_choice_3": "Up; down; straight ahead", "answer_choice_4": "Down; straight ahead; up", "answer_id": 0, "answer": "Straight ahead; straight ahead; down", "question_type": [ "Egocentric & Spatial Perception", "Action Recognition", "State Changes", "Object Recognition" ], "objects_of_interest": [ "knife-184d4fcac806f1f6", "small bowl-d7506f6043427514", "white bowl-63a9bfda1bf3f34b", "knife2-0ef68b39845dd1f8", "dishwasher brush-5dcf79fa622b3e10", "large bowl-301f1353cc4016fd", "small saucepan-8aaf0ab59d275137", "used kitchen roll-677cc83e279799d1", "spoon-6bde2ce481ceed16", "plate-3ad3d3f8372bc06b", "sponge-c17425c6f342f8aa", "white cloth-07a3dc1ff5f05466", "dishwasher tablet-4dc20cb87089e5d3", "small frying pan-cd5dd79cc2402496", "pizza peel-bcedfda3e55ac0d6", "washing up liquid bottle-50648ef22a7062b3", "pizza spatula1-b6bf5382ef08beb0", "pizza spatula2-cc2bec2a4e44efe3", "plastic box-d5b35a191777d1f2", "plastic box lid-fe97377b799386ad", "sink strainer-3202c768a4e2f37f", "cloth2-ba957fe9957f969e", "dough scraper-c22465f46d5e8eff", "pizza oven-8860315eea5698e5", "orange cloth-4e3125ae759cfc6c", "coffee grinder-da26efd8d9dbb285", "cleaner-0e92cdf383e33bc1", "kettle-a77eb92372c0788d", "microwave oven-247d931a640d9fd9", "red box-20aa93b87a0f4845", "small cup-d894577c7bc187e6", "soap tray-18a3b6de3fd43d98", "red glass-b97e64636f1eeb66", "green glass-c59a2df00897bd92", "tea towel1-0e18f9d4e9342683", "chair1-e178a45feabacf8f", "cloth bag-282f770ae1c40b2b", "stack of two glass ramekins-718e87cf4ef03ed9", "scale-7dbecec7b90b4920", "tea towel2-6c81551755ce0456" ], "reasoning": "1. First, I determine the timestamps in the video when the user does all 3 actions. \n2. The user makes the last fold in the first yellow towel (tea towel1-0e18f9d4e9342683) at 18:00, where they are looking straight ahead.\n3. Next, the user first faces the seating area at 01:08, where they are also looking straight ahead. \n4. Finally, the light blue rolling pin comes into view between 18:11-18:17, when the user is sorting through the cabinet while looking down.", "referenced_object_masks": [ "tea towel1-0e18f9d4e9342683" ] }, { "key": "P03-20240218-094452:0eab03d05b6be11a124f092a230d2d767f5a29a5", "video_id": "P03-20240218-094452", "question": "The second time I pour hot water into the coffee grinds, how many circles do I make with the hot water?", "answer_choice_0": "4", "answer_choice_1": "5", "answer_choice_2": "6", "answer_choice_3": "3", "answer_choice_4": "2", "answer_id": 0, "answer": "4", "question_type": [ "Action Recognition", "Counting", "Event Occurence", "Object Recognition", "Situational Awareness", "Temporal Reasoning" ], "objects_of_interest": [ "something long and white-449c110dd7bb304b", "hoover charger-4e8ef85738d76854", "buns package-1e0a7dce05008c66", "package of frozen food-c0af8c59a26d36e5", "steam cooker-4bffeb5a879b210e", "rice measuring cup-ca5c6eeb6a5111e9", "black cap-eec6bdde683a7264", "four buns-5aefa9d7c7b9f43a", "bun 1-b153b46483471edc", "bun 2-84a75bc192c1de64", "dishwasher brush-9522757b807d2844", "grey cloth-223b662e1a6b3f65", "saucepan-cb7b4af5dd382732", "other piece of padding-d1e249a63ce1c98b", "pan-c3fe95200e66347d", "small frying pan-a39a6b4e9c4f894a", "smaller saucepan-3a5a53c78d86b2a7", "another larger saucepan-e7840252be57868f", "cloth-f013b49ae87fbf35", "lid-c8f06d9b2762a44a", "larger lid-bb8d6ac5951230e3", "plate-45fcc9fab911551b", "another plate-0791ef8271b30668", "small bowl-63e03354dc8a7770", "second small bowl-7c76e947f5260475", "stack of small bowls-c8af4a9e9cec794b", "large chopping board-de6f9f9ad64084f6", "smaller chopping board-c956dc0ca7d2c860", "fork-3211e1268cb7c307", "knife-5f481b39babbe740", "another knife-3c8adaacf8da0ba0", "another fork-d20b12bb6e8fa10f", "spoon-e8abb2d89ec57c0b", "teaspoon-1b3d5f7097fb5d67", "dishwasher cutlery tray-953bdfeb301f8e0d", "knife in dishwasher cutlery tray-3003f2e17300cdeb", "green knife in dishwasher cutlery tray-7866f99b18a29858", "spoon in dishwasher cutlery tray-794d93a9959561d6", "fork in dishwasher cutlery tray-19cb6452869192a6", "coffee spoon in dishwasher cutlery tray-7b69724270924207", "another spoon in dishwasher cutlery tray-a30e7b72794583d8", "another knife in dishwasher cutlery tray-c74afe2dbe9d38d2", "another fork in dishwasher cutlery tray-d66149e07df06890", "teaspoon in dishwasher cutlery tray-c678067bc8afa8e3", "another teaspoon in dishwasher cutlery tray-40e2fc7a314e5c07", "third fork in dishwasher cutlery tray-dfdc1cad36d24bf0", "third teaspoon in dishwasher cutlery tray-bd407d3562b5a5e3", "fourth teaspoon in dishwasher cutlery tray-aeb522f581fb7cd6", "two small grey plates-c1a9aa81a9575967", "one grey plate-34566b83d7c9713c", "fork on gray plate-d38762e2560c6aa5", "two small red plates-d203fa0af72f8cdf", "one red plate-1a2d1ff8e5ad8358", "bowl-135e0dfb6200084b", "ladle-f5851f27ec78a906", "spatula-485fb33cb31ea658", "white bowl-743a192de234a5b3", "plastic colander-1a470738623a3853", "white bowl inside plastic colander-a2da3094d2ea88c6", "grey bowl-de10639976089e78", "second white bowl-344218667f6d7425", "metal v60 coffee filter-ff8b2539e792bae5", "stack of four bowls-4e84a8ebb474cc07", "second small white bowl-7e373113f20278ae", "stack of four small bowls-a8583570f0407761", "new small bowl 1-326960c1c66c5cb0", "new small bowl 2-cc454d7b92c7c794", "right green cup-c26ce56033267a24", "left green cup-2db9e03a38cf3d27", "big grey cup-ebc65813ceb92966", "third green cup-3f56bc662dac0e69", "printed cup-7088cfd472efb9d4", "transparent cup-acc33a8af845cbc6", "transparent cup with blue pattern-44261cccd91ca9b9", "wooden spoon-5fc7faf0ec2572ba", "scissors-3912136e4b70d8d0", "clear plastic box-628c8fcc0cc63d20", "clear plastic box lid-4ed9d1af85419b14", "wooden pizza peel-ee1a56fe1fb252a2", "dough scraper two-4e12b1def9b0e1d0", "dough scraper steel-30bbaf52cc718dbe", "pizza utensils-7652ced72417c90d", "cake mould-b1697183bc4b6ec8", "new cake mould 2-a5786ff65e5d3cf2", "dough scraper-3b5112ce201d68e2", "rolling pin-fab5e8b30e2f7a0a", "colander-1eda12f891a123e8", "large bowl-dc020554539664e2", "coffee jug-5bf682ebe7469336", "v60 rack-ee8733904266d8fc", "v60 coffee filter-15225e36ef83154c", "coffee grinder-57d22fd7cc640d64", "coffee packet-861ac93bb78fac2e", "clip-ee98a7319017ee0c", "red spoon-5ab2099d40423ca8", "coffee grinder top lid-d4f233de88dcb53f", "kettle-7ee96b72534023dd", "white plate-565a45d3f467e7a3", "cling film ball-ee06a2883ef92089", "the top one of two small grey plates-76bb7db989ed6749", "coffee container-30ea4677f09d18c4", "orange cloth-c1934fd49f6621c2", "bread plastic packaging-20b7cb0754ce4586", "slice of bread-bf74962e8134050e", "tablecloth-8ae87725bcebbc1a", "second tablecloth-a985a9aef85303f1", "plastic bottle-f665f8e6f5d29ce8", "red glass-762ae64171db9c3a", "green glass-236693d0f663eeab", "tissue box-128cbae444c479f3", "peanut butter jar-653b89d28122b0aa", "butter knife-5e5490be80423458", "bread and plate-f68defa01ab3d275" ], "reasoning": "1. I watched the whole video. \n2. At 11:54, I am pouring the hot water from a kettle (kettle-7ee96b72534023dd) over the coffee grinds for the first time.\n3. At 12:26, I am pouring the hot water from a kettle (kettle-7ee96b72534023dd) over the coffee grinds for the second time.\n4. From 12:26-12:35, I make 4 circles with the hot water from the kettle (kettle-7ee96b72534023dd) being poured into the coffee grinds .", "referenced_object_masks": [ "kettle-7ee96b72534023dd" ] }, { "key": "P03-20240218-094452:1c43289ea2b1ca06a4f43882312e2ea02a40f709", "video_id": "P03-20240218-094452", "question": "How many items from the dishwasher did I dry off with a kitchen towel?", "answer_choice_0": "7", "answer_choice_1": "5", "answer_choice_2": "4", "answer_choice_3": "6", "answer_choice_4": "8", "answer_id": 0, "answer": "7", "question_type": [ "Action Recognition", "Counting", "Event Occurence", "Object Recognition", "Situational Awareness", "Temporal Reasoning" ], "objects_of_interest": [ "something long and white-449c110dd7bb304b", "hoover charger-4e8ef85738d76854", "buns package-1e0a7dce05008c66", "package of frozen food-c0af8c59a26d36e5", "steam cooker-4bffeb5a879b210e", "rice measuring cup-ca5c6eeb6a5111e9", "black cap-eec6bdde683a7264", "four buns-5aefa9d7c7b9f43a", "bun 1-b153b46483471edc", "bun 2-84a75bc192c1de64", "dishwasher brush-9522757b807d2844", "grey cloth-223b662e1a6b3f65", "saucepan-cb7b4af5dd382732", "other piece of padding-d1e249a63ce1c98b", "pan-c3fe95200e66347d", "small frying pan-a39a6b4e9c4f894a", "smaller saucepan-3a5a53c78d86b2a7", "another larger saucepan-e7840252be57868f", "cloth-f013b49ae87fbf35", "lid-c8f06d9b2762a44a", "larger lid-bb8d6ac5951230e3", "plate-45fcc9fab911551b", "another plate-0791ef8271b30668", "small bowl-63e03354dc8a7770", "second small bowl-7c76e947f5260475", "stack of small bowls-c8af4a9e9cec794b", "large chopping board-de6f9f9ad64084f6", "smaller chopping board-c956dc0ca7d2c860", "fork-3211e1268cb7c307", "knife-5f481b39babbe740", "another knife-3c8adaacf8da0ba0", "another fork-d20b12bb6e8fa10f", "spoon-e8abb2d89ec57c0b", "teaspoon-1b3d5f7097fb5d67", "dishwasher cutlery tray-953bdfeb301f8e0d", "knife in dishwasher cutlery tray-3003f2e17300cdeb", "green knife in dishwasher cutlery tray-7866f99b18a29858", "spoon in dishwasher cutlery tray-794d93a9959561d6", "fork in dishwasher cutlery tray-19cb6452869192a6", "coffee spoon in dishwasher cutlery tray-7b69724270924207", "another spoon in dishwasher cutlery tray-a30e7b72794583d8", "another knife in dishwasher cutlery tray-c74afe2dbe9d38d2", "another fork in dishwasher cutlery tray-d66149e07df06890", "teaspoon in dishwasher cutlery tray-c678067bc8afa8e3", "another teaspoon in dishwasher cutlery tray-40e2fc7a314e5c07", "third fork in dishwasher cutlery tray-dfdc1cad36d24bf0", "third teaspoon in dishwasher cutlery tray-bd407d3562b5a5e3", "fourth teaspoon in dishwasher cutlery tray-aeb522f581fb7cd6", "two small grey plates-c1a9aa81a9575967", "one grey plate-34566b83d7c9713c", "fork on gray plate-d38762e2560c6aa5", "two small red plates-d203fa0af72f8cdf", "one red plate-1a2d1ff8e5ad8358", "bowl-135e0dfb6200084b", "ladle-f5851f27ec78a906", "spatula-485fb33cb31ea658", "white bowl-743a192de234a5b3", "plastic colander-1a470738623a3853", "white bowl inside plastic colander-a2da3094d2ea88c6", "grey bowl-de10639976089e78", "second white bowl-344218667f6d7425", "metal v60 coffee filter-ff8b2539e792bae5", "stack of four bowls-4e84a8ebb474cc07", "second small white bowl-7e373113f20278ae", "stack of four small bowls-a8583570f0407761", "new small bowl 1-326960c1c66c5cb0", "new small bowl 2-cc454d7b92c7c794", "right green cup-c26ce56033267a24", "left green cup-2db9e03a38cf3d27", "big grey cup-ebc65813ceb92966", "third green cup-3f56bc662dac0e69", "printed cup-7088cfd472efb9d4", "transparent cup-acc33a8af845cbc6", "transparent cup with blue pattern-44261cccd91ca9b9", "wooden spoon-5fc7faf0ec2572ba", "scissors-3912136e4b70d8d0", "clear plastic box-628c8fcc0cc63d20", "clear plastic box lid-4ed9d1af85419b14", "wooden pizza peel-ee1a56fe1fb252a2", "dough scraper two-4e12b1def9b0e1d0", "dough scraper steel-30bbaf52cc718dbe", "pizza utensils-7652ced72417c90d", "cake mould-b1697183bc4b6ec8", "new cake mould 2-a5786ff65e5d3cf2", "dough scraper-3b5112ce201d68e2", "rolling pin-fab5e8b30e2f7a0a", "colander-1eda12f891a123e8", "large bowl-dc020554539664e2", "coffee jug-5bf682ebe7469336", "v60 rack-ee8733904266d8fc", "v60 coffee filter-15225e36ef83154c", "coffee grinder-57d22fd7cc640d64", "coffee packet-861ac93bb78fac2e", "clip-ee98a7319017ee0c", "red spoon-5ab2099d40423ca8", "coffee grinder top lid-d4f233de88dcb53f", "kettle-7ee96b72534023dd", "white plate-565a45d3f467e7a3", "cling film ball-ee06a2883ef92089", "the top one of two small grey plates-76bb7db989ed6749", "coffee container-30ea4677f09d18c4", "orange cloth-c1934fd49f6621c2", "bread plastic packaging-20b7cb0754ce4586", "slice of bread-bf74962e8134050e", "tablecloth-8ae87725bcebbc1a", "second tablecloth-a985a9aef85303f1", "plastic bottle-f665f8e6f5d29ce8", "red glass-762ae64171db9c3a", "green glass-236693d0f663eeab", "tissue box-128cbae444c479f3", "peanut butter jar-653b89d28122b0aa", "butter knife-5e5490be80423458", "bread and plate-f68defa01ab3d275" ], "reasoning": "1. I watched the whole video and saw the user dry off the following items from the dishwasher with a kitchen towel (cloth-f013b49ae87fbf35). \n2. At 02:47, the user dries off a silver pan (another larger saucepan-e7840252be57868f). \n3. At 04:45, the user dries a gray bowl (small bowl-63e03354dc8a7770). \n4. At 05:01, the user dries another gray bowl (second small bowl-7c76e947f5260475). \n5. At 05:08, the user dries another gray bowl (grey bowl-de10639976089e78).\n6. At 05:13, the user dries another gray bowl (second white bowl-344218667f6d7425).\n7. At 05:52, the user dries a blue and green coffee mug (left green cup-2db9e03a38cf3d27). \n8. Lastly, at 07:32, the user dries a white bowl (large bowl-dc020554539664e2) -- this had been taken from the dishwasher from 05:03-05:04.\n9. The total number of items from the dishwasher that needed extra drying with the kitchen towel (cloth-f013b49ae87fbf35) was 7.", "referenced_object_masks": [ "large bowl-dc020554539664e2", "small bowl-63e03354dc8a7770", "cloth-f013b49ae87fbf35", "left green cup-2db9e03a38cf3d27", "second white bowl-344218667f6d7425", "second small bowl-7c76e947f5260475", "another larger saucepan-e7840252be57868f", "grey bowl-de10639976089e78" ] }, { "key": "P03-20240218-094452:357866b6c428b0c521c8a3b88c1022c03a46ae6e", "video_id": "P03-20240218-094452", "question": "If I were to sandwich the objects that I handle directly after throwing away the plastic wrap, so that the top and bottom objects matched each other while being different from the middle object, what would be the sandwich order from top to bottom by item material?", "answer_choice_0": "Plastic, metal, plastic", "answer_choice_1": "Plastic, ceramic, metal", "answer_choice_2": "Metal, ceramic, metal", "answer_choice_3": "Plastic, ceramic, plastic", "answer_choice_4": "Ceramic, metal, ceramic", "answer_id": 4, "answer": "Ceramic, metal, ceramic", "question_type": [ "Action Recognition", "Counterfactual", "Object Recognition", "Temporal Reasoning", "Situational Awareness" ], "objects_of_interest": [ "something long and white-449c110dd7bb304b", "hoover charger-4e8ef85738d76854", "buns package-1e0a7dce05008c66", "package of frozen food-c0af8c59a26d36e5", "steam cooker-4bffeb5a879b210e", "rice measuring cup-ca5c6eeb6a5111e9", "black cap-eec6bdde683a7264", "four buns-5aefa9d7c7b9f43a", "bun 1-b153b46483471edc", "bun 2-84a75bc192c1de64", "dishwasher brush-9522757b807d2844", "grey cloth-223b662e1a6b3f65", "saucepan-cb7b4af5dd382732", "other piece of padding-d1e249a63ce1c98b", "pan-c3fe95200e66347d", "small frying pan-a39a6b4e9c4f894a", "smaller saucepan-3a5a53c78d86b2a7", "another larger saucepan-e7840252be57868f", "cloth-f013b49ae87fbf35", "lid-c8f06d9b2762a44a", "larger lid-bb8d6ac5951230e3", "plate-45fcc9fab911551b", "another plate-0791ef8271b30668", "small bowl-63e03354dc8a7770", "second small bowl-7c76e947f5260475", "stack of small bowls-c8af4a9e9cec794b", "large chopping board-de6f9f9ad64084f6", "smaller chopping board-c956dc0ca7d2c860", "fork-3211e1268cb7c307", "knife-5f481b39babbe740", "another knife-3c8adaacf8da0ba0", "another fork-d20b12bb6e8fa10f", "spoon-e8abb2d89ec57c0b", "teaspoon-1b3d5f7097fb5d67", "dishwasher cutlery tray-953bdfeb301f8e0d", "knife in dishwasher cutlery tray-3003f2e17300cdeb", "green knife in dishwasher cutlery tray-7866f99b18a29858", "spoon in dishwasher cutlery tray-794d93a9959561d6", "fork in dishwasher cutlery tray-19cb6452869192a6", "coffee spoon in dishwasher cutlery tray-7b69724270924207", "another spoon in dishwasher cutlery tray-a30e7b72794583d8", "another knife in dishwasher cutlery tray-c74afe2dbe9d38d2", "another fork in dishwasher cutlery tray-d66149e07df06890", "teaspoon in dishwasher cutlery tray-c678067bc8afa8e3", "another teaspoon in dishwasher cutlery tray-40e2fc7a314e5c07", "third fork in dishwasher cutlery tray-dfdc1cad36d24bf0", "third teaspoon in dishwasher cutlery tray-bd407d3562b5a5e3", "fourth teaspoon in dishwasher cutlery tray-aeb522f581fb7cd6", "two small grey plates-c1a9aa81a9575967", "one grey plate-34566b83d7c9713c", "fork on gray plate-d38762e2560c6aa5", "two small red plates-d203fa0af72f8cdf", "one red plate-1a2d1ff8e5ad8358", "bowl-135e0dfb6200084b", "ladle-f5851f27ec78a906", "spatula-485fb33cb31ea658", "white bowl-743a192de234a5b3", "plastic colander-1a470738623a3853", "white bowl inside plastic colander-a2da3094d2ea88c6", "grey bowl-de10639976089e78", "second white bowl-344218667f6d7425", "metal v60 coffee filter-ff8b2539e792bae5", "stack of four bowls-4e84a8ebb474cc07", "second small white bowl-7e373113f20278ae", "stack of four small bowls-a8583570f0407761", "new small bowl 1-326960c1c66c5cb0", "new small bowl 2-cc454d7b92c7c794", "right green cup-c26ce56033267a24", "left green cup-2db9e03a38cf3d27", "big grey cup-ebc65813ceb92966", "third green cup-3f56bc662dac0e69", "printed cup-7088cfd472efb9d4", "transparent cup-acc33a8af845cbc6", "transparent cup with blue pattern-44261cccd91ca9b9", "wooden spoon-5fc7faf0ec2572ba", "scissors-3912136e4b70d8d0", "clear plastic box-628c8fcc0cc63d20", "clear plastic box lid-4ed9d1af85419b14", "wooden pizza peel-ee1a56fe1fb252a2", "dough scraper two-4e12b1def9b0e1d0", "dough scraper steel-30bbaf52cc718dbe", "pizza utensils-7652ced72417c90d", "cake mould-b1697183bc4b6ec8", "new cake mould 2-a5786ff65e5d3cf2", "dough scraper-3b5112ce201d68e2", "rolling pin-fab5e8b30e2f7a0a", "colander-1eda12f891a123e8", "large bowl-dc020554539664e2", "coffee jug-5bf682ebe7469336", "v60 rack-ee8733904266d8fc", "v60 coffee filter-15225e36ef83154c", "coffee grinder-57d22fd7cc640d64", "coffee packet-861ac93bb78fac2e", "clip-ee98a7319017ee0c", "red spoon-5ab2099d40423ca8", "coffee grinder top lid-d4f233de88dcb53f", "kettle-7ee96b72534023dd", "white plate-565a45d3f467e7a3", "cling film ball-ee06a2883ef92089", "the top one of two small grey plates-76bb7db989ed6749", "coffee container-30ea4677f09d18c4", "orange cloth-c1934fd49f6621c2", "bread plastic packaging-20b7cb0754ce4586", "slice of bread-bf74962e8134050e", "tablecloth-8ae87725bcebbc1a", "second tablecloth-a985a9aef85303f1", "plastic bottle-f665f8e6f5d29ce8", "red glass-762ae64171db9c3a", "green glass-236693d0f663eeab", "tissue box-128cbae444c479f3", "peanut butter jar-653b89d28122b0aa", "butter knife-5e5490be80423458", "bread and plate-f68defa01ab3d275" ], "reasoning": "1. I found the moment where the plastic wrap (cling film ball-ee06a2883ef92089) was thrown away by the user at 10:02.\n2. After the wrap was thrown away, the user moved over to the counter at 10:04, and picked up a fork (fork on gray plate-d38762e2560c6aa5) that was on top of a gray plate (the top one of two small grey plates-76bb7db989ed6749).\n3. Underneath the top plate (the top one of two small grey plates-76bb7db989ed6749) was another plate made of the same material.\n4. If you were to sandwich those items based on the material order, it would be \"ceramic, metal, ceramic\".", "referenced_object_masks": [ "fork on gray plate-d38762e2560c6aa5", "the top one of two small grey plates-76bb7db989ed6749", "cling film ball-ee06a2883ef92089" ] }, { "key": "P03-20240218-094452:6a042fa7bee0ca75aca66adc2b583260e5307977", "video_id": "P03-20240218-094452", "question": "What do the longest kitchen item that I handle and the object immediately to my right when I plugged in the vacuum have in common?", "answer_choice_0": "They both have sharp edges", "answer_choice_1": "They're both for sitting on", "answer_choice_2": "They're both rectangular", "answer_choice_3": "They both have handles", "answer_choice_4": "They're both wooden", "answer_id": 4, "answer": "They're both wooden", "question_type": [ "Egocentric & Spatial Perception", "Event Occurence", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "something long and white-449c110dd7bb304b", "hoover charger-4e8ef85738d76854", "buns package-1e0a7dce05008c66", "package of frozen food-c0af8c59a26d36e5", "steam cooker-4bffeb5a879b210e", "rice measuring cup-ca5c6eeb6a5111e9", "black cap-eec6bdde683a7264", "four buns-5aefa9d7c7b9f43a", "bun 1-b153b46483471edc", "bun 2-84a75bc192c1de64", "dishwasher brush-9522757b807d2844", "grey cloth-223b662e1a6b3f65", "saucepan-cb7b4af5dd382732", "other piece of padding-d1e249a63ce1c98b", "pan-c3fe95200e66347d", "small frying pan-a39a6b4e9c4f894a", "smaller saucepan-3a5a53c78d86b2a7", "another larger saucepan-e7840252be57868f", "cloth-f013b49ae87fbf35", "lid-c8f06d9b2762a44a", "larger lid-bb8d6ac5951230e3", "plate-45fcc9fab911551b", "another plate-0791ef8271b30668", "small bowl-63e03354dc8a7770", "second small bowl-7c76e947f5260475", "stack of small bowls-c8af4a9e9cec794b", "large chopping board-de6f9f9ad64084f6", "smaller chopping board-c956dc0ca7d2c860", "fork-3211e1268cb7c307", "knife-5f481b39babbe740", "another knife-3c8adaacf8da0ba0", "another fork-d20b12bb6e8fa10f", "spoon-e8abb2d89ec57c0b", "teaspoon-1b3d5f7097fb5d67", "dishwasher cutlery tray-953bdfeb301f8e0d", "knife in dishwasher cutlery tray-3003f2e17300cdeb", "green knife in dishwasher cutlery tray-7866f99b18a29858", "spoon in dishwasher cutlery tray-794d93a9959561d6", "fork in dishwasher cutlery tray-19cb6452869192a6", "coffee spoon in dishwasher cutlery tray-7b69724270924207", "another spoon in dishwasher cutlery tray-a30e7b72794583d8", "another knife in dishwasher cutlery tray-c74afe2dbe9d38d2", "another fork in dishwasher cutlery tray-d66149e07df06890", "teaspoon in dishwasher cutlery tray-c678067bc8afa8e3", "another teaspoon in dishwasher cutlery tray-40e2fc7a314e5c07", "third fork in dishwasher cutlery tray-dfdc1cad36d24bf0", "third teaspoon in dishwasher cutlery tray-bd407d3562b5a5e3", "fourth teaspoon in dishwasher cutlery tray-aeb522f581fb7cd6", "two small grey plates-c1a9aa81a9575967", "one grey plate-34566b83d7c9713c", "fork on gray plate-d38762e2560c6aa5", "two small red plates-d203fa0af72f8cdf", "one red plate-1a2d1ff8e5ad8358", "bowl-135e0dfb6200084b", "ladle-f5851f27ec78a906", "spatula-485fb33cb31ea658", "white bowl-743a192de234a5b3", "plastic colander-1a470738623a3853", "white bowl inside plastic colander-a2da3094d2ea88c6", "grey bowl-de10639976089e78", "second white bowl-344218667f6d7425", "metal v60 coffee filter-ff8b2539e792bae5", "stack of four bowls-4e84a8ebb474cc07", "second small white bowl-7e373113f20278ae", "stack of four small bowls-a8583570f0407761", "new small bowl 1-326960c1c66c5cb0", "new small bowl 2-cc454d7b92c7c794", "right green cup-c26ce56033267a24", "left green cup-2db9e03a38cf3d27", "big grey cup-ebc65813ceb92966", "third green cup-3f56bc662dac0e69", "printed cup-7088cfd472efb9d4", "transparent cup-acc33a8af845cbc6", "transparent cup with blue pattern-44261cccd91ca9b9", "wooden spoon-5fc7faf0ec2572ba", "scissors-3912136e4b70d8d0", "clear plastic box-628c8fcc0cc63d20", "clear plastic box lid-4ed9d1af85419b14", "wooden pizza peel-ee1a56fe1fb252a2", "dough scraper two-4e12b1def9b0e1d0", "dough scraper steel-30bbaf52cc718dbe", "pizza utensils-7652ced72417c90d", "cake mould-b1697183bc4b6ec8", "new cake mould 2-a5786ff65e5d3cf2", "dough scraper-3b5112ce201d68e2", "rolling pin-fab5e8b30e2f7a0a", "colander-1eda12f891a123e8", "large bowl-dc020554539664e2", "coffee jug-5bf682ebe7469336", "v60 rack-ee8733904266d8fc", "v60 coffee filter-15225e36ef83154c", "coffee grinder-57d22fd7cc640d64", "coffee packet-861ac93bb78fac2e", "clip-ee98a7319017ee0c", "red spoon-5ab2099d40423ca8", "coffee grinder top lid-d4f233de88dcb53f", "kettle-7ee96b72534023dd", "white plate-565a45d3f467e7a3", "cling film ball-ee06a2883ef92089", "the top one of two small grey plates-76bb7db989ed6749", "coffee container-30ea4677f09d18c4", "orange cloth-c1934fd49f6621c2", "bread plastic packaging-20b7cb0754ce4586", "slice of bread-bf74962e8134050e", "tablecloth-8ae87725bcebbc1a", "second tablecloth-a985a9aef85303f1", "plastic bottle-f665f8e6f5d29ce8", "red glass-762ae64171db9c3a", "green glass-236693d0f663eeab", "tissue box-128cbae444c479f3", "peanut butter jar-653b89d28122b0aa", "butter knife-5e5490be80423458", "bread and plate-f68defa01ab3d275" ], "reasoning": "1. I watched the video to see if I could identify the longest kitchen tool.\n2. At 06:29, I saw the user lay the pizza paddle (wooden pizza peel-ee1a56fe1fb252a2) on top of a tupperware (clear plastic box-628c8fcc0cc63d20). I could see the peel was roughly the same size, but its handle extended beyond the edges of the box, making it the longest kitchen tool the user handled.\n3. I then looked for where the user plugged in the vacuum (something long and white-449c110dd7bb304b) and saw them plug it into the wall at 00:23. I looked to the right and identified a wooden chair as being next to the user.\n4. I noticed that the similarity was that both the pizza paddle (wooden pizza peel-ee1a56fe1fb252a2) and the chair were made of wood.", "referenced_object_masks": [ "something long and white-449c110dd7bb304b", "clear plastic box-628c8fcc0cc63d20", "wooden pizza peel-ee1a56fe1fb252a2" ] }, { "key": "P03-20240218-094452:7c735f397ef06f4adb51a83c3017d703ed0ede31", "video_id": "P03-20240218-094452", "question": "How long do I hold the vacuum cleaner before I hold the first black object in my right hand?", "answer_choice_0": "18 seconds", "answer_choice_1": "29 seconds", "answer_choice_2": "20 seconds", "answer_choice_3": "6 seconds", "answer_choice_4": "8 seconds", "answer_id": 4, "answer": "8 seconds", "question_type": [ "Action Recognition", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "something long and white-449c110dd7bb304b", "hoover charger-4e8ef85738d76854", "buns package-1e0a7dce05008c66", "package of frozen food-c0af8c59a26d36e5", "steam cooker-4bffeb5a879b210e", "rice measuring cup-ca5c6eeb6a5111e9", "black cap-eec6bdde683a7264", "four buns-5aefa9d7c7b9f43a", "bun 1-b153b46483471edc", "bun 2-84a75bc192c1de64", "dishwasher brush-9522757b807d2844", "grey cloth-223b662e1a6b3f65", "saucepan-cb7b4af5dd382732", "other piece of padding-d1e249a63ce1c98b", "pan-c3fe95200e66347d", "small frying pan-a39a6b4e9c4f894a", "smaller saucepan-3a5a53c78d86b2a7", "another larger saucepan-e7840252be57868f", "cloth-f013b49ae87fbf35", "lid-c8f06d9b2762a44a", "larger lid-bb8d6ac5951230e3", "plate-45fcc9fab911551b", "another plate-0791ef8271b30668", "small bowl-63e03354dc8a7770", "second small bowl-7c76e947f5260475", "stack of small bowls-c8af4a9e9cec794b", "large chopping board-de6f9f9ad64084f6", "smaller chopping board-c956dc0ca7d2c860", "fork-3211e1268cb7c307", "knife-5f481b39babbe740", "another knife-3c8adaacf8da0ba0", "another fork-d20b12bb6e8fa10f", "spoon-e8abb2d89ec57c0b", "teaspoon-1b3d5f7097fb5d67", "dishwasher cutlery tray-953bdfeb301f8e0d", "knife in dishwasher cutlery tray-3003f2e17300cdeb", "green knife in dishwasher cutlery tray-7866f99b18a29858", "spoon in dishwasher cutlery tray-794d93a9959561d6", "fork in dishwasher cutlery tray-19cb6452869192a6", "coffee spoon in dishwasher cutlery tray-7b69724270924207", "another spoon in dishwasher cutlery tray-a30e7b72794583d8", "another knife in dishwasher cutlery tray-c74afe2dbe9d38d2", "another fork in dishwasher cutlery tray-d66149e07df06890", "teaspoon in dishwasher cutlery tray-c678067bc8afa8e3", "another teaspoon in dishwasher cutlery tray-40e2fc7a314e5c07", "third fork in dishwasher cutlery tray-dfdc1cad36d24bf0", "third teaspoon in dishwasher cutlery tray-bd407d3562b5a5e3", "fourth teaspoon in dishwasher cutlery tray-aeb522f581fb7cd6", "two small grey plates-c1a9aa81a9575967", "one grey plate-34566b83d7c9713c", "fork on gray plate-d38762e2560c6aa5", "two small red plates-d203fa0af72f8cdf", "one red plate-1a2d1ff8e5ad8358", "bowl-135e0dfb6200084b", "ladle-f5851f27ec78a906", "spatula-485fb33cb31ea658", "white bowl-743a192de234a5b3", "plastic colander-1a470738623a3853", "white bowl inside plastic colander-a2da3094d2ea88c6", "grey bowl-de10639976089e78", "second white bowl-344218667f6d7425", "metal v60 coffee filter-ff8b2539e792bae5", "stack of four bowls-4e84a8ebb474cc07", "second small white bowl-7e373113f20278ae", "stack of four small bowls-a8583570f0407761", "new small bowl 1-326960c1c66c5cb0", "new small bowl 2-cc454d7b92c7c794", "right green cup-c26ce56033267a24", "left green cup-2db9e03a38cf3d27", "big grey cup-ebc65813ceb92966", "third green cup-3f56bc662dac0e69", "printed cup-7088cfd472efb9d4", "transparent cup-acc33a8af845cbc6", "transparent cup with blue pattern-44261cccd91ca9b9", "wooden spoon-5fc7faf0ec2572ba", "scissors-3912136e4b70d8d0", "clear plastic box-628c8fcc0cc63d20", "clear plastic box lid-4ed9d1af85419b14", "wooden pizza peel-ee1a56fe1fb252a2", "dough scraper two-4e12b1def9b0e1d0", "dough scraper steel-30bbaf52cc718dbe", "pizza utensils-7652ced72417c90d", "cake mould-b1697183bc4b6ec8", "new cake mould 2-a5786ff65e5d3cf2", "dough scraper-3b5112ce201d68e2", "rolling pin-fab5e8b30e2f7a0a", "colander-1eda12f891a123e8", "large bowl-dc020554539664e2", "coffee jug-5bf682ebe7469336", "v60 rack-ee8733904266d8fc", "v60 coffee filter-15225e36ef83154c", "coffee grinder-57d22fd7cc640d64", "coffee packet-861ac93bb78fac2e", "clip-ee98a7319017ee0c", "red spoon-5ab2099d40423ca8", "coffee grinder top lid-d4f233de88dcb53f", "kettle-7ee96b72534023dd", "white plate-565a45d3f467e7a3", "cling film ball-ee06a2883ef92089", "the top one of two small grey plates-76bb7db989ed6749", "coffee container-30ea4677f09d18c4", "orange cloth-c1934fd49f6621c2", "bread plastic packaging-20b7cb0754ce4586", "slice of bread-bf74962e8134050e", "tablecloth-8ae87725bcebbc1a", "second tablecloth-a985a9aef85303f1", "plastic bottle-f665f8e6f5d29ce8", "red glass-762ae64171db9c3a", "green glass-236693d0f663eeab", "tissue box-128cbae444c479f3", "peanut butter jar-653b89d28122b0aa", "butter knife-5e5490be80423458", "bread and plate-f68defa01ab3d275" ], "reasoning": "1. I looked for the moment in the video where the user grabbed the first black object with their right hand, and saw that it happened at 0:09.\n2. I saw that the user was holding a charger (hoover charger-4e8ef85738d76854), and scrolled backwards throughout the video to find the moment the user began to hold the vacuum cleaner (something long and white-449c110dd7bb304b).\n3. At 00:01, I saw the user reach for the vacuum (something long and white-449c110dd7bb304b), and continue to hold it until 00:09 until they reached for and held the black charger (hoover charger-4e8ef85738d76854) in their right hand.", "referenced_object_masks": [ "hoover charger-4e8ef85738d76854", "something long and white-449c110dd7bb304b" ] }, { "key": "P03-20240218-094452:868aa14e82f2c98376e4e5dc256ae1dd966b55b8", "video_id": "P03-20240218-094452", "question": "If the largest amount of plates I held in one hand while holding more than one plate, was raised to the power of how many times I held multiple plates in one hand, what would the total be?", "answer_choice_0": "6 plates", "answer_choice_1": "24 plates", "answer_choice_2": "8 plates", "answer_choice_3": "10 plates", "answer_choice_4": "12 plates", "answer_id": 2, "answer": "8 plates", "question_type": [ "Counting", "Counterfactual", "Egocentric & Spatial Perception", "Numerical Reasoning", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "something long and white-449c110dd7bb304b", "hoover charger-4e8ef85738d76854", "buns package-1e0a7dce05008c66", "package of frozen food-c0af8c59a26d36e5", "steam cooker-4bffeb5a879b210e", "rice measuring cup-ca5c6eeb6a5111e9", "black cap-eec6bdde683a7264", "four buns-5aefa9d7c7b9f43a", "bun 1-b153b46483471edc", "bun 2-84a75bc192c1de64", "dishwasher brush-9522757b807d2844", "grey cloth-223b662e1a6b3f65", "saucepan-cb7b4af5dd382732", "other piece of padding-d1e249a63ce1c98b", "pan-c3fe95200e66347d", "small frying pan-a39a6b4e9c4f894a", "smaller saucepan-3a5a53c78d86b2a7", "another larger saucepan-e7840252be57868f", "cloth-f013b49ae87fbf35", "lid-c8f06d9b2762a44a", "larger lid-bb8d6ac5951230e3", "plate-45fcc9fab911551b", "another plate-0791ef8271b30668", "small bowl-63e03354dc8a7770", "second small bowl-7c76e947f5260475", "stack of small bowls-c8af4a9e9cec794b", "large chopping board-de6f9f9ad64084f6", "smaller chopping board-c956dc0ca7d2c860", "fork-3211e1268cb7c307", "knife-5f481b39babbe740", "another knife-3c8adaacf8da0ba0", "another fork-d20b12bb6e8fa10f", "spoon-e8abb2d89ec57c0b", "teaspoon-1b3d5f7097fb5d67", "dishwasher cutlery tray-953bdfeb301f8e0d", "knife in dishwasher cutlery tray-3003f2e17300cdeb", "green knife in dishwasher cutlery tray-7866f99b18a29858", "spoon in dishwasher cutlery tray-794d93a9959561d6", "fork in dishwasher cutlery tray-19cb6452869192a6", "coffee spoon in dishwasher cutlery tray-7b69724270924207", "another spoon in dishwasher cutlery tray-a30e7b72794583d8", "another knife in dishwasher cutlery tray-c74afe2dbe9d38d2", "another fork in dishwasher cutlery tray-d66149e07df06890", "teaspoon in dishwasher cutlery tray-c678067bc8afa8e3", "another teaspoon in dishwasher cutlery tray-40e2fc7a314e5c07", "third fork in dishwasher cutlery tray-dfdc1cad36d24bf0", "third teaspoon in dishwasher cutlery tray-bd407d3562b5a5e3", "fourth teaspoon in dishwasher cutlery tray-aeb522f581fb7cd6", "two small grey plates-c1a9aa81a9575967", "one grey plate-34566b83d7c9713c", "fork on gray plate-d38762e2560c6aa5", "two small red plates-d203fa0af72f8cdf", "one red plate-1a2d1ff8e5ad8358", "bowl-135e0dfb6200084b", "ladle-f5851f27ec78a906", "spatula-485fb33cb31ea658", "white bowl-743a192de234a5b3", "plastic colander-1a470738623a3853", "white bowl inside plastic colander-a2da3094d2ea88c6", "grey bowl-de10639976089e78", "second white bowl-344218667f6d7425", "metal v60 coffee filter-ff8b2539e792bae5", "stack of four bowls-4e84a8ebb474cc07", "second small white bowl-7e373113f20278ae", "stack of four small bowls-a8583570f0407761", "new small bowl 1-326960c1c66c5cb0", "new small bowl 2-cc454d7b92c7c794", "right green cup-c26ce56033267a24", "left green cup-2db9e03a38cf3d27", "big grey cup-ebc65813ceb92966", "third green cup-3f56bc662dac0e69", "printed cup-7088cfd472efb9d4", "transparent cup-acc33a8af845cbc6", "transparent cup with blue pattern-44261cccd91ca9b9", "wooden spoon-5fc7faf0ec2572ba", "scissors-3912136e4b70d8d0", "clear plastic box-628c8fcc0cc63d20", "clear plastic box lid-4ed9d1af85419b14", "wooden pizza peel-ee1a56fe1fb252a2", "dough scraper two-4e12b1def9b0e1d0", "dough scraper steel-30bbaf52cc718dbe", "pizza utensils-7652ced72417c90d", "cake mould-b1697183bc4b6ec8", "new cake mould 2-a5786ff65e5d3cf2", "dough scraper-3b5112ce201d68e2", "rolling pin-fab5e8b30e2f7a0a", "colander-1eda12f891a123e8", "large bowl-dc020554539664e2", "coffee jug-5bf682ebe7469336", "v60 rack-ee8733904266d8fc", "v60 coffee filter-15225e36ef83154c", "coffee grinder-57d22fd7cc640d64", "coffee packet-861ac93bb78fac2e", "clip-ee98a7319017ee0c", "red spoon-5ab2099d40423ca8", "coffee grinder top lid-d4f233de88dcb53f", "kettle-7ee96b72534023dd", "white plate-565a45d3f467e7a3", "cling film ball-ee06a2883ef92089", "the top one of two small grey plates-76bb7db989ed6749", "coffee container-30ea4677f09d18c4", "orange cloth-c1934fd49f6621c2", "bread plastic packaging-20b7cb0754ce4586", "slice of bread-bf74962e8134050e", "tablecloth-8ae87725bcebbc1a", "second tablecloth-a985a9aef85303f1", "plastic bottle-f665f8e6f5d29ce8", "red glass-762ae64171db9c3a", "green glass-236693d0f663eeab", "tissue box-128cbae444c479f3", "peanut butter jar-653b89d28122b0aa", "butter knife-5e5490be80423458", "bread and plate-f68defa01ab3d275" ], "reasoning": "1. I watched the video to see the maximum amount of plates the user held in their hands.\n2. I found that the largest number of plates the user ever held in their hands, was two, at three instances. At 03:13, the user held two plates (another plate-0791ef8271b30668), another time at 04:33, and a final time at 04:37 with different plates (plate-45fcc9fab911551b).\n3. If I raise two to the power of three, which is the largest number of plates held in the user's hand, raised to the power of how many times multiple plates were held in one hand, the answer would be 8.", "referenced_object_masks": [ "plate-45fcc9fab911551b", "another plate-0791ef8271b30668" ] }, { "key": "P03-20240218-094452:d6c7a954833c20a7bebdb7c9e7228535c916770c", "video_id": "P03-20240218-094452", "question": "How many plates from the dishwasher do I put into the dish cabinet to the right of the sink?", "answer_choice_0": "3", "answer_choice_1": "2", "answer_choice_2": "1", "answer_choice_3": "4", "answer_choice_4": "0", "answer_id": 3, "answer": "4", "question_type": [ "Object Recognition", "Situational Awareness", "Temporal Reasoning" ], "objects_of_interest": [ "something long and white-449c110dd7bb304b", "hoover charger-4e8ef85738d76854", "buns package-1e0a7dce05008c66", "package of frozen food-c0af8c59a26d36e5", "steam cooker-4bffeb5a879b210e", "rice measuring cup-ca5c6eeb6a5111e9", "black cap-eec6bdde683a7264", "four buns-5aefa9d7c7b9f43a", "bun 1-b153b46483471edc", "bun 2-84a75bc192c1de64", "dishwasher brush-9522757b807d2844", "grey cloth-223b662e1a6b3f65", "saucepan-cb7b4af5dd382732", "other piece of padding-d1e249a63ce1c98b", "pan-c3fe95200e66347d", "small frying pan-a39a6b4e9c4f894a", "smaller saucepan-3a5a53c78d86b2a7", "another larger saucepan-e7840252be57868f", "cloth-f013b49ae87fbf35", "lid-c8f06d9b2762a44a", "larger lid-bb8d6ac5951230e3", "plate-45fcc9fab911551b", "another plate-0791ef8271b30668", "small bowl-63e03354dc8a7770", "second small bowl-7c76e947f5260475", "stack of small bowls-c8af4a9e9cec794b", "large chopping board-de6f9f9ad64084f6", "smaller chopping board-c956dc0ca7d2c860", "fork-3211e1268cb7c307", "knife-5f481b39babbe740", "another knife-3c8adaacf8da0ba0", "another fork-d20b12bb6e8fa10f", "spoon-e8abb2d89ec57c0b", "teaspoon-1b3d5f7097fb5d67", "dishwasher cutlery tray-953bdfeb301f8e0d", "knife in dishwasher cutlery tray-3003f2e17300cdeb", "green knife in dishwasher cutlery tray-7866f99b18a29858", "spoon in dishwasher cutlery tray-794d93a9959561d6", "fork in dishwasher cutlery tray-19cb6452869192a6", "coffee spoon in dishwasher cutlery tray-7b69724270924207", "another spoon in dishwasher cutlery tray-a30e7b72794583d8", "another knife in dishwasher cutlery tray-c74afe2dbe9d38d2", "another fork in dishwasher cutlery tray-d66149e07df06890", "teaspoon in dishwasher cutlery tray-c678067bc8afa8e3", "another teaspoon in dishwasher cutlery tray-40e2fc7a314e5c07", "third fork in dishwasher cutlery tray-dfdc1cad36d24bf0", "third teaspoon in dishwasher cutlery tray-bd407d3562b5a5e3", "fourth teaspoon in dishwasher cutlery tray-aeb522f581fb7cd6", "two small grey plates-c1a9aa81a9575967", "one grey plate-34566b83d7c9713c", "fork on gray plate-d38762e2560c6aa5", "two small red plates-d203fa0af72f8cdf", "one red plate-1a2d1ff8e5ad8358", "bowl-135e0dfb6200084b", "ladle-f5851f27ec78a906", "spatula-485fb33cb31ea658", "white bowl-743a192de234a5b3", "plastic colander-1a470738623a3853", "white bowl inside plastic colander-a2da3094d2ea88c6", "grey bowl-de10639976089e78", "second white bowl-344218667f6d7425", "metal v60 coffee filter-ff8b2539e792bae5", "stack of four bowls-4e84a8ebb474cc07", "second small white bowl-7e373113f20278ae", "stack of four small bowls-a8583570f0407761", "new small bowl 1-326960c1c66c5cb0", "new small bowl 2-cc454d7b92c7c794", "right green cup-c26ce56033267a24", "left green cup-2db9e03a38cf3d27", "big grey cup-ebc65813ceb92966", "third green cup-3f56bc662dac0e69", "printed cup-7088cfd472efb9d4", "transparent cup-acc33a8af845cbc6", "transparent cup with blue pattern-44261cccd91ca9b9", "wooden spoon-5fc7faf0ec2572ba", "scissors-3912136e4b70d8d0", "clear plastic box-628c8fcc0cc63d20", "clear plastic box lid-4ed9d1af85419b14", "wooden pizza peel-ee1a56fe1fb252a2", "dough scraper two-4e12b1def9b0e1d0", "dough scraper steel-30bbaf52cc718dbe", "pizza utensils-7652ced72417c90d", "cake mould-b1697183bc4b6ec8", "new cake mould 2-a5786ff65e5d3cf2", "dough scraper-3b5112ce201d68e2", "rolling pin-fab5e8b30e2f7a0a", "colander-1eda12f891a123e8", "large bowl-dc020554539664e2", "coffee jug-5bf682ebe7469336", "v60 rack-ee8733904266d8fc", "v60 coffee filter-15225e36ef83154c", "coffee grinder-57d22fd7cc640d64", "coffee packet-861ac93bb78fac2e", "clip-ee98a7319017ee0c", "red spoon-5ab2099d40423ca8", "coffee grinder top lid-d4f233de88dcb53f", "kettle-7ee96b72534023dd", "white plate-565a45d3f467e7a3", "cling film ball-ee06a2883ef92089", "the top one of two small grey plates-76bb7db989ed6749", "coffee container-30ea4677f09d18c4", "orange cloth-c1934fd49f6621c2", "bread plastic packaging-20b7cb0754ce4586", "slice of bread-bf74962e8134050e", "tablecloth-8ae87725bcebbc1a", "second tablecloth-a985a9aef85303f1", "plastic bottle-f665f8e6f5d29ce8", "red glass-762ae64171db9c3a", "green glass-236693d0f663eeab", "tissue box-128cbae444c479f3", "peanut butter jar-653b89d28122b0aa", "butter knife-5e5490be80423458", "bread and plate-f68defa01ab3d275" ], "reasoning": "1. I watched the whole video.\n2. At 03:05, I took 2 white plates from the dishwasher and put them in the dish cabinet to the right of the sink.\n3. At 04:35, I took 2 tan and brown plates from the dishwasher and put them in the dish cabinet to the right of the sink.\n4. So 2 white plates plus 2 tan and brown plates equals 4 plates I put in the dish cabinet.", "referenced_object_masks": [] }, { "key": "P03-20240218-094452:d7d7208643a0661af9b87d126d3eaf97a17d6552", "video_id": "P03-20240218-094452", "question": "What is the ratio of the number of times I looked at my watch to the number of times I went to the kitchen table?", "answer_choice_0": "1:2", "answer_choice_1": "2:3", "answer_choice_2": "2:5", "answer_choice_3": "3:5", "answer_choice_4": "3:4", "answer_id": 3, "answer": "3:5", "question_type": [ "Action Recognition", "Counting", "Goal Reasoning", "Numerical Reasoning", "Object Recognition", "Situational Awareness", "Temporal Reasoning" ], "objects_of_interest": [ "something long and white-449c110dd7bb304b", "hoover charger-4e8ef85738d76854", "buns package-1e0a7dce05008c66", "package of frozen food-c0af8c59a26d36e5", "steam cooker-4bffeb5a879b210e", "rice measuring cup-ca5c6eeb6a5111e9", "black cap-eec6bdde683a7264", "four buns-5aefa9d7c7b9f43a", "bun 1-b153b46483471edc", "bun 2-84a75bc192c1de64", "dishwasher brush-9522757b807d2844", "grey cloth-223b662e1a6b3f65", "saucepan-cb7b4af5dd382732", "other piece of padding-d1e249a63ce1c98b", "pan-c3fe95200e66347d", "small frying pan-a39a6b4e9c4f894a", "smaller saucepan-3a5a53c78d86b2a7", "another larger saucepan-e7840252be57868f", "cloth-f013b49ae87fbf35", "lid-c8f06d9b2762a44a", "larger lid-bb8d6ac5951230e3", "plate-45fcc9fab911551b", "another plate-0791ef8271b30668", "small bowl-63e03354dc8a7770", "second small bowl-7c76e947f5260475", "stack of small bowls-c8af4a9e9cec794b", "large chopping board-de6f9f9ad64084f6", "smaller chopping board-c956dc0ca7d2c860", "fork-3211e1268cb7c307", "knife-5f481b39babbe740", "another knife-3c8adaacf8da0ba0", "another fork-d20b12bb6e8fa10f", "spoon-e8abb2d89ec57c0b", "teaspoon-1b3d5f7097fb5d67", "dishwasher cutlery tray-953bdfeb301f8e0d", "knife in dishwasher cutlery tray-3003f2e17300cdeb", "green knife in dishwasher cutlery tray-7866f99b18a29858", "spoon in dishwasher cutlery tray-794d93a9959561d6", "fork in dishwasher cutlery tray-19cb6452869192a6", "coffee spoon in dishwasher cutlery tray-7b69724270924207", "another spoon in dishwasher cutlery tray-a30e7b72794583d8", "another knife in dishwasher cutlery tray-c74afe2dbe9d38d2", "another fork in dishwasher cutlery tray-d66149e07df06890", "teaspoon in dishwasher cutlery tray-c678067bc8afa8e3", "another teaspoon in dishwasher cutlery tray-40e2fc7a314e5c07", "third fork in dishwasher cutlery tray-dfdc1cad36d24bf0", "third teaspoon in dishwasher cutlery tray-bd407d3562b5a5e3", "fourth teaspoon in dishwasher cutlery tray-aeb522f581fb7cd6", "two small grey plates-c1a9aa81a9575967", "one grey plate-34566b83d7c9713c", "fork on gray plate-d38762e2560c6aa5", "two small red plates-d203fa0af72f8cdf", "one red plate-1a2d1ff8e5ad8358", "bowl-135e0dfb6200084b", "ladle-f5851f27ec78a906", "spatula-485fb33cb31ea658", "white bowl-743a192de234a5b3", "plastic colander-1a470738623a3853", "white bowl inside plastic colander-a2da3094d2ea88c6", "grey bowl-de10639976089e78", "second white bowl-344218667f6d7425", "metal v60 coffee filter-ff8b2539e792bae5", "stack of four bowls-4e84a8ebb474cc07", "second small white bowl-7e373113f20278ae", "stack of four small bowls-a8583570f0407761", "new small bowl 1-326960c1c66c5cb0", "new small bowl 2-cc454d7b92c7c794", "right green cup-c26ce56033267a24", "left green cup-2db9e03a38cf3d27", "big grey cup-ebc65813ceb92966", "third green cup-3f56bc662dac0e69", "printed cup-7088cfd472efb9d4", "transparent cup-acc33a8af845cbc6", "transparent cup with blue pattern-44261cccd91ca9b9", "wooden spoon-5fc7faf0ec2572ba", "scissors-3912136e4b70d8d0", "clear plastic box-628c8fcc0cc63d20", "clear plastic box lid-4ed9d1af85419b14", "wooden pizza peel-ee1a56fe1fb252a2", "dough scraper two-4e12b1def9b0e1d0", "dough scraper steel-30bbaf52cc718dbe", "pizza utensils-7652ced72417c90d", "cake mould-b1697183bc4b6ec8", "new cake mould 2-a5786ff65e5d3cf2", "dough scraper-3b5112ce201d68e2", "rolling pin-fab5e8b30e2f7a0a", "colander-1eda12f891a123e8", "large bowl-dc020554539664e2", "coffee jug-5bf682ebe7469336", "v60 rack-ee8733904266d8fc", "v60 coffee filter-15225e36ef83154c", "coffee grinder-57d22fd7cc640d64", "coffee packet-861ac93bb78fac2e", "clip-ee98a7319017ee0c", "red spoon-5ab2099d40423ca8", "coffee grinder top lid-d4f233de88dcb53f", "kettle-7ee96b72534023dd", "white plate-565a45d3f467e7a3", "cling film ball-ee06a2883ef92089", "the top one of two small grey plates-76bb7db989ed6749", "coffee container-30ea4677f09d18c4", "orange cloth-c1934fd49f6621c2", "bread plastic packaging-20b7cb0754ce4586", "slice of bread-bf74962e8134050e", "tablecloth-8ae87725bcebbc1a", "second tablecloth-a985a9aef85303f1", "plastic bottle-f665f8e6f5d29ce8", "red glass-762ae64171db9c3a", "green glass-236693d0f663eeab", "tissue box-128cbae444c479f3", "peanut butter jar-653b89d28122b0aa", "butter knife-5e5490be80423458", "bread and plate-f68defa01ab3d275" ], "reasoning": "1. I looked at my watch at 01:43, 13:57, and 16:25. So it was 3 times. \n2. I went to the kitchen table at 12:15 to bring cloth napkins . \n3. At 13:10, I brought a knife (butter knife-5e5490be80423458) to the kitchen table .\n4. At 15:36, I brought 2 coffee mugs with coffee to the kitchen table .\n5. At 16:04, I brought a fork (fork on gray plate-d38762e2560c6aa5) to the kitchen table .\n6. At 17:01, I brought toast and an omelet (bread and plate-f68defa01ab3d275) to the kitchen table .\n7. The number of times I went to the kitchen table is 5.\n8. The ratio of the number of times I looked at my watch to the number of times I went to the kitchen table is 3:5.", "referenced_object_masks": [ "fork on gray plate-d38762e2560c6aa5", "butter knife-5e5490be80423458", "bread and plate-f68defa01ab3d275" ] }, { "key": "P03-20240218-094452:fcc738a8654d4b9e98521de8296443aadf549842", "video_id": "P03-20240218-094452", "question": "Where is the washing machine in relation to me at 12:40?", "answer_choice_0": "To my right", "answer_choice_1": "Directly in front of me", "answer_choice_2": "Down the hall to my left", "answer_choice_3": "To my left", "answer_choice_4": "Down the hall to my right", "answer_id": 3, "answer": "To my left", "question_type": [ "Object Recognition", "Egocentric & Spatial Perception", "Temporal Reasoning" ], "objects_of_interest": [ "something long and white-449c110dd7bb304b", "hoover charger-4e8ef85738d76854", "buns package-1e0a7dce05008c66", "package of frozen food-c0af8c59a26d36e5", "steam cooker-4bffeb5a879b210e", "rice measuring cup-ca5c6eeb6a5111e9", "black cap-eec6bdde683a7264", "four buns-5aefa9d7c7b9f43a", "bun 1-b153b46483471edc", "bun 2-84a75bc192c1de64", "dishwasher brush-9522757b807d2844", "grey cloth-223b662e1a6b3f65", "saucepan-cb7b4af5dd382732", "other piece of padding-d1e249a63ce1c98b", "pan-c3fe95200e66347d", "small frying pan-a39a6b4e9c4f894a", "smaller saucepan-3a5a53c78d86b2a7", "another larger saucepan-e7840252be57868f", "cloth-f013b49ae87fbf35", "lid-c8f06d9b2762a44a", "larger lid-bb8d6ac5951230e3", "plate-45fcc9fab911551b", "another plate-0791ef8271b30668", "small bowl-63e03354dc8a7770", "second small bowl-7c76e947f5260475", "stack of small bowls-c8af4a9e9cec794b", "large chopping board-de6f9f9ad64084f6", "smaller chopping board-c956dc0ca7d2c860", "fork-3211e1268cb7c307", "knife-5f481b39babbe740", "another knife-3c8adaacf8da0ba0", "another fork-d20b12bb6e8fa10f", "spoon-e8abb2d89ec57c0b", "teaspoon-1b3d5f7097fb5d67", "dishwasher cutlery tray-953bdfeb301f8e0d", "knife in dishwasher cutlery tray-3003f2e17300cdeb", "green knife in dishwasher cutlery tray-7866f99b18a29858", "spoon in dishwasher cutlery tray-794d93a9959561d6", "fork in dishwasher cutlery tray-19cb6452869192a6", "coffee spoon in dishwasher cutlery tray-7b69724270924207", "another spoon in dishwasher cutlery tray-a30e7b72794583d8", "another knife in dishwasher cutlery tray-c74afe2dbe9d38d2", "another fork in dishwasher cutlery tray-d66149e07df06890", "teaspoon in dishwasher cutlery tray-c678067bc8afa8e3", "another teaspoon in dishwasher cutlery tray-40e2fc7a314e5c07", "third fork in dishwasher cutlery tray-dfdc1cad36d24bf0", "third teaspoon in dishwasher cutlery tray-bd407d3562b5a5e3", "fourth teaspoon in dishwasher cutlery tray-aeb522f581fb7cd6", "two small grey plates-c1a9aa81a9575967", "one grey plate-34566b83d7c9713c", "fork on gray plate-d38762e2560c6aa5", "two small red plates-d203fa0af72f8cdf", "one red plate-1a2d1ff8e5ad8358", "bowl-135e0dfb6200084b", "ladle-f5851f27ec78a906", "spatula-485fb33cb31ea658", "white bowl-743a192de234a5b3", "plastic colander-1a470738623a3853", "white bowl inside plastic colander-a2da3094d2ea88c6", "grey bowl-de10639976089e78", "second white bowl-344218667f6d7425", "metal v60 coffee filter-ff8b2539e792bae5", "stack of four bowls-4e84a8ebb474cc07", "second small white bowl-7e373113f20278ae", "stack of four small bowls-a8583570f0407761", "new small bowl 1-326960c1c66c5cb0", "new small bowl 2-cc454d7b92c7c794", "right green cup-c26ce56033267a24", "left green cup-2db9e03a38cf3d27", "big grey cup-ebc65813ceb92966", "third green cup-3f56bc662dac0e69", "printed cup-7088cfd472efb9d4", "transparent cup-acc33a8af845cbc6", "transparent cup with blue pattern-44261cccd91ca9b9", "wooden spoon-5fc7faf0ec2572ba", "scissors-3912136e4b70d8d0", "clear plastic box-628c8fcc0cc63d20", "clear plastic box lid-4ed9d1af85419b14", "wooden pizza peel-ee1a56fe1fb252a2", "dough scraper two-4e12b1def9b0e1d0", "dough scraper steel-30bbaf52cc718dbe", "pizza utensils-7652ced72417c90d", "cake mould-b1697183bc4b6ec8", "new cake mould 2-a5786ff65e5d3cf2", "dough scraper-3b5112ce201d68e2", "rolling pin-fab5e8b30e2f7a0a", "colander-1eda12f891a123e8", "large bowl-dc020554539664e2", "coffee jug-5bf682ebe7469336", "v60 rack-ee8733904266d8fc", "v60 coffee filter-15225e36ef83154c", "coffee grinder-57d22fd7cc640d64", "coffee packet-861ac93bb78fac2e", "clip-ee98a7319017ee0c", "red spoon-5ab2099d40423ca8", "coffee grinder top lid-d4f233de88dcb53f", "kettle-7ee96b72534023dd", "white plate-565a45d3f467e7a3", "cling film ball-ee06a2883ef92089", "the top one of two small grey plates-76bb7db989ed6749", "coffee container-30ea4677f09d18c4", "orange cloth-c1934fd49f6621c2", "bread plastic packaging-20b7cb0754ce4586", "slice of bread-bf74962e8134050e", "tablecloth-8ae87725bcebbc1a", "second tablecloth-a985a9aef85303f1", "plastic bottle-f665f8e6f5d29ce8", "red glass-762ae64171db9c3a", "green glass-236693d0f663eeab", "tissue box-128cbae444c479f3", "peanut butter jar-653b89d28122b0aa", "butter knife-5e5490be80423458", "bread and plate-f68defa01ab3d275" ], "reasoning": "1. I watched the whole video. \n2. At 00:03, I can see the washing machine to the right of the kitchen sink .\n3. At 02:29, I am emptying the dishwasher which is on the left side of the kitchen. The sink and washing machine are to my right. \n4. At 12:40, I am getting things out of a cabinet on the right side of the kitchen. \n5. The washing machine is to my left, next to the sink.", "referenced_object_masks": [] }, { "key": "P03-20240218-094452:ff9e5291e24e937671be25b6e828f26698364392", "video_id": "P03-20240218-094452", "question": "How many times do I wipe the counter between grinding the coffee and starting to brew it?", "answer_choice_0": "9", "answer_choice_1": "8", "answer_choice_2": "11", "answer_choice_3": "7", "answer_choice_4": "10", "answer_id": 0, "answer": "9", "question_type": [ "Action Recognition", "Counterfactual", "Event Occurence", "Object Recognition", "Counting", "Temporal Reasoning" ], "objects_of_interest": [ "something long and white-449c110dd7bb304b", "hoover charger-4e8ef85738d76854", "buns package-1e0a7dce05008c66", "package of frozen food-c0af8c59a26d36e5", "steam cooker-4bffeb5a879b210e", "rice measuring cup-ca5c6eeb6a5111e9", "black cap-eec6bdde683a7264", "four buns-5aefa9d7c7b9f43a", "bun 1-b153b46483471edc", "bun 2-84a75bc192c1de64", "dishwasher brush-9522757b807d2844", "grey cloth-223b662e1a6b3f65", "saucepan-cb7b4af5dd382732", "other piece of padding-d1e249a63ce1c98b", "pan-c3fe95200e66347d", "small frying pan-a39a6b4e9c4f894a", "smaller saucepan-3a5a53c78d86b2a7", "another larger saucepan-e7840252be57868f", "cloth-f013b49ae87fbf35", "lid-c8f06d9b2762a44a", "larger lid-bb8d6ac5951230e3", "plate-45fcc9fab911551b", "another plate-0791ef8271b30668", "small bowl-63e03354dc8a7770", "second small bowl-7c76e947f5260475", "stack of small bowls-c8af4a9e9cec794b", "large chopping board-de6f9f9ad64084f6", "smaller chopping board-c956dc0ca7d2c860", "fork-3211e1268cb7c307", "knife-5f481b39babbe740", "another knife-3c8adaacf8da0ba0", "another fork-d20b12bb6e8fa10f", "spoon-e8abb2d89ec57c0b", "teaspoon-1b3d5f7097fb5d67", "dishwasher cutlery tray-953bdfeb301f8e0d", "knife in dishwasher cutlery tray-3003f2e17300cdeb", "green knife in dishwasher cutlery tray-7866f99b18a29858", "spoon in dishwasher cutlery tray-794d93a9959561d6", "fork in dishwasher cutlery tray-19cb6452869192a6", "coffee spoon in dishwasher cutlery tray-7b69724270924207", "another spoon in dishwasher cutlery tray-a30e7b72794583d8", "another knife in dishwasher cutlery tray-c74afe2dbe9d38d2", "another fork in dishwasher cutlery tray-d66149e07df06890", "teaspoon in dishwasher cutlery tray-c678067bc8afa8e3", "another teaspoon in dishwasher cutlery tray-40e2fc7a314e5c07", "third fork in dishwasher cutlery tray-dfdc1cad36d24bf0", "third teaspoon in dishwasher cutlery tray-bd407d3562b5a5e3", "fourth teaspoon in dishwasher cutlery tray-aeb522f581fb7cd6", "two small grey plates-c1a9aa81a9575967", "one grey plate-34566b83d7c9713c", "fork on gray plate-d38762e2560c6aa5", "two small red plates-d203fa0af72f8cdf", "one red plate-1a2d1ff8e5ad8358", "bowl-135e0dfb6200084b", "ladle-f5851f27ec78a906", "spatula-485fb33cb31ea658", "white bowl-743a192de234a5b3", "plastic colander-1a470738623a3853", "white bowl inside plastic colander-a2da3094d2ea88c6", "grey bowl-de10639976089e78", "second white bowl-344218667f6d7425", "metal v60 coffee filter-ff8b2539e792bae5", "stack of four bowls-4e84a8ebb474cc07", "second small white bowl-7e373113f20278ae", "stack of four small bowls-a8583570f0407761", "new small bowl 1-326960c1c66c5cb0", "new small bowl 2-cc454d7b92c7c794", "right green cup-c26ce56033267a24", "left green cup-2db9e03a38cf3d27", "big grey cup-ebc65813ceb92966", "third green cup-3f56bc662dac0e69", "printed cup-7088cfd472efb9d4", "transparent cup-acc33a8af845cbc6", "transparent cup with blue pattern-44261cccd91ca9b9", "wooden spoon-5fc7faf0ec2572ba", "scissors-3912136e4b70d8d0", "clear plastic box-628c8fcc0cc63d20", "clear plastic box lid-4ed9d1af85419b14", "wooden pizza peel-ee1a56fe1fb252a2", "dough scraper two-4e12b1def9b0e1d0", "dough scraper steel-30bbaf52cc718dbe", "pizza utensils-7652ced72417c90d", "cake mould-b1697183bc4b6ec8", "new cake mould 2-a5786ff65e5d3cf2", "dough scraper-3b5112ce201d68e2", "rolling pin-fab5e8b30e2f7a0a", "colander-1eda12f891a123e8", "large bowl-dc020554539664e2", "coffee jug-5bf682ebe7469336", "v60 rack-ee8733904266d8fc", "v60 coffee filter-15225e36ef83154c", "coffee grinder-57d22fd7cc640d64", "coffee packet-861ac93bb78fac2e", "clip-ee98a7319017ee0c", "red spoon-5ab2099d40423ca8", "coffee grinder top lid-d4f233de88dcb53f", "kettle-7ee96b72534023dd", "white plate-565a45d3f467e7a3", "cling film ball-ee06a2883ef92089", "the top one of two small grey plates-76bb7db989ed6749", "coffee container-30ea4677f09d18c4", "orange cloth-c1934fd49f6621c2", "bread plastic packaging-20b7cb0754ce4586", "slice of bread-bf74962e8134050e", "tablecloth-8ae87725bcebbc1a", "second tablecloth-a985a9aef85303f1", "plastic bottle-f665f8e6f5d29ce8", "red glass-762ae64171db9c3a", "green glass-236693d0f663eeab", "tissue box-128cbae444c479f3", "peanut butter jar-653b89d28122b0aa", "butter knife-5e5490be80423458", "bread and plate-f68defa01ab3d275" ], "reasoning": "1. I looked for the user to grind coffee and brew it in order to define the range to count how many times they wiped the counter.\n2. I saw the user grind coffee in a grinder (coffee grinder-57d22fd7cc640d64) at 08:49.\n3. I saw the user begin to brew their coffee by pouring hot water from a kettle (kettle-7ee96b72534023dd) over their coffee grounds at 11:54.\n4. I counted how many times the user wiped the counter between 08:49-11:54. I saw the user pick up an orange cloth (orange cloth-c1934fd49f6621c2) at 10:31 and use it to begin wiping the counter at 10:43.\n5. I counted the user wiping the counter 9 times between 10:43 and 10:54 before hanging the cloth (orange cloth-c1934fd49f6621c2) on the faucet at 10:56.\n6. At 11:04 I saw the user pull a white towel (cloth-f013b49ae87fbf35) across the top of the counter, but they appeared to be retrieving the cloth (cloth-f013b49ae87fbf35) and not wiping, so I didn't count this instance.", "referenced_object_masks": [ "cloth-f013b49ae87fbf35", "coffee grinder-57d22fd7cc640d64", "kettle-7ee96b72534023dd", "orange cloth-c1934fd49f6621c2" ] }, { "key": "P03-20240218-101509:40dd0bf7dc35c9523350df1852528ef22850a064", "video_id": "P03-20240218-101509", "question": "If I walked into the kitchen through the kitchen door, what would be the second object I passed immediately to my left?", "answer_choice_0": "A shoe rack", "answer_choice_1": "A light switch", "answer_choice_2": "A pack of tissues", "answer_choice_3": "A water dispenser", "answer_choice_4": "A table", "answer_id": 3, "answer": "A water dispenser", "question_type": [ "Counterfactual", "Action Recognition", "Egocentric & Spatial Perception", "Object Recognition" ], "objects_of_interest": [ "fork-948d5da449ae356e", "dirty cutlery-e1a4231e610e424e", "knife-dd61fb04a2b2d2f1", "plate1-97dc21acfc402d2a", "plate2-2a86823d4fde5230", "dishwasher brush-4f2b39ba9daa25f7", "chopstick-4a7987d2f0db2756", "ramekin-3d1611045bf25276", "chair-7c5ee2fbf9232866", "other chair-031dc6d8cdc03948", "used tissues-2dc0b4d033b669e7", "crumpled tissues-547c064787ffc9b2", "jar-1a7d06ef0847780d", "cup-7bb51eff3acef73a", "toaster-6db4454959156097", "food box-18794822d1f496a3", "box-70f323f530e44f70", "food box lid-69894f926a7ddf9f", "bun-d17103bbcde45171", "other bun-1fd09e70bb55261f", "tea towels-5051a6c5efc84053", "tea towels2-2dcda7389db5a0bf", "kettle-f6d92e8afc1b1976", "coffee filter-c3eb2d0e27fc0334", "coffee filter holder-911203ba19184fca", "jug-195cbfdda79200a7", "cloth-40bcef18ee6d59d7", "coffee grinder-bbf871543a18594f", "steaming tray-febf19494b206440" ], "reasoning": "1. I observed the layout of the kitchen and noted the position of the kitchen door at 00:07.\n2. Because the question asks what would be on the user's left when entering, I noted the objects to the right of the door when the user is looking out in that direction.\n3. I can see a pack of tissue closest to the door on a windowsill, and just past that, closer to the center of the kitchen, there is a water dispenser on the counter below.\n4. Therefore if the user were to walk in from the kitchen door, the water dispenser would be the second object they pass on their left.", "referenced_object_masks": [] }, { "key": "P03-20240218-101509:78d8df0d60b5f7caa83c97c3bcafe242f41b1c0d", "video_id": "P03-20240218-101509", "question": "Excluding when I gathered them together on the table, how many total folds did I make with the tea towels?", "answer_choice_0": "5", "answer_choice_1": "4", "answer_choice_2": "7", "answer_choice_3": "8", "answer_choice_4": "6", "answer_id": 4, "answer": "6", "question_type": [ "Action Recognition", "Counting", "Egocentric & Spatial Perception", "Object Recognition", "Numerical Reasoning" ], "objects_of_interest": [ "fork-948d5da449ae356e", "dirty cutlery-e1a4231e610e424e", "knife-dd61fb04a2b2d2f1", "plate1-97dc21acfc402d2a", "plate2-2a86823d4fde5230", "dishwasher brush-4f2b39ba9daa25f7", "chopstick-4a7987d2f0db2756", "ramekin-3d1611045bf25276", "chair-7c5ee2fbf9232866", "other chair-031dc6d8cdc03948", "used tissues-2dc0b4d033b669e7", "crumpled tissues-547c064787ffc9b2", "jar-1a7d06ef0847780d", "cup-7bb51eff3acef73a", "toaster-6db4454959156097", "food box-18794822d1f496a3", "box-70f323f530e44f70", "food box lid-69894f926a7ddf9f", "bun-d17103bbcde45171", "other bun-1fd09e70bb55261f", "tea towels-5051a6c5efc84053", "tea towels2-2dcda7389db5a0bf", "kettle-f6d92e8afc1b1976", "coffee filter-c3eb2d0e27fc0334", "coffee filter holder-911203ba19184fca", "jug-195cbfdda79200a7", "cloth-40bcef18ee6d59d7", "coffee grinder-bbf871543a18594f", "steaming tray-febf19494b206440" ], "reasoning": "1. I watched for the user to interact with tea towels during the video.\n2. I saw the user gather the first tea towel (tea towels2-2dcda7389db5a0bf) at 02:12.\n3. I counted the user fold the first tea towel (tea towels2-2dcda7389db5a0bf) three times, first from 02:17-02:18, second at 02:21, and third at 02:22.\n4. I saw the user gather the second tea towel (tea towels-5051a6c5efc84053) at 02:27.\n5. I counted the user fold the second tea towel (tea towels-5051a6c5efc84053) three times, first at 02:35, second at 02:38, and third at 02:39.\n6. I added 3 and 3 to total 6.", "referenced_object_masks": [ "tea towels-5051a6c5efc84053", "tea towels2-2dcda7389db5a0bf" ] }, { "key": "P03-20240218-101509:90b36895e3187a1d7bc8e6b8a1b323cd0db4b928", "video_id": "P03-20240218-101509", "question": "What is one type of object on the surface immediately above where I stored the toaster?", "answer_choice_0": "Microwave", "answer_choice_1": "Coffee grinder", "answer_choice_2": "Cutting board", "answer_choice_3": "Pressure cooker", "answer_choice_4": "Paper towels", "answer_id": 2, "answer": "Cutting board", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Object Recognition" ], "objects_of_interest": [ "fork-948d5da449ae356e", "dirty cutlery-e1a4231e610e424e", "knife-dd61fb04a2b2d2f1", "plate1-97dc21acfc402d2a", "plate2-2a86823d4fde5230", "dishwasher brush-4f2b39ba9daa25f7", "chopstick-4a7987d2f0db2756", "ramekin-3d1611045bf25276", "chair-7c5ee2fbf9232866", "other chair-031dc6d8cdc03948", "used tissues-2dc0b4d033b669e7", "crumpled tissues-547c064787ffc9b2", "jar-1a7d06ef0847780d", "cup-7bb51eff3acef73a", "toaster-6db4454959156097", "food box-18794822d1f496a3", "box-70f323f530e44f70", "food box lid-69894f926a7ddf9f", "bun-d17103bbcde45171", "other bun-1fd09e70bb55261f", "tea towels-5051a6c5efc84053", "tea towels2-2dcda7389db5a0bf", "kettle-f6d92e8afc1b1976", "coffee filter-c3eb2d0e27fc0334", "coffee filter holder-911203ba19184fca", "jug-195cbfdda79200a7", "cloth-40bcef18ee6d59d7", "coffee grinder-bbf871543a18594f", "steaming tray-febf19494b206440" ], "reasoning": "1. I watched until I saw the user pick up the toaster (toaster-6db4454959156097) from its place on the counter at 01:27.\n2. From 01:27-01:35 I saw the user turn around and open a cabinet under the counter and store the toaster (toaster-6db4454959156097) on the bottom shelf.\n3. I looked at the shelf above the toaster (toaster-6db4454959156097), which is the most immediate surface above, and saw 2 plastic cutting boards .", "referenced_object_masks": [ "toaster-6db4454959156097" ] }, { "key": "P03-20240218-101509:922267a63e8b379700acd48f9291faa587258d34", "video_id": "P03-20240218-101509", "question": "Including the glass panels on the doors, how many more windowpanes are there than steamed buns?", "answer_choice_0": "7", "answer_choice_1": "6", "answer_choice_2": "9", "answer_choice_3": "8", "answer_choice_4": "5", "answer_id": 0, "answer": "7", "question_type": [ "Counting", "Numerical Reasoning", "Object Recognition" ], "objects_of_interest": [ "fork-948d5da449ae356e", "dirty cutlery-e1a4231e610e424e", "knife-dd61fb04a2b2d2f1", "plate1-97dc21acfc402d2a", "plate2-2a86823d4fde5230", "dishwasher brush-4f2b39ba9daa25f7", "chopstick-4a7987d2f0db2756", "ramekin-3d1611045bf25276", "chair-7c5ee2fbf9232866", "other chair-031dc6d8cdc03948", "used tissues-2dc0b4d033b669e7", "crumpled tissues-547c064787ffc9b2", "jar-1a7d06ef0847780d", "cup-7bb51eff3acef73a", "toaster-6db4454959156097", "food box-18794822d1f496a3", "box-70f323f530e44f70", "food box lid-69894f926a7ddf9f", "bun-d17103bbcde45171", "other bun-1fd09e70bb55261f", "tea towels-5051a6c5efc84053", "tea towels2-2dcda7389db5a0bf", "kettle-f6d92e8afc1b1976", "coffee filter-c3eb2d0e27fc0334", "coffee filter holder-911203ba19184fca", "jug-195cbfdda79200a7", "cloth-40bcef18ee6d59d7", "coffee grinder-bbf871543a18594f", "steaming tray-febf19494b206440" ], "reasoning": "1. I watched the video and looked for windowpanes to appear throughout.\n2. I saw the first two windowpanes above the sink at 00:06, a third windowpane in the door and a fourth beside it at 00:07, three more windowpanes through the windows to the left at 00:19, and two more windowpanes in the front door at 00:57 for a total of 9.\n3. I looked for steamed buns and counted two buns in a pressure cooker at 00:19.\n4. I subtracted 2 from 9 to reach 7.", "referenced_object_masks": [] }, { "key": "P03-20240218-101509:9657adfd93966abc64f6d8fe76e998029a47759a", "video_id": "P03-20240218-101509", "question": "After I finish folding the tea towels, how many fewer items are on the table than at the start of the video?", "answer_choice_0": "9", "answer_choice_1": "12", "answer_choice_2": "11", "answer_choice_3": "8", "answer_choice_4": "10", "answer_id": 2, "answer": "11", "question_type": [ "Action Recognition", "Counting", "Egocentric & Spatial Perception", "Object Recognition", "Numerical Reasoning", "State Changes" ], "objects_of_interest": [ "fork-948d5da449ae356e", "dirty cutlery-e1a4231e610e424e", "knife-dd61fb04a2b2d2f1", "plate1-97dc21acfc402d2a", "plate2-2a86823d4fde5230", "dishwasher brush-4f2b39ba9daa25f7", "chopstick-4a7987d2f0db2756", "ramekin-3d1611045bf25276", "chair-7c5ee2fbf9232866", "other chair-031dc6d8cdc03948", "used tissues-2dc0b4d033b669e7", "crumpled tissues-547c064787ffc9b2", "jar-1a7d06ef0847780d", "cup-7bb51eff3acef73a", "toaster-6db4454959156097", "food box-18794822d1f496a3", "box-70f323f530e44f70", "food box lid-69894f926a7ddf9f", "bun-d17103bbcde45171", "other bun-1fd09e70bb55261f", "tea towels-5051a6c5efc84053", "tea towels2-2dcda7389db5a0bf", "kettle-f6d92e8afc1b1976", "coffee filter-c3eb2d0e27fc0334", "coffee filter holder-911203ba19184fca", "jug-195cbfdda79200a7", "cloth-40bcef18ee6d59d7", "coffee grinder-bbf871543a18594f", "steaming tray-febf19494b206440" ], "reasoning": "1. I watched the video and looked for a table with objects on it and got a clear view at 00:01.\n2. I can count 15 objects on the table at 00:01.\n3. I can see the user take the last tea towel (tea towels2-2dcda7389db5a0bf) from the table at 02:28.\n4. I can count 4 objects remaining on the table at 02:28.\n5. I subtracted 4 from 15 to reach 11.", "referenced_object_masks": [ "tea towels2-2dcda7389db5a0bf" ] }, { "key": "P03-20240218-140054:0fb2efe27323e73f07073191d8361f4c62509fe5", "video_id": "P03-20240218-140054", "question": "What motion do I use most with the object I use to clean the floors?", "answer_choice_0": "Sweeping motions", "answer_choice_1": "Circular motions", "answer_choice_2": "Push and pull", "answer_choice_3": "Scrubbing motions", "answer_choice_4": "Dabbing motions", "answer_id": 2, "answer": "Push and pull", "question_type": [ "Object Recognition", "Event Occurence", "Action Recognition" ], "objects_of_interest": [ "container-3db98c624e413b2d", "hoover-fa42a926c802af23", "hoover charger-a0ed1d356c17b389", "chair-2fefa7fb254fe497", "second chair-11232e9af4b48fc0", "pack of taiwanese pancakes-ee302825e73eb626", "second saucepan-ffe93f5370033200", "new track 1-11a097c1ffc7cd41", "third saucepan-a3508894581f53d4", "larger saucepan-d632760b19d7e71d", "new track 3-5cafbfb12a904c77", "container box-23ea8486f458f6ac", "dirty mug-dbbc1376dee81466", "rice cooker bowl-02ad871e2228da09", "rice cooker tray-4dd2f4ce6ec011bf", "right placemat-5f3795bbcbf3af68", "left placemat-22e05f03d8cfa22f", "water filter tank-002884c4d601ff0e", "laundry detergent cap-b8cdcbc511ace0b1", "plastic bag-a7907e8b1b98e5b2", "box-7292668b779034a7", "scissors-64d7d9c0d7b996fb", "safety cover of scissors-989a23c52aedebd5", "dry wooden spoon-b3f394aa2d811358", "small bowl-141df35b7a42703a", "other food bag-3f630662c6d6df73", "bottle of sesame oil-933272b80328c99f", "another smaller bottle-cecf7711b49e5304", "another bottle-27d3b981e862da6c", "bottle of soy sauce-f9c400376933e0f2", "small cup-9c3bcbf40b2d6e87", "knife-37f2303d21d59017", "garlic cloves-a6635f799069d2db", "cloth-ab363aebeb8a4521", "coffee filter stand-87876f19c710b2fb", "coffee filter-98e44309a4afddda", "new coffee filter cone-b101b208b921ce07", "red spoon-591d984a7e12315e", "frozen dumplings-6afa13ccb0f4fb62", "second set of frozen dumplings-25135ae7dded01a8", "last frozen dumpling-d74a8290a9239a04", "empty plastic bag-63a325a5cdbccba8", "orange cloth-5ebfaaee70535f6a", "first chopsticks-1e42f51bdc8c76e7", "second chopsticks-9fb14c743e71e22e", "first red chopstick-e4688d3a0dd2b34e", "second red chopstick-5bf110b7efab0714", "other pink butterfly shaped chopstick holder-ab75d44751329343", "blue butterfly shaped chopstick holder-abe050773fd1b530", "red chopsticks-3a4b1d0fbf79b413", "folded white cloth-9eb79751f6a7636e", "top empty plastic bag-8125f8410a873826", "other empty food bag-25b1b286ba900bf6", "dishwasher brush-a28fbab556e8b019", "small sponge-13244022c8cacc49", "washing up liquid-a06b408a43cc43ab", "new small bowl-ebb4474d35cc2b7c", "second bowl in stack of bowls-c270ecc940081bc8", "top bowl in stack of bowls-7b4f34b9a92e4a1f", "stack of small bowls-a160bfdbc189c8c4", "bowl2-1eef02d7d93c4931", "bottom two big bowls-eaed4b7ecc151b96", "top two big bowls-1244e9499e09d47b", "new bottom bowls-5c0ad1805c5cc614", "glass-8ce3e24bf8ab5c6c", "new bottom bowl-b9e128f5249ea986", "new bottom bowl 2-4c5a23d029c68fc1", "first big bowl-e3bdac110de97f87", "plastic container-8dfb8583a0fea3b6", "box of grapes-d58b9f1d65ebe5b2", "plastic sleeve-3e3ccd99ab3c7c95", "grapes-83d33d9598b3cc68", "new track 6-5bcf3facf279db57", "small bunch of grapes-4db9de2cf57b75a4", "bunch of grapes-39380d796a05df50", "last bunch of grapes-612ff9ffd3a0b645", "plastic box-fc168a7d19f5f915", "small plastic colander-a0ac641b94e24899", "colander-156217f7f4400618", "new sieve-573cb366ca5a60dc", "plate-d707b77979daed6b", "new small bottom bowl-c13da7e8da35f780", "small stack of bowls-d1891513ba1b5022", "final bowl-5b35fac800bf9fa5", "bowl of grapes-3f45f17ad1296808" ], "reasoning": "1. I identified the primary object used in cleaning the floors from 00:22-02:22.\n2. I took note of the vacuum cleaner (hoover-fa42a926c802af23) the user uses to clean.\n3. I saw that it's either impossible, or highly unlikely that the vacuum (hoover-fa42a926c802af23) would be used for sweeping, scrubbing, dabbing, or circular motions. Indeed, from 00:22-02:22, the vacuum is used in a push and pull motion.\n4. Therefore, the answer is push and pull.", "referenced_object_masks": [ "hoover-fa42a926c802af23" ] }, { "key": "P03-20240218-140054:74622b4ccc275619d03732d4fa0e6e86137ce9be", "video_id": "P03-20240218-140054", "question": "What do I run the vacuum over that is directly below the water filter?", "answer_choice_0": "A gray rug", "answer_choice_1": "A red rug", "answer_choice_2": "A black doormat", "answer_choice_3": "A green rug", "answer_choice_4": "A red doormat", "answer_id": 2, "answer": "A black doormat", "question_type": [ "Action Recognition", "Object Recognition", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "container-3db98c624e413b2d", "hoover-fa42a926c802af23", "hoover charger-a0ed1d356c17b389", "chair-2fefa7fb254fe497", "second chair-11232e9af4b48fc0", "pack of taiwanese pancakes-ee302825e73eb626", "second saucepan-ffe93f5370033200", "new track 1-11a097c1ffc7cd41", "third saucepan-a3508894581f53d4", "larger saucepan-d632760b19d7e71d", "new track 3-5cafbfb12a904c77", "container box-23ea8486f458f6ac", "dirty mug-dbbc1376dee81466", "rice cooker bowl-02ad871e2228da09", "rice cooker tray-4dd2f4ce6ec011bf", "right placemat-5f3795bbcbf3af68", "left placemat-22e05f03d8cfa22f", "water filter tank-002884c4d601ff0e", "laundry detergent cap-b8cdcbc511ace0b1", "plastic bag-a7907e8b1b98e5b2", "box-7292668b779034a7", "scissors-64d7d9c0d7b996fb", "safety cover of scissors-989a23c52aedebd5", "dry wooden spoon-b3f394aa2d811358", "small bowl-141df35b7a42703a", "other food bag-3f630662c6d6df73", "bottle of sesame oil-933272b80328c99f", "another smaller bottle-cecf7711b49e5304", "another bottle-27d3b981e862da6c", "bottle of soy sauce-f9c400376933e0f2", "small cup-9c3bcbf40b2d6e87", "knife-37f2303d21d59017", "garlic cloves-a6635f799069d2db", "cloth-ab363aebeb8a4521", "coffee filter stand-87876f19c710b2fb", "coffee filter-98e44309a4afddda", "new coffee filter cone-b101b208b921ce07", "red spoon-591d984a7e12315e", "frozen dumplings-6afa13ccb0f4fb62", "second set of frozen dumplings-25135ae7dded01a8", "last frozen dumpling-d74a8290a9239a04", "empty plastic bag-63a325a5cdbccba8", "orange cloth-5ebfaaee70535f6a", "first chopsticks-1e42f51bdc8c76e7", "second chopsticks-9fb14c743e71e22e", "first red chopstick-e4688d3a0dd2b34e", "second red chopstick-5bf110b7efab0714", "other pink butterfly shaped chopstick holder-ab75d44751329343", "blue butterfly shaped chopstick holder-abe050773fd1b530", "red chopsticks-3a4b1d0fbf79b413", "folded white cloth-9eb79751f6a7636e", "top empty plastic bag-8125f8410a873826", "other empty food bag-25b1b286ba900bf6", "dishwasher brush-a28fbab556e8b019", "small sponge-13244022c8cacc49", "washing up liquid-a06b408a43cc43ab", "new small bowl-ebb4474d35cc2b7c", "second bowl in stack of bowls-c270ecc940081bc8", "top bowl in stack of bowls-7b4f34b9a92e4a1f", "stack of small bowls-a160bfdbc189c8c4", "bowl2-1eef02d7d93c4931", "bottom two big bowls-eaed4b7ecc151b96", "top two big bowls-1244e9499e09d47b", "new bottom bowls-5c0ad1805c5cc614", "glass-8ce3e24bf8ab5c6c", "new bottom bowl-b9e128f5249ea986", "new bottom bowl 2-4c5a23d029c68fc1", "first big bowl-e3bdac110de97f87", "plastic container-8dfb8583a0fea3b6", "box of grapes-d58b9f1d65ebe5b2", "plastic sleeve-3e3ccd99ab3c7c95", "grapes-83d33d9598b3cc68", "new track 6-5bcf3facf279db57", "small bunch of grapes-4db9de2cf57b75a4", "bunch of grapes-39380d796a05df50", "last bunch of grapes-612ff9ffd3a0b645", "plastic box-fc168a7d19f5f915", "small plastic colander-a0ac641b94e24899", "colander-156217f7f4400618", "new sieve-573cb366ca5a60dc", "plate-d707b77979daed6b", "new small bottom bowl-c13da7e8da35f780", "small stack of bowls-d1891513ba1b5022", "final bowl-5b35fac800bf9fa5", "bowl of grapes-3f45f17ad1296808" ], "reasoning": "1. I watched for the user to begin using the vacuum (hoover-fa42a926c802af23), which occurs at 0:22.\n2. I then watched to see the water filter (water filter tank-002884c4d601ff0e) which is on the counter above the small refrigerator, and first seen at 0:25.\n3. I waited for the user to go an vacuum below the water filter, which occurs at 0:57.\n4. I looked to see what was underneath the water filter (water filter tank-002884c4d601ff0e) that the vacuum went over (hoover-fa42a926c802af23) and it is a black doormat .", "referenced_object_masks": [ "hoover-fa42a926c802af23", "water filter tank-002884c4d601ff0e" ] }, { "key": "P03-20240218-140054:767487b6d8bd5e3a523af5b11c15f4ebfd00604f", "video_id": "P03-20240218-140054", "question": "How many small bowls are left in the upper cabinet to the right of the sink, after I take one out?", "answer_choice_0": "2", "answer_choice_1": "4", "answer_choice_2": "1", "answer_choice_3": "5", "answer_choice_4": "3", "answer_id": 4, "answer": "3", "question_type": [ "Action Recognition", "Cause and Effect", "Object Recognition", "Counting" ], "objects_of_interest": [ "container-3db98c624e413b2d", "hoover-fa42a926c802af23", "hoover charger-a0ed1d356c17b389", "chair-2fefa7fb254fe497", "second chair-11232e9af4b48fc0", "pack of taiwanese pancakes-ee302825e73eb626", "second saucepan-ffe93f5370033200", "new track 1-11a097c1ffc7cd41", "third saucepan-a3508894581f53d4", "larger saucepan-d632760b19d7e71d", "new track 3-5cafbfb12a904c77", "container box-23ea8486f458f6ac", "dirty mug-dbbc1376dee81466", "rice cooker bowl-02ad871e2228da09", "rice cooker tray-4dd2f4ce6ec011bf", "right placemat-5f3795bbcbf3af68", "left placemat-22e05f03d8cfa22f", "water filter tank-002884c4d601ff0e", "laundry detergent cap-b8cdcbc511ace0b1", "plastic bag-a7907e8b1b98e5b2", "box-7292668b779034a7", "scissors-64d7d9c0d7b996fb", "safety cover of scissors-989a23c52aedebd5", "dry wooden spoon-b3f394aa2d811358", "small bowl-141df35b7a42703a", "other food bag-3f630662c6d6df73", "bottle of sesame oil-933272b80328c99f", "another smaller bottle-cecf7711b49e5304", "another bottle-27d3b981e862da6c", "bottle of soy sauce-f9c400376933e0f2", "small cup-9c3bcbf40b2d6e87", "knife-37f2303d21d59017", "garlic cloves-a6635f799069d2db", "cloth-ab363aebeb8a4521", "coffee filter stand-87876f19c710b2fb", "coffee filter-98e44309a4afddda", "new coffee filter cone-b101b208b921ce07", "red spoon-591d984a7e12315e", "frozen dumplings-6afa13ccb0f4fb62", "second set of frozen dumplings-25135ae7dded01a8", "last frozen dumpling-d74a8290a9239a04", "empty plastic bag-63a325a5cdbccba8", "orange cloth-5ebfaaee70535f6a", "first chopsticks-1e42f51bdc8c76e7", "second chopsticks-9fb14c743e71e22e", "first red chopstick-e4688d3a0dd2b34e", "second red chopstick-5bf110b7efab0714", "other pink butterfly shaped chopstick holder-ab75d44751329343", "blue butterfly shaped chopstick holder-abe050773fd1b530", "red chopsticks-3a4b1d0fbf79b413", "folded white cloth-9eb79751f6a7636e", "top empty plastic bag-8125f8410a873826", "other empty food bag-25b1b286ba900bf6", "dishwasher brush-a28fbab556e8b019", "small sponge-13244022c8cacc49", "washing up liquid-a06b408a43cc43ab", "new small bowl-ebb4474d35cc2b7c", "second bowl in stack of bowls-c270ecc940081bc8", "top bowl in stack of bowls-7b4f34b9a92e4a1f", "stack of small bowls-a160bfdbc189c8c4", "bowl2-1eef02d7d93c4931", "bottom two big bowls-eaed4b7ecc151b96", "top two big bowls-1244e9499e09d47b", "new bottom bowls-5c0ad1805c5cc614", "glass-8ce3e24bf8ab5c6c", "new bottom bowl-b9e128f5249ea986", "new bottom bowl 2-4c5a23d029c68fc1", "first big bowl-e3bdac110de97f87", "plastic container-8dfb8583a0fea3b6", "box of grapes-d58b9f1d65ebe5b2", "plastic sleeve-3e3ccd99ab3c7c95", "grapes-83d33d9598b3cc68", "new track 6-5bcf3facf279db57", "small bunch of grapes-4db9de2cf57b75a4", "bunch of grapes-39380d796a05df50", "last bunch of grapes-612ff9ffd3a0b645", "plastic box-fc168a7d19f5f915", "small plastic colander-a0ac641b94e24899", "colander-156217f7f4400618", "new sieve-573cb366ca5a60dc", "plate-d707b77979daed6b", "new small bottom bowl-c13da7e8da35f780", "small stack of bowls-d1891513ba1b5022", "final bowl-5b35fac800bf9fa5", "bowl of grapes-3f45f17ad1296808" ], "reasoning": "1. I watched the video to see when the user reaches for a bowl (small bowl-141df35b7a42703a) in the upper cabinet to the right of the sink , and saw this happen at 06:55.\n2. As the user reaches for a bowl (small bowl-141df35b7a42703a) from the cabinet at 06:56, they grab one from the top of the stack of small bowls.\n3. As they grab a small bowl, the number in the stack reduces from four, to three at 06:57.", "referenced_object_masks": [ "small bowl-141df35b7a42703a" ] }, { "key": "P03-20240218-140054:8ca1c9907f94ee6baa90b2ba93a617ae0f2e86ab", "video_id": "P03-20240218-140054", "question": "Where do I begin to place the rice cooker tray, but change my mind?", "answer_choice_0": "On the drying rack", "answer_choice_1": "On the counter", "answer_choice_2": "In the rice cooker", "answer_choice_3": "Dish rack", "answer_choice_4": "Bottom shelf of a cabinet", "answer_id": 4, "answer": "Bottom shelf of a cabinet", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "container-3db98c624e413b2d", "hoover-fa42a926c802af23", "hoover charger-a0ed1d356c17b389", "chair-2fefa7fb254fe497", "second chair-11232e9af4b48fc0", "pack of taiwanese pancakes-ee302825e73eb626", "second saucepan-ffe93f5370033200", "new track 1-11a097c1ffc7cd41", "third saucepan-a3508894581f53d4", "larger saucepan-d632760b19d7e71d", "new track 3-5cafbfb12a904c77", "container box-23ea8486f458f6ac", "dirty mug-dbbc1376dee81466", "rice cooker bowl-02ad871e2228da09", "rice cooker tray-4dd2f4ce6ec011bf", "right placemat-5f3795bbcbf3af68", "left placemat-22e05f03d8cfa22f", "water filter tank-002884c4d601ff0e", "laundry detergent cap-b8cdcbc511ace0b1", "plastic bag-a7907e8b1b98e5b2", "box-7292668b779034a7", "scissors-64d7d9c0d7b996fb", "safety cover of scissors-989a23c52aedebd5", "dry wooden spoon-b3f394aa2d811358", "small bowl-141df35b7a42703a", "other food bag-3f630662c6d6df73", "bottle of sesame oil-933272b80328c99f", "another smaller bottle-cecf7711b49e5304", "another bottle-27d3b981e862da6c", "bottle of soy sauce-f9c400376933e0f2", "small cup-9c3bcbf40b2d6e87", "knife-37f2303d21d59017", "garlic cloves-a6635f799069d2db", "cloth-ab363aebeb8a4521", "coffee filter stand-87876f19c710b2fb", "coffee filter-98e44309a4afddda", "new coffee filter cone-b101b208b921ce07", "red spoon-591d984a7e12315e", "frozen dumplings-6afa13ccb0f4fb62", "second set of frozen dumplings-25135ae7dded01a8", "last frozen dumpling-d74a8290a9239a04", "empty plastic bag-63a325a5cdbccba8", "orange cloth-5ebfaaee70535f6a", "first chopsticks-1e42f51bdc8c76e7", "second chopsticks-9fb14c743e71e22e", "first red chopstick-e4688d3a0dd2b34e", "second red chopstick-5bf110b7efab0714", "other pink butterfly shaped chopstick holder-ab75d44751329343", "blue butterfly shaped chopstick holder-abe050773fd1b530", "red chopsticks-3a4b1d0fbf79b413", "folded white cloth-9eb79751f6a7636e", "top empty plastic bag-8125f8410a873826", "other empty food bag-25b1b286ba900bf6", "dishwasher brush-a28fbab556e8b019", "small sponge-13244022c8cacc49", "washing up liquid-a06b408a43cc43ab", "new small bowl-ebb4474d35cc2b7c", "second bowl in stack of bowls-c270ecc940081bc8", "top bowl in stack of bowls-7b4f34b9a92e4a1f", "stack of small bowls-a160bfdbc189c8c4", "bowl2-1eef02d7d93c4931", "bottom two big bowls-eaed4b7ecc151b96", "top two big bowls-1244e9499e09d47b", "new bottom bowls-5c0ad1805c5cc614", "glass-8ce3e24bf8ab5c6c", "new bottom bowl-b9e128f5249ea986", "new bottom bowl 2-4c5a23d029c68fc1", "first big bowl-e3bdac110de97f87", "plastic container-8dfb8583a0fea3b6", "box of grapes-d58b9f1d65ebe5b2", "plastic sleeve-3e3ccd99ab3c7c95", "grapes-83d33d9598b3cc68", "new track 6-5bcf3facf279db57", "small bunch of grapes-4db9de2cf57b75a4", "bunch of grapes-39380d796a05df50", "last bunch of grapes-612ff9ffd3a0b645", "plastic box-fc168a7d19f5f915", "small plastic colander-a0ac641b94e24899", "colander-156217f7f4400618", "new sieve-573cb366ca5a60dc", "plate-d707b77979daed6b", "new small bottom bowl-c13da7e8da35f780", "small stack of bowls-d1891513ba1b5022", "final bowl-5b35fac800bf9fa5", "bowl of grapes-3f45f17ad1296808" ], "reasoning": "1. I identified the rice cooker tray (rice cooker tray-4dd2f4ce6ec011bf) in its initial position on the dish drainer at 04:38.\n2. After the user picked up the rice cooker tray (rice cooker tray-4dd2f4ce6ec011bf) at 04:40, they moved toward a lower cabinet, and then bent down to open the door.\n3. The user moved their hand to place the rice cooker tray (rice cooker tray-4dd2f4ce6ec011bf) on the lower shelf at 04:42, before apparently changing their mind.\n4. The user's hand then quickly raises the rice cooker tray (rice cooker tray-4dd2f4ce6ec011bf) to the top shelf, placing it there at 04:44.", "referenced_object_masks": [ "rice cooker tray-4dd2f4ce6ec011bf" ] }, { "key": "P03-20240218-140054:9247893ff61b1538e68957b69f9a309ad59e4cde", "video_id": "P03-20240218-140054", "question": "In the two minutes after I handle the scissors, which of the following items do I not touch?", "answer_choice_0": "Garlic clove", "answer_choice_1": "Ceramic bowl", "answer_choice_2": "Dumpling bag", "answer_choice_3": "Soy sauce bottle", "answer_choice_4": "Saucepan handle", "answer_id": 4, "answer": "Saucepan handle", "question_type": [ "Action Recognition", "Object Recognition", "Temporal Reasoning", "Event Occurence" ], "objects_of_interest": [ "container-3db98c624e413b2d", "hoover-fa42a926c802af23", "hoover charger-a0ed1d356c17b389", "chair-2fefa7fb254fe497", "second chair-11232e9af4b48fc0", "pack of taiwanese pancakes-ee302825e73eb626", "second saucepan-ffe93f5370033200", "new track 1-11a097c1ffc7cd41", "third saucepan-a3508894581f53d4", "larger saucepan-d632760b19d7e71d", "new track 3-5cafbfb12a904c77", "container box-23ea8486f458f6ac", "dirty mug-dbbc1376dee81466", "rice cooker bowl-02ad871e2228da09", "rice cooker tray-4dd2f4ce6ec011bf", "right placemat-5f3795bbcbf3af68", "left placemat-22e05f03d8cfa22f", "water filter tank-002884c4d601ff0e", "laundry detergent cap-b8cdcbc511ace0b1", "plastic bag-a7907e8b1b98e5b2", "box-7292668b779034a7", "scissors-64d7d9c0d7b996fb", "safety cover of scissors-989a23c52aedebd5", "dry wooden spoon-b3f394aa2d811358", "small bowl-141df35b7a42703a", "other food bag-3f630662c6d6df73", "bottle of sesame oil-933272b80328c99f", "another smaller bottle-cecf7711b49e5304", "another bottle-27d3b981e862da6c", "bottle of soy sauce-f9c400376933e0f2", "small cup-9c3bcbf40b2d6e87", "knife-37f2303d21d59017", "garlic cloves-a6635f799069d2db", "cloth-ab363aebeb8a4521", "coffee filter stand-87876f19c710b2fb", "coffee filter-98e44309a4afddda", "new coffee filter cone-b101b208b921ce07", "red spoon-591d984a7e12315e", "frozen dumplings-6afa13ccb0f4fb62", "second set of frozen dumplings-25135ae7dded01a8", "last frozen dumpling-d74a8290a9239a04", "empty plastic bag-63a325a5cdbccba8", "orange cloth-5ebfaaee70535f6a", "first chopsticks-1e42f51bdc8c76e7", "second chopsticks-9fb14c743e71e22e", "first red chopstick-e4688d3a0dd2b34e", "second red chopstick-5bf110b7efab0714", "other pink butterfly shaped chopstick holder-ab75d44751329343", "blue butterfly shaped chopstick holder-abe050773fd1b530", "red chopsticks-3a4b1d0fbf79b413", "folded white cloth-9eb79751f6a7636e", "top empty plastic bag-8125f8410a873826", "other empty food bag-25b1b286ba900bf6", "dishwasher brush-a28fbab556e8b019", "small sponge-13244022c8cacc49", "washing up liquid-a06b408a43cc43ab", "new small bowl-ebb4474d35cc2b7c", "second bowl in stack of bowls-c270ecc940081bc8", "top bowl in stack of bowls-7b4f34b9a92e4a1f", "stack of small bowls-a160bfdbc189c8c4", "bowl2-1eef02d7d93c4931", "bottom two big bowls-eaed4b7ecc151b96", "top two big bowls-1244e9499e09d47b", "new bottom bowls-5c0ad1805c5cc614", "glass-8ce3e24bf8ab5c6c", "new bottom bowl-b9e128f5249ea986", "new bottom bowl 2-4c5a23d029c68fc1", "first big bowl-e3bdac110de97f87", "plastic container-8dfb8583a0fea3b6", "box of grapes-d58b9f1d65ebe5b2", "plastic sleeve-3e3ccd99ab3c7c95", "grapes-83d33d9598b3cc68", "new track 6-5bcf3facf279db57", "small bunch of grapes-4db9de2cf57b75a4", "bunch of grapes-39380d796a05df50", "last bunch of grapes-612ff9ffd3a0b645", "plastic box-fc168a7d19f5f915", "small plastic colander-a0ac641b94e24899", "colander-156217f7f4400618", "new sieve-573cb366ca5a60dc", "plate-d707b77979daed6b", "new small bottom bowl-c13da7e8da35f780", "small stack of bowls-d1891513ba1b5022", "final bowl-5b35fac800bf9fa5", "bowl of grapes-3f45f17ad1296808" ], "reasoning": "1. I watch the video to determine when the user handles the scissors (scissors-64d7d9c0d7b996fb) - this happens from 06:45-06:47. Since the question asks about the following two minutes, I determine that I should watch the video from 06:47-08:47, observing what the user touches as it relates to the answer candidates.\n2. From 06:56-06:59, I observe the user take a ceramic bowl (small bowl-141df35b7a42703a) out of the cabinet, eliminating D as a potential answer.\n3. From 07:06-07:13, I observe the user open both dumpling bags (frozen dumplings-6afa13ccb0f4fb62, second set of frozen dumplings-25135ae7dded01a8), eliminating A as a potential answer. \n4. From 07:44-07:47, I observe the user take a bottle of soy sauce (bottle of soy sauce-f9c400376933e0f2) out of the bottom right drawer next to the stove and place it on the countertop, eliminating C as a potential answer.\n5. From 08:18-08:27, I observe the user take a garlic clove (garlic cloves-a6635f799069d2db) out of a ceramic cup (small cup-9c3bcbf40b2d6e87) and place it on the countertop, eliminating E as a potential answer choice.\n6. With only B left as a viable option, I watch the rest of the span, and then the entire span again, to make sure that the user does not touch the saucepan (larger saucepan-d632760b19d7e71d). When I observe that the user does not touch the saucepan (larger saucepan-d632760b19d7e71d) during the span, I determine that answer choice B is correct.", "referenced_object_masks": [ "scissors-64d7d9c0d7b996fb", "small bowl-141df35b7a42703a", "small cup-9c3bcbf40b2d6e87", "bottle of soy sauce-f9c400376933e0f2", "larger saucepan-d632760b19d7e71d", "frozen dumplings-6afa13ccb0f4fb62", "second set of frozen dumplings-25135ae7dded01a8", "garlic cloves-a6635f799069d2db" ] }, { "key": "P03-20240218-140054:971656ee20fe359440078153426d4b915e926b09", "video_id": "P03-20240218-140054", "question": "What is the product of the number of items I remove from the dish drying rack over the course of the video, and the number of dishes with food I place on the table over the course of the video?", "answer_choice_0": "15", "answer_choice_1": "25", "answer_choice_2": "12", "answer_choice_3": "16", "answer_choice_4": "20", "answer_id": 0, "answer": "15", "question_type": [ "Action Recognition", "Counting", "Numerical Reasoning", "Object Recognition" ], "objects_of_interest": [ "container-3db98c624e413b2d", "hoover-fa42a926c802af23", "hoover charger-a0ed1d356c17b389", "chair-2fefa7fb254fe497", "second chair-11232e9af4b48fc0", "pack of taiwanese pancakes-ee302825e73eb626", "second saucepan-ffe93f5370033200", "new track 1-11a097c1ffc7cd41", "third saucepan-a3508894581f53d4", "larger saucepan-d632760b19d7e71d", "new track 3-5cafbfb12a904c77", "container box-23ea8486f458f6ac", "dirty mug-dbbc1376dee81466", "rice cooker bowl-02ad871e2228da09", "rice cooker tray-4dd2f4ce6ec011bf", "right placemat-5f3795bbcbf3af68", "left placemat-22e05f03d8cfa22f", "water filter tank-002884c4d601ff0e", "laundry detergent cap-b8cdcbc511ace0b1", "plastic bag-a7907e8b1b98e5b2", "box-7292668b779034a7", "scissors-64d7d9c0d7b996fb", "safety cover of scissors-989a23c52aedebd5", "dry wooden spoon-b3f394aa2d811358", "small bowl-141df35b7a42703a", "other food bag-3f630662c6d6df73", "bottle of sesame oil-933272b80328c99f", "another smaller bottle-cecf7711b49e5304", "another bottle-27d3b981e862da6c", "bottle of soy sauce-f9c400376933e0f2", "small cup-9c3bcbf40b2d6e87", "knife-37f2303d21d59017", "garlic cloves-a6635f799069d2db", "cloth-ab363aebeb8a4521", "coffee filter stand-87876f19c710b2fb", "coffee filter-98e44309a4afddda", "new coffee filter cone-b101b208b921ce07", "red spoon-591d984a7e12315e", "frozen dumplings-6afa13ccb0f4fb62", "second set of frozen dumplings-25135ae7dded01a8", "last frozen dumpling-d74a8290a9239a04", "empty plastic bag-63a325a5cdbccba8", "orange cloth-5ebfaaee70535f6a", "first chopsticks-1e42f51bdc8c76e7", "second chopsticks-9fb14c743e71e22e", "first red chopstick-e4688d3a0dd2b34e", "second red chopstick-5bf110b7efab0714", "other pink butterfly shaped chopstick holder-ab75d44751329343", "blue butterfly shaped chopstick holder-abe050773fd1b530", "red chopsticks-3a4b1d0fbf79b413", "folded white cloth-9eb79751f6a7636e", "top empty plastic bag-8125f8410a873826", "other empty food bag-25b1b286ba900bf6", "dishwasher brush-a28fbab556e8b019", "small sponge-13244022c8cacc49", "washing up liquid-a06b408a43cc43ab", "new small bowl-ebb4474d35cc2b7c", "second bowl in stack of bowls-c270ecc940081bc8", "top bowl in stack of bowls-7b4f34b9a92e4a1f", "stack of small bowls-a160bfdbc189c8c4", "bowl2-1eef02d7d93c4931", "bottom two big bowls-eaed4b7ecc151b96", "top two big bowls-1244e9499e09d47b", "new bottom bowls-5c0ad1805c5cc614", "glass-8ce3e24bf8ab5c6c", "new bottom bowl-b9e128f5249ea986", "new bottom bowl 2-4c5a23d029c68fc1", "first big bowl-e3bdac110de97f87", "plastic container-8dfb8583a0fea3b6", "box of grapes-d58b9f1d65ebe5b2", "plastic sleeve-3e3ccd99ab3c7c95", "grapes-83d33d9598b3cc68", "new track 6-5bcf3facf279db57", "small bunch of grapes-4db9de2cf57b75a4", "bunch of grapes-39380d796a05df50", "last bunch of grapes-612ff9ffd3a0b645", "plastic box-fc168a7d19f5f915", "small plastic colander-a0ac641b94e24899", "colander-156217f7f4400618", "new sieve-573cb366ca5a60dc", "plate-d707b77979daed6b", "new small bottom bowl-c13da7e8da35f780", "small stack of bowls-d1891513ba1b5022", "final bowl-5b35fac800bf9fa5", "bowl of grapes-3f45f17ad1296808" ], "reasoning": "1. I first watch the video to identify items removed from the dish drying rack over the right sink basin. The first item I see removed from the dish drying rack is a round rice cooker tray (rice cooker tray-4dd2f4ce6ec011bf) at 04:39.\n2. I continue watching and observe that the second item removed from the dish drying rack is a coffee filter stand (coffee filter stand-87876f19c710b2fb) at 10:01.\n3. Next, I observe that the third item removed from the dish drying rack is a coffee metal filter stand (coffee filter-98e44309a4afddda) at 10:03.\n4. I then observe that the fourth item removed is a plastic coffee filter cone (new coffee filter cone-b101b208b921ce07) at 10:03.\n5. Next, I observe that the fifth item removed from the dish drying rack is a red plastic measuring cup (red spoon-591d984a7e12315e) at 10:08.\n6. I do not observe any more items being removed from the dish drying rack over the course of the video, so I determine that the first number of my equation is 5.\n7. Next, I watch to see how many dishes with food the user places on the table over the course of the video - the first instance I see of this is at 11:48-11:50, when the user places the bowl with soy sauce (small bowl-141df35b7a42703a) on the table.\n8. I continue watching and see the second dish with dumplings on it (plate-d707b77979daed6b) placed on the table from 18:21-18:23.\n9. I continue watching and see the third and final dish containing food, a bowl of grapes (bowl of grapes-3f45f17ad1296808), placed on the table at 18:44. From this, I determine my second number is three.\n10. I multiply the number of items removed from the dish drying rack, 5, by the number of dishes containing food placed on the table, 3, to get a product of 15 - making A the correct answer.", "referenced_object_masks": [ "coffee filter stand-87876f19c710b2fb", "rice cooker tray-4dd2f4ce6ec011bf", "red spoon-591d984a7e12315e", "small bowl-141df35b7a42703a", "bowl of grapes-3f45f17ad1296808", "plate-d707b77979daed6b", "new coffee filter cone-b101b208b921ce07", "coffee filter-98e44309a4afddda" ] }, { "key": "P03-20240218-140054:a658da9cec1605e8111fdc333b5fa085331b6bc0", "video_id": "P03-20240218-140054", "question": "Where was the first freezer drawer I opened in relation to the drawer where I retrieved the Taiwanese pancakes?", "answer_choice_0": "Two below", "answer_choice_1": "One above", "answer_choice_2": "One below", "answer_choice_3": "Two above", "answer_choice_4": "It was the same drawer", "answer_id": 2, "answer": "One below", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Object Recognition" ], "objects_of_interest": [ "container-3db98c624e413b2d", "hoover-fa42a926c802af23", "hoover charger-a0ed1d356c17b389", "chair-2fefa7fb254fe497", "second chair-11232e9af4b48fc0", "pack of taiwanese pancakes-ee302825e73eb626", "second saucepan-ffe93f5370033200", "new track 1-11a097c1ffc7cd41", "third saucepan-a3508894581f53d4", "larger saucepan-d632760b19d7e71d", "new track 3-5cafbfb12a904c77", "container box-23ea8486f458f6ac", "dirty mug-dbbc1376dee81466", "rice cooker bowl-02ad871e2228da09", "rice cooker tray-4dd2f4ce6ec011bf", "right placemat-5f3795bbcbf3af68", "left placemat-22e05f03d8cfa22f", "water filter tank-002884c4d601ff0e", "laundry detergent cap-b8cdcbc511ace0b1", "plastic bag-a7907e8b1b98e5b2", "box-7292668b779034a7", "scissors-64d7d9c0d7b996fb", "safety cover of scissors-989a23c52aedebd5", "dry wooden spoon-b3f394aa2d811358", "small bowl-141df35b7a42703a", "other food bag-3f630662c6d6df73", "bottle of sesame oil-933272b80328c99f", "another smaller bottle-cecf7711b49e5304", "another bottle-27d3b981e862da6c", "bottle of soy sauce-f9c400376933e0f2", "small cup-9c3bcbf40b2d6e87", "knife-37f2303d21d59017", "garlic cloves-a6635f799069d2db", "cloth-ab363aebeb8a4521", "coffee filter stand-87876f19c710b2fb", "coffee filter-98e44309a4afddda", "new coffee filter cone-b101b208b921ce07", "red spoon-591d984a7e12315e", "frozen dumplings-6afa13ccb0f4fb62", "second set of frozen dumplings-25135ae7dded01a8", "last frozen dumpling-d74a8290a9239a04", "empty plastic bag-63a325a5cdbccba8", "orange cloth-5ebfaaee70535f6a", "first chopsticks-1e42f51bdc8c76e7", "second chopsticks-9fb14c743e71e22e", "first red chopstick-e4688d3a0dd2b34e", "second red chopstick-5bf110b7efab0714", "other pink butterfly shaped chopstick holder-ab75d44751329343", "blue butterfly shaped chopstick holder-abe050773fd1b530", "red chopsticks-3a4b1d0fbf79b413", "folded white cloth-9eb79751f6a7636e", "top empty plastic bag-8125f8410a873826", "other empty food bag-25b1b286ba900bf6", "dishwasher brush-a28fbab556e8b019", "small sponge-13244022c8cacc49", "washing up liquid-a06b408a43cc43ab", "new small bowl-ebb4474d35cc2b7c", "second bowl in stack of bowls-c270ecc940081bc8", "top bowl in stack of bowls-7b4f34b9a92e4a1f", "stack of small bowls-a160bfdbc189c8c4", "bowl2-1eef02d7d93c4931", "bottom two big bowls-eaed4b7ecc151b96", "top two big bowls-1244e9499e09d47b", "new bottom bowls-5c0ad1805c5cc614", "glass-8ce3e24bf8ab5c6c", "new bottom bowl-b9e128f5249ea986", "new bottom bowl 2-4c5a23d029c68fc1", "first big bowl-e3bdac110de97f87", "plastic container-8dfb8583a0fea3b6", "box of grapes-d58b9f1d65ebe5b2", "plastic sleeve-3e3ccd99ab3c7c95", "grapes-83d33d9598b3cc68", "new track 6-5bcf3facf279db57", "small bunch of grapes-4db9de2cf57b75a4", "bunch of grapes-39380d796a05df50", "last bunch of grapes-612ff9ffd3a0b645", "plastic box-fc168a7d19f5f915", "small plastic colander-a0ac641b94e24899", "colander-156217f7f4400618", "new sieve-573cb366ca5a60dc", "plate-d707b77979daed6b", "new small bottom bowl-c13da7e8da35f780", "small stack of bowls-d1891513ba1b5022", "final bowl-5b35fac800bf9fa5", "bowl of grapes-3f45f17ad1296808" ], "reasoning": "1. I watched for the user to open a freezer, which occurs at 2:33.\n2. After opening the freezer door, I waited for the user to open a drawer inside the freezer, which happens for the first time at 2:36.\n3. The user rummages inside but does not remove anything before closing the drawer again.\n4. The user opens a second drawer at 2:43, removing a packet of Taiwanese pancakes (pack of taiwanese pancakes-ee302825e73eb626) at 2:44.\n5. The drawer from which the pancakes (pack of taiwanese pancakes-ee302825e73eb626) is one drawer above the first drawer opened, making the first drawer one below the drawer with the pancakes (pack of taiwanese pancakes-ee302825e73eb626).", "referenced_object_masks": [ "pack of taiwanese pancakes-ee302825e73eb626" ] }, { "key": "P03-20240218-140054:a6c469f2cff0cb173c30e73c63c5722ca635d1b7", "video_id": "P03-20240218-140054", "question": "While I am filling the larger saucepan with water, how many coffee cups would be in the sink if there were two more present?", "answer_choice_0": "6", "answer_choice_1": "2", "answer_choice_2": "5", "answer_choice_3": "3", "answer_choice_4": "4", "answer_id": 3, "answer": "3", "question_type": [ "Counting", "Numerical Reasoning", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "container-3db98c624e413b2d", "hoover-fa42a926c802af23", "hoover charger-a0ed1d356c17b389", "chair-2fefa7fb254fe497", "second chair-11232e9af4b48fc0", "pack of taiwanese pancakes-ee302825e73eb626", "second saucepan-ffe93f5370033200", "new track 1-11a097c1ffc7cd41", "third saucepan-a3508894581f53d4", "larger saucepan-d632760b19d7e71d", "new track 3-5cafbfb12a904c77", "container box-23ea8486f458f6ac", "dirty mug-dbbc1376dee81466", "rice cooker bowl-02ad871e2228da09", "rice cooker tray-4dd2f4ce6ec011bf", "right placemat-5f3795bbcbf3af68", "left placemat-22e05f03d8cfa22f", "water filter tank-002884c4d601ff0e", "laundry detergent cap-b8cdcbc511ace0b1", "plastic bag-a7907e8b1b98e5b2", "box-7292668b779034a7", "scissors-64d7d9c0d7b996fb", "safety cover of scissors-989a23c52aedebd5", "dry wooden spoon-b3f394aa2d811358", "small bowl-141df35b7a42703a", "other food bag-3f630662c6d6df73", "bottle of sesame oil-933272b80328c99f", "another smaller bottle-cecf7711b49e5304", "another bottle-27d3b981e862da6c", "bottle of soy sauce-f9c400376933e0f2", "small cup-9c3bcbf40b2d6e87", "knife-37f2303d21d59017", "garlic cloves-a6635f799069d2db", "cloth-ab363aebeb8a4521", "coffee filter stand-87876f19c710b2fb", "coffee filter-98e44309a4afddda", "new coffee filter cone-b101b208b921ce07", "red spoon-591d984a7e12315e", "frozen dumplings-6afa13ccb0f4fb62", "second set of frozen dumplings-25135ae7dded01a8", "last frozen dumpling-d74a8290a9239a04", "empty plastic bag-63a325a5cdbccba8", "orange cloth-5ebfaaee70535f6a", "first chopsticks-1e42f51bdc8c76e7", "second chopsticks-9fb14c743e71e22e", "first red chopstick-e4688d3a0dd2b34e", "second red chopstick-5bf110b7efab0714", "other pink butterfly shaped chopstick holder-ab75d44751329343", "blue butterfly shaped chopstick holder-abe050773fd1b530", "red chopsticks-3a4b1d0fbf79b413", "folded white cloth-9eb79751f6a7636e", "top empty plastic bag-8125f8410a873826", "other empty food bag-25b1b286ba900bf6", "dishwasher brush-a28fbab556e8b019", "small sponge-13244022c8cacc49", "washing up liquid-a06b408a43cc43ab", "new small bowl-ebb4474d35cc2b7c", "second bowl in stack of bowls-c270ecc940081bc8", "top bowl in stack of bowls-7b4f34b9a92e4a1f", "stack of small bowls-a160bfdbc189c8c4", "bowl2-1eef02d7d93c4931", "bottom two big bowls-eaed4b7ecc151b96", "top two big bowls-1244e9499e09d47b", "new bottom bowls-5c0ad1805c5cc614", "glass-8ce3e24bf8ab5c6c", "new bottom bowl-b9e128f5249ea986", "new bottom bowl 2-4c5a23d029c68fc1", "first big bowl-e3bdac110de97f87", "plastic container-8dfb8583a0fea3b6", "box of grapes-d58b9f1d65ebe5b2", "plastic sleeve-3e3ccd99ab3c7c95", "grapes-83d33d9598b3cc68", "new track 6-5bcf3facf279db57", "small bunch of grapes-4db9de2cf57b75a4", "bunch of grapes-39380d796a05df50", "last bunch of grapes-612ff9ffd3a0b645", "plastic box-fc168a7d19f5f915", "small plastic colander-a0ac641b94e24899", "colander-156217f7f4400618", "new sieve-573cb366ca5a60dc", "plate-d707b77979daed6b", "new small bottom bowl-c13da7e8da35f780", "small stack of bowls-d1891513ba1b5022", "final bowl-5b35fac800bf9fa5", "bowl of grapes-3f45f17ad1296808" ], "reasoning": "1. I watched for the user to get out the larger saucepan (larger saucepan-d632760b19d7e71d), which is retrieved from a cabinet at 3:13.\n2. I then watched for the user to fill the larger saucepan (larger saucepan-d632760b19d7e71d) with water, which they begin to do at 3:24.\n3. During this time there is 1 coffee cup (dirty mug-dbbc1376dee81466) in the sink. \n4. 2 more coffee cups would make 3.", "referenced_object_masks": [ "larger saucepan-d632760b19d7e71d", "dirty mug-dbbc1376dee81466" ] }, { "key": "P03-20240218-140054:ac920773d1828405f779d5718422bed5169b2b85", "video_id": "P03-20240218-140054", "question": "During the part of the video in which I put away the plastic bag, lay placemats on the table, fill the large water container, put away a wooden spoon, and place a cap on a bottle, in close succession, which of these events would take place chronologically third if the true third and fifth events in the sequence were swapped?", "answer_choice_0": "Putting away a wooden spoon", "answer_choice_1": "Laying placemats on the table", "answer_choice_2": "Putting away the plastic bag", "answer_choice_3": "Filling the large water container", "answer_choice_4": "Placing a cap on a bottle", "answer_id": 0, "answer": "Putting away a wooden spoon", "question_type": [ "Action Recognition", "Counting", "Event Occurence", "Counterfactual", "Object Recognition" ], "objects_of_interest": [ "container-3db98c624e413b2d", "hoover-fa42a926c802af23", "hoover charger-a0ed1d356c17b389", "chair-2fefa7fb254fe497", "second chair-11232e9af4b48fc0", "pack of taiwanese pancakes-ee302825e73eb626", "second saucepan-ffe93f5370033200", "new track 1-11a097c1ffc7cd41", "third saucepan-a3508894581f53d4", "larger saucepan-d632760b19d7e71d", "new track 3-5cafbfb12a904c77", "container box-23ea8486f458f6ac", "dirty mug-dbbc1376dee81466", "rice cooker bowl-02ad871e2228da09", "rice cooker tray-4dd2f4ce6ec011bf", "right placemat-5f3795bbcbf3af68", "left placemat-22e05f03d8cfa22f", "water filter tank-002884c4d601ff0e", "laundry detergent cap-b8cdcbc511ace0b1", "plastic bag-a7907e8b1b98e5b2", "box-7292668b779034a7", "scissors-64d7d9c0d7b996fb", "safety cover of scissors-989a23c52aedebd5", "dry wooden spoon-b3f394aa2d811358", "small bowl-141df35b7a42703a", "other food bag-3f630662c6d6df73", "bottle of sesame oil-933272b80328c99f", "another smaller bottle-cecf7711b49e5304", "another bottle-27d3b981e862da6c", "bottle of soy sauce-f9c400376933e0f2", "small cup-9c3bcbf40b2d6e87", "knife-37f2303d21d59017", "garlic cloves-a6635f799069d2db", "cloth-ab363aebeb8a4521", "coffee filter stand-87876f19c710b2fb", "coffee filter-98e44309a4afddda", "new coffee filter cone-b101b208b921ce07", "red spoon-591d984a7e12315e", "frozen dumplings-6afa13ccb0f4fb62", "second set of frozen dumplings-25135ae7dded01a8", "last frozen dumpling-d74a8290a9239a04", "empty plastic bag-63a325a5cdbccba8", "orange cloth-5ebfaaee70535f6a", "first chopsticks-1e42f51bdc8c76e7", "second chopsticks-9fb14c743e71e22e", "first red chopstick-e4688d3a0dd2b34e", "second red chopstick-5bf110b7efab0714", "other pink butterfly shaped chopstick holder-ab75d44751329343", "blue butterfly shaped chopstick holder-abe050773fd1b530", "red chopsticks-3a4b1d0fbf79b413", "folded white cloth-9eb79751f6a7636e", "top empty plastic bag-8125f8410a873826", "other empty food bag-25b1b286ba900bf6", "dishwasher brush-a28fbab556e8b019", "small sponge-13244022c8cacc49", "washing up liquid-a06b408a43cc43ab", "new small bowl-ebb4474d35cc2b7c", "second bowl in stack of bowls-c270ecc940081bc8", "top bowl in stack of bowls-7b4f34b9a92e4a1f", "stack of small bowls-a160bfdbc189c8c4", "bowl2-1eef02d7d93c4931", "bottom two big bowls-eaed4b7ecc151b96", "top two big bowls-1244e9499e09d47b", "new bottom bowls-5c0ad1805c5cc614", "glass-8ce3e24bf8ab5c6c", "new bottom bowl-b9e128f5249ea986", "new bottom bowl 2-4c5a23d029c68fc1", "first big bowl-e3bdac110de97f87", "plastic container-8dfb8583a0fea3b6", "box of grapes-d58b9f1d65ebe5b2", "plastic sleeve-3e3ccd99ab3c7c95", "grapes-83d33d9598b3cc68", "new track 6-5bcf3facf279db57", "small bunch of grapes-4db9de2cf57b75a4", "bunch of grapes-39380d796a05df50", "last bunch of grapes-612ff9ffd3a0b645", "plastic box-fc168a7d19f5f915", "small plastic colander-a0ac641b94e24899", "colander-156217f7f4400618", "new sieve-573cb366ca5a60dc", "plate-d707b77979daed6b", "new small bottom bowl-c13da7e8da35f780", "small stack of bowls-d1891513ba1b5022", "final bowl-5b35fac800bf9fa5", "bowl of grapes-3f45f17ad1296808" ], "reasoning": "1. To find this answer, I watch the video to see which of the events laid out by the question takes place first, chronologically - I observe the first event from 05:26-05:33, when the user takes out two placemats (left placemat-22e05f03d8cfa22f, right placemat-5f3795bbcbf3af68) from the large cabinet and places them on the table.\n2. I continue watching until I observe the second event, which occurs from 05:38-06:15, when the user takes the large water container (water filter tank-002884c4d601ff0e) over to the sink and fills it with water.\n3. I continue watching until I observe the third event, which occurs from 06:20-06:27, when the user grabs a small green plastic cap (laundry detergent cap-b8cdcbc511ace0b1) from the windowsill, walks to the lower sink cabinet, opens that door, and places the cap on a large white bottle under the sink.\n4. I continue watching until I observe the fourth event, which occurs from 06:30-06:41, when the user grabs a small plastic bag filled with evenly-spaced yellow circles (plastic bag-a7907e8b1b98e5b2) and puts it away in a drawer to the right of the stovetop.\n5. I continue watching until I observe the fifth event, which occurs from 06:49-06:52, when the user grabs a wooden spoon (dry wooden spoon-b3f394aa2d811358) from the sink and places it in a drawer to the right of the stovetop.\n6. Having observed all the events in order, I then counterfactually imagine that the third event - the cap (laundry detergent cap-b8cdcbc511ace0b1) being placed on the bottle , and the fifth event - the wooden spoon (dry wooden spoon-b3f394aa2d811358) being put away, are swapped in the order in which they occurred. Therefore, the new third chronological event would be the wooden spoon (dry wooden spoon-b3f394aa2d811358) being put away, making the answer C.", "referenced_object_masks": [ "dry wooden spoon-b3f394aa2d811358", "water filter tank-002884c4d601ff0e", "plastic bag-a7907e8b1b98e5b2", "left placemat-22e05f03d8cfa22f", "right placemat-5f3795bbcbf3af68", "laundry detergent cap-b8cdcbc511ace0b1" ] }, { "key": "P03-20240218-140054:aff1a662c414752cbe0d3d3d2a364d21076b6223", "video_id": "P03-20240218-140054", "question": "If I swapped the order of the second and the fourth items I loaded into the dishwasher, where would the new fourth item be located in relation to the largest white plate already inside the dishwasher?", "answer_choice_0": "Above and to the right", "answer_choice_1": "Above and to the left", "answer_choice_2": "Above and directly in front of", "answer_choice_3": "Level with and to the left", "answer_choice_4": "Level with and behind", "answer_id": 1, "answer": "Above and to the left", "question_type": [ "Egocentric & Spatial Perception", "Action Recognition", "Counting", "Object Recognition", "Counterfactual" ], "objects_of_interest": [ "container-3db98c624e413b2d", "hoover-fa42a926c802af23", "hoover charger-a0ed1d356c17b389", "chair-2fefa7fb254fe497", "second chair-11232e9af4b48fc0", "pack of taiwanese pancakes-ee302825e73eb626", "second saucepan-ffe93f5370033200", "new track 1-11a097c1ffc7cd41", "third saucepan-a3508894581f53d4", "larger saucepan-d632760b19d7e71d", "new track 3-5cafbfb12a904c77", "container box-23ea8486f458f6ac", "dirty mug-dbbc1376dee81466", "rice cooker bowl-02ad871e2228da09", "rice cooker tray-4dd2f4ce6ec011bf", "right placemat-5f3795bbcbf3af68", "left placemat-22e05f03d8cfa22f", "water filter tank-002884c4d601ff0e", "laundry detergent cap-b8cdcbc511ace0b1", "plastic bag-a7907e8b1b98e5b2", "box-7292668b779034a7", "scissors-64d7d9c0d7b996fb", "safety cover of scissors-989a23c52aedebd5", "dry wooden spoon-b3f394aa2d811358", "small bowl-141df35b7a42703a", "other food bag-3f630662c6d6df73", "bottle of sesame oil-933272b80328c99f", "another smaller bottle-cecf7711b49e5304", "another bottle-27d3b981e862da6c", "bottle of soy sauce-f9c400376933e0f2", "small cup-9c3bcbf40b2d6e87", "knife-37f2303d21d59017", "garlic cloves-a6635f799069d2db", "cloth-ab363aebeb8a4521", "coffee filter stand-87876f19c710b2fb", "coffee filter-98e44309a4afddda", "new coffee filter cone-b101b208b921ce07", "red spoon-591d984a7e12315e", "frozen dumplings-6afa13ccb0f4fb62", "second set of frozen dumplings-25135ae7dded01a8", "last frozen dumpling-d74a8290a9239a04", "empty plastic bag-63a325a5cdbccba8", "orange cloth-5ebfaaee70535f6a", "first chopsticks-1e42f51bdc8c76e7", "second chopsticks-9fb14c743e71e22e", "first red chopstick-e4688d3a0dd2b34e", "second red chopstick-5bf110b7efab0714", "other pink butterfly shaped chopstick holder-ab75d44751329343", "blue butterfly shaped chopstick holder-abe050773fd1b530", "red chopsticks-3a4b1d0fbf79b413", "folded white cloth-9eb79751f6a7636e", "top empty plastic bag-8125f8410a873826", "other empty food bag-25b1b286ba900bf6", "dishwasher brush-a28fbab556e8b019", "small sponge-13244022c8cacc49", "washing up liquid-a06b408a43cc43ab", "new small bowl-ebb4474d35cc2b7c", "second bowl in stack of bowls-c270ecc940081bc8", "top bowl in stack of bowls-7b4f34b9a92e4a1f", "stack of small bowls-a160bfdbc189c8c4", "bowl2-1eef02d7d93c4931", "bottom two big bowls-eaed4b7ecc151b96", "top two big bowls-1244e9499e09d47b", "new bottom bowls-5c0ad1805c5cc614", "glass-8ce3e24bf8ab5c6c", "new bottom bowl-b9e128f5249ea986", "new bottom bowl 2-4c5a23d029c68fc1", "first big bowl-e3bdac110de97f87", "plastic container-8dfb8583a0fea3b6", "box of grapes-d58b9f1d65ebe5b2", "plastic sleeve-3e3ccd99ab3c7c95", "grapes-83d33d9598b3cc68", "new track 6-5bcf3facf279db57", "small bunch of grapes-4db9de2cf57b75a4", "bunch of grapes-39380d796a05df50", "last bunch of grapes-612ff9ffd3a0b645", "plastic box-fc168a7d19f5f915", "small plastic colander-a0ac641b94e24899", "colander-156217f7f4400618", "new sieve-573cb366ca5a60dc", "plate-d707b77979daed6b", "new small bottom bowl-c13da7e8da35f780", "small stack of bowls-d1891513ba1b5022", "final bowl-5b35fac800bf9fa5", "bowl of grapes-3f45f17ad1296808" ], "reasoning": "1. To determine this, I begin counting the items loaded into the dishwasher, to reach the fourth. First, I observe a tupperware container lid being loaded into the back left corner of the upper dishwasher drawer from 03:58-04:01.\n2. Second, I observe the tupperware container (container box-23ea8486f458f6ac) being loaded into the back left corner of the upper dishwasher drawer, in front of the lid , from 04:05-04:07.\n3. Third, I observe a pale blue ceramic mug (dirty mug-dbbc1376dee81466) being loaded into the middle of the right edge of the upper dishwasher drawer from 04:07-04:12.\n4. Fourth, I observe a large round rice cooker bowl (rice cooker bowl-02ad871e2228da09) being loaded into the back left corner of the lower level of the dishwasher from 04:32-04:35.\n5. Next, I counterfactually assume the orders in which I loaded the second item, the tupperware container (container box-23ea8486f458f6ac), and the fourth item, the rice cooker bowl (rice cooker bowl-02ad871e2228da09), are swapped, so that the rice cooker is now second and the container fourth in the order of loading.\n6. Next, I must determine the new fourth item's location in relation to the white plate - the container (container box-23ea8486f458f6ac) is loaded into the back upper left corner of the dishwasher, and the largest white plate is loaded into the back bottom right corner of the dishwasher. Therefore, the container (container box-23ea8486f458f6ac) can be said to be above and to the left of the white plate , matching answer choice D.", "referenced_object_masks": [ "container box-23ea8486f458f6ac", "rice cooker bowl-02ad871e2228da09", "dirty mug-dbbc1376dee81466" ] }, { "key": "P03-20240218-140054:c164dbc6d96b2c4f5adf2e5d2eee47ddf789bb64", "video_id": "P03-20240218-140054", "question": "What is the quotient of the number of bricks on the top row of the outside wall which are visible from the left window pane while I fill the saucepan with water and the number of places that the vacuum cleaner rests throughtout the video?", "answer_choice_0": "Infinity", "answer_choice_1": "6", "answer_choice_2": "5", "answer_choice_3": "3", "answer_choice_4": "2.5", "answer_id": 3, "answer": "3", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Object Recognition" ], "objects_of_interest": [ "container-3db98c624e413b2d", "hoover-fa42a926c802af23", "hoover charger-a0ed1d356c17b389", "chair-2fefa7fb254fe497", "second chair-11232e9af4b48fc0", "pack of taiwanese pancakes-ee302825e73eb626", "second saucepan-ffe93f5370033200", "new track 1-11a097c1ffc7cd41", "third saucepan-a3508894581f53d4", "larger saucepan-d632760b19d7e71d", "new track 3-5cafbfb12a904c77", "container box-23ea8486f458f6ac", "dirty mug-dbbc1376dee81466", "rice cooker bowl-02ad871e2228da09", "rice cooker tray-4dd2f4ce6ec011bf", "right placemat-5f3795bbcbf3af68", "left placemat-22e05f03d8cfa22f", "water filter tank-002884c4d601ff0e", "laundry detergent cap-b8cdcbc511ace0b1", "plastic bag-a7907e8b1b98e5b2", "box-7292668b779034a7", "scissors-64d7d9c0d7b996fb", "safety cover of scissors-989a23c52aedebd5", "dry wooden spoon-b3f394aa2d811358", "small bowl-141df35b7a42703a", "other food bag-3f630662c6d6df73", "bottle of sesame oil-933272b80328c99f", "another smaller bottle-cecf7711b49e5304", "another bottle-27d3b981e862da6c", "bottle of soy sauce-f9c400376933e0f2", "small cup-9c3bcbf40b2d6e87", "knife-37f2303d21d59017", "garlic cloves-a6635f799069d2db", "cloth-ab363aebeb8a4521", "coffee filter stand-87876f19c710b2fb", "coffee filter-98e44309a4afddda", "new coffee filter cone-b101b208b921ce07", "red spoon-591d984a7e12315e", "frozen dumplings-6afa13ccb0f4fb62", "second set of frozen dumplings-25135ae7dded01a8", "last frozen dumpling-d74a8290a9239a04", "empty plastic bag-63a325a5cdbccba8", "orange cloth-5ebfaaee70535f6a", "first chopsticks-1e42f51bdc8c76e7", "second chopsticks-9fb14c743e71e22e", "first red chopstick-e4688d3a0dd2b34e", "second red chopstick-5bf110b7efab0714", "other pink butterfly shaped chopstick holder-ab75d44751329343", "blue butterfly shaped chopstick holder-abe050773fd1b530", "red chopsticks-3a4b1d0fbf79b413", "folded white cloth-9eb79751f6a7636e", "top empty plastic bag-8125f8410a873826", "other empty food bag-25b1b286ba900bf6", "dishwasher brush-a28fbab556e8b019", "small sponge-13244022c8cacc49", "washing up liquid-a06b408a43cc43ab", "new small bowl-ebb4474d35cc2b7c", "second bowl in stack of bowls-c270ecc940081bc8", "top bowl in stack of bowls-7b4f34b9a92e4a1f", "stack of small bowls-a160bfdbc189c8c4", "bowl2-1eef02d7d93c4931", "bottom two big bowls-eaed4b7ecc151b96", "top two big bowls-1244e9499e09d47b", "new bottom bowls-5c0ad1805c5cc614", "glass-8ce3e24bf8ab5c6c", "new bottom bowl-b9e128f5249ea986", "new bottom bowl 2-4c5a23d029c68fc1", "first big bowl-e3bdac110de97f87", "plastic container-8dfb8583a0fea3b6", "box of grapes-d58b9f1d65ebe5b2", "plastic sleeve-3e3ccd99ab3c7c95", "grapes-83d33d9598b3cc68", "new track 6-5bcf3facf279db57", "small bunch of grapes-4db9de2cf57b75a4", "bunch of grapes-39380d796a05df50", "last bunch of grapes-612ff9ffd3a0b645", "plastic box-fc168a7d19f5f915", "small plastic colander-a0ac641b94e24899", "colander-156217f7f4400618", "new sieve-573cb366ca5a60dc", "plate-d707b77979daed6b", "new small bottom bowl-c13da7e8da35f780", "small stack of bowls-d1891513ba1b5022", "final bowl-5b35fac800bf9fa5", "bowl of grapes-3f45f17ad1296808" ], "reasoning": "1. I took note of where the vacuum (hoover-fa42a926c802af23) was resting at the beginning of the video, which can be seen directly behind the dining table at 00:00.\n2. At 00:22 the user begins to use the vacuum (hoover-fa42a926c802af23), setting it on the other side of the dining table at 02:29 when finished vacuuming. I note that this is the second place that the vacuum (hoover-fa42a926c802af23) rests.\n3. At 03:14 the user takes out a saucepan (larger saucepan-d632760b19d7e71d) and then proceeds to fill it with water at 03:24-03:39.\n4. I slow the video down from 03:24-03:39 to note how many bricks on the top row of the wall outside are visible through the left window pane and to note any variations with movement. I conclude that 6 bricks are visible during this timeframe without any variations.\n5. I continued to watch until the end of the video to see whether the user moved the vacuum (hoover-fa42a926c802af23) again and determined that it was not moved again.\n6. Finally, I found the quotient of the bricks (6) and the number of resting places (2): 6/2=3.", "referenced_object_masks": [ "hoover-fa42a926c802af23", "larger saucepan-d632760b19d7e71d" ] }, { "key": "P03-20240218-140054:d674fff665e706bdfec3b17f6de45a3074103776", "video_id": "P03-20240218-140054", "question": "Where would the door I open most frequently in the final five minutes of the video be located in relation to me, if I were standing directly in front of and facing the dishwasher?", "answer_choice_0": "Ahead, above, and to the right", "answer_choice_1": "Behind, below, and to the left", "answer_choice_2": "Behind, above, and to the left", "answer_choice_3": "Behind, above, and to the right", "answer_choice_4": "Ahead, at eye level, and to the left", "answer_id": 3, "answer": "Behind, above, and to the right", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Counting", "Temporal Reasoning", "Situational Awareness", "Counterfactual" ], "objects_of_interest": [ "container-3db98c624e413b2d", "hoover-fa42a926c802af23", "hoover charger-a0ed1d356c17b389", "chair-2fefa7fb254fe497", "second chair-11232e9af4b48fc0", "pack of taiwanese pancakes-ee302825e73eb626", "second saucepan-ffe93f5370033200", "new track 1-11a097c1ffc7cd41", "third saucepan-a3508894581f53d4", "larger saucepan-d632760b19d7e71d", "new track 3-5cafbfb12a904c77", "container box-23ea8486f458f6ac", "dirty mug-dbbc1376dee81466", "rice cooker bowl-02ad871e2228da09", "rice cooker tray-4dd2f4ce6ec011bf", "right placemat-5f3795bbcbf3af68", "left placemat-22e05f03d8cfa22f", "water filter tank-002884c4d601ff0e", "laundry detergent cap-b8cdcbc511ace0b1", "plastic bag-a7907e8b1b98e5b2", "box-7292668b779034a7", "scissors-64d7d9c0d7b996fb", "safety cover of scissors-989a23c52aedebd5", "dry wooden spoon-b3f394aa2d811358", "small bowl-141df35b7a42703a", "other food bag-3f630662c6d6df73", "bottle of sesame oil-933272b80328c99f", "another smaller bottle-cecf7711b49e5304", "another bottle-27d3b981e862da6c", "bottle of soy sauce-f9c400376933e0f2", "small cup-9c3bcbf40b2d6e87", "knife-37f2303d21d59017", "garlic cloves-a6635f799069d2db", "cloth-ab363aebeb8a4521", "coffee filter stand-87876f19c710b2fb", "coffee filter-98e44309a4afddda", "new coffee filter cone-b101b208b921ce07", "red spoon-591d984a7e12315e", "frozen dumplings-6afa13ccb0f4fb62", "second set of frozen dumplings-25135ae7dded01a8", "last frozen dumpling-d74a8290a9239a04", "empty plastic bag-63a325a5cdbccba8", "orange cloth-5ebfaaee70535f6a", "first chopsticks-1e42f51bdc8c76e7", "second chopsticks-9fb14c743e71e22e", "first red chopstick-e4688d3a0dd2b34e", "second red chopstick-5bf110b7efab0714", "other pink butterfly shaped chopstick holder-ab75d44751329343", "blue butterfly shaped chopstick holder-abe050773fd1b530", "red chopsticks-3a4b1d0fbf79b413", "folded white cloth-9eb79751f6a7636e", "top empty plastic bag-8125f8410a873826", "other empty food bag-25b1b286ba900bf6", "dishwasher brush-a28fbab556e8b019", "small sponge-13244022c8cacc49", "washing up liquid-a06b408a43cc43ab", "new small bowl-ebb4474d35cc2b7c", "second bowl in stack of bowls-c270ecc940081bc8", "top bowl in stack of bowls-7b4f34b9a92e4a1f", "stack of small bowls-a160bfdbc189c8c4", "bowl2-1eef02d7d93c4931", "bottom two big bowls-eaed4b7ecc151b96", "top two big bowls-1244e9499e09d47b", "new bottom bowls-5c0ad1805c5cc614", "glass-8ce3e24bf8ab5c6c", "new bottom bowl-b9e128f5249ea986", "new bottom bowl 2-4c5a23d029c68fc1", "first big bowl-e3bdac110de97f87", "plastic container-8dfb8583a0fea3b6", "box of grapes-d58b9f1d65ebe5b2", "plastic sleeve-3e3ccd99ab3c7c95", "grapes-83d33d9598b3cc68", "new track 6-5bcf3facf279db57", "small bunch of grapes-4db9de2cf57b75a4", "bunch of grapes-39380d796a05df50", "last bunch of grapes-612ff9ffd3a0b645", "plastic box-fc168a7d19f5f915", "small plastic colander-a0ac641b94e24899", "colander-156217f7f4400618", "new sieve-573cb366ca5a60dc", "plate-d707b77979daed6b", "new small bottom bowl-c13da7e8da35f780", "small stack of bowls-d1891513ba1b5022", "final bowl-5b35fac800bf9fa5", "bowl of grapes-3f45f17ad1296808" ], "reasoning": "1. To determine this answer, I first watch from the user's perspective as the user opens doors in the video's final five minutes. Since the video is 18:46 long, I watch from 13:46-18:46.\n2. At 14:46, I observe the user open their first door, the leftmost drawer of the upper left cabinet in relation to the stovetop, counting this as the first instance of that door being opened.\n3. At 15:13, I observe the user open the top door of the fridge , counting this as the first instance of that door being opened.\n4. At 17:57, I observe the user open the leftmost drawer of the upper left cabinet in relation to the stovetop, counting this as the second instance of that door being opened.\n5. I watch the rest of the video and determine that no more doors are open, meaning that the leftmost upper cabinet door is the one most frequently opened during this span.\n6. Next, I hypothetically assume that the user is standing in front of, and facing, the dishwasher - from here, I imagine where the target cabinet door would be located. I picture the spatial layout of the kitchen and determine that the cabinet would be behind, above, and to the right of the user as they faced the dishwasher, matching answer choice C.", "referenced_object_masks": [] }, { "key": "P03-20240218-140054:f3c753029c436b47e2b24697aa15259f59024f25", "video_id": "P03-20240218-140054", "question": "What did I do with the container holding the grapes, after I was finished washing them in the sink?", "answer_choice_0": "Left it in the sink in front of the user", "answer_choice_1": "Put it under the sink beneath the user", "answer_choice_2": "Threw it away after walking to the trash can", "answer_choice_3": "Placed it in the fridge on a shelf eye level with the user", "answer_choice_4": "Placed it in a larger container at the furthest end of the hallway from the user", "answer_id": 4, "answer": "Placed it in a larger container at the furthest end of the hallway from the user", "question_type": [ "Action Recognition", "Object Recognition", "Egocentric & Spatial Perception", "Situational Awareness" ], "objects_of_interest": [ "container-3db98c624e413b2d", "hoover-fa42a926c802af23", "hoover charger-a0ed1d356c17b389", "chair-2fefa7fb254fe497", "second chair-11232e9af4b48fc0", "pack of taiwanese pancakes-ee302825e73eb626", "second saucepan-ffe93f5370033200", "new track 1-11a097c1ffc7cd41", "third saucepan-a3508894581f53d4", "larger saucepan-d632760b19d7e71d", "new track 3-5cafbfb12a904c77", "container box-23ea8486f458f6ac", "dirty mug-dbbc1376dee81466", "rice cooker bowl-02ad871e2228da09", "rice cooker tray-4dd2f4ce6ec011bf", "right placemat-5f3795bbcbf3af68", "left placemat-22e05f03d8cfa22f", "water filter tank-002884c4d601ff0e", "laundry detergent cap-b8cdcbc511ace0b1", "plastic bag-a7907e8b1b98e5b2", "box-7292668b779034a7", "scissors-64d7d9c0d7b996fb", "safety cover of scissors-989a23c52aedebd5", "dry wooden spoon-b3f394aa2d811358", "small bowl-141df35b7a42703a", "other food bag-3f630662c6d6df73", "bottle of sesame oil-933272b80328c99f", "another smaller bottle-cecf7711b49e5304", "another bottle-27d3b981e862da6c", "bottle of soy sauce-f9c400376933e0f2", "small cup-9c3bcbf40b2d6e87", "knife-37f2303d21d59017", "garlic cloves-a6635f799069d2db", "cloth-ab363aebeb8a4521", "coffee filter stand-87876f19c710b2fb", "coffee filter-98e44309a4afddda", "new coffee filter cone-b101b208b921ce07", "red spoon-591d984a7e12315e", "frozen dumplings-6afa13ccb0f4fb62", "second set of frozen dumplings-25135ae7dded01a8", "last frozen dumpling-d74a8290a9239a04", "empty plastic bag-63a325a5cdbccba8", "orange cloth-5ebfaaee70535f6a", "first chopsticks-1e42f51bdc8c76e7", "second chopsticks-9fb14c743e71e22e", "first red chopstick-e4688d3a0dd2b34e", "second red chopstick-5bf110b7efab0714", "other pink butterfly shaped chopstick holder-ab75d44751329343", "blue butterfly shaped chopstick holder-abe050773fd1b530", "red chopsticks-3a4b1d0fbf79b413", "folded white cloth-9eb79751f6a7636e", "top empty plastic bag-8125f8410a873826", "other empty food bag-25b1b286ba900bf6", "dishwasher brush-a28fbab556e8b019", "small sponge-13244022c8cacc49", "washing up liquid-a06b408a43cc43ab", "new small bowl-ebb4474d35cc2b7c", "second bowl in stack of bowls-c270ecc940081bc8", "top bowl in stack of bowls-7b4f34b9a92e4a1f", "stack of small bowls-a160bfdbc189c8c4", "bowl2-1eef02d7d93c4931", "bottom two big bowls-eaed4b7ecc151b96", "top two big bowls-1244e9499e09d47b", "new bottom bowls-5c0ad1805c5cc614", "glass-8ce3e24bf8ab5c6c", "new bottom bowl-b9e128f5249ea986", "new bottom bowl 2-4c5a23d029c68fc1", "first big bowl-e3bdac110de97f87", "plastic container-8dfb8583a0fea3b6", "box of grapes-d58b9f1d65ebe5b2", "plastic sleeve-3e3ccd99ab3c7c95", "grapes-83d33d9598b3cc68", "new track 6-5bcf3facf279db57", "small bunch of grapes-4db9de2cf57b75a4", "bunch of grapes-39380d796a05df50", "last bunch of grapes-612ff9ffd3a0b645", "plastic box-fc168a7d19f5f915", "small plastic colander-a0ac641b94e24899", "colander-156217f7f4400618", "new sieve-573cb366ca5a60dc", "plate-d707b77979daed6b", "new small bottom bowl-c13da7e8da35f780", "small stack of bowls-d1891513ba1b5022", "final bowl-5b35fac800bf9fa5", "bowl of grapes-3f45f17ad1296808" ], "reasoning": "1. I identified the timestamp where the user began to wash grapes (bunch of grapes-39380d796a05df50), and finishes washing grapes (last bunch of grapes-612ff9ffd3a0b645) in the sink from 15:42-16:13.\n2. After the user washed the grapes and placed them in a bowl on the sink counter, I then observed the user empty the remaining water and shake the container (plastic box-fc168a7d19f5f915) dry.\n3. The user then turned to walk the length of their hallway, passed the fridge, and then bent down to open a container with a green lid under a side table.", "referenced_object_masks": [ "bunch of grapes-39380d796a05df50", "plastic box-fc168a7d19f5f915", "last bunch of grapes-612ff9ffd3a0b645" ] }, { "key": "P03-20240218-140054:f6b08c4193578ac5fcd431aa373534220fb90eb1", "video_id": "P03-20240218-140054", "question": "What object would the 1st chair I touched displace if I moved it to the opposite wall to the right?", "answer_choice_0": "A plastic bag", "answer_choice_1": "The second chair", "answer_choice_2": "A silver trashcan", "answer_choice_3": "The vacuum", "answer_choice_4": "The vacuum charger", "answer_id": 2, "answer": "A silver trashcan", "question_type": [ "Action Recognition", "Object Recognition", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "container-3db98c624e413b2d", "hoover-fa42a926c802af23", "hoover charger-a0ed1d356c17b389", "chair-2fefa7fb254fe497", "second chair-11232e9af4b48fc0", "pack of taiwanese pancakes-ee302825e73eb626", "second saucepan-ffe93f5370033200", "new track 1-11a097c1ffc7cd41", "third saucepan-a3508894581f53d4", "larger saucepan-d632760b19d7e71d", "new track 3-5cafbfb12a904c77", "container box-23ea8486f458f6ac", "dirty mug-dbbc1376dee81466", "rice cooker bowl-02ad871e2228da09", "rice cooker tray-4dd2f4ce6ec011bf", "right placemat-5f3795bbcbf3af68", "left placemat-22e05f03d8cfa22f", "water filter tank-002884c4d601ff0e", "laundry detergent cap-b8cdcbc511ace0b1", "plastic bag-a7907e8b1b98e5b2", "box-7292668b779034a7", "scissors-64d7d9c0d7b996fb", "safety cover of scissors-989a23c52aedebd5", "dry wooden spoon-b3f394aa2d811358", "small bowl-141df35b7a42703a", "other food bag-3f630662c6d6df73", "bottle of sesame oil-933272b80328c99f", "another smaller bottle-cecf7711b49e5304", "another bottle-27d3b981e862da6c", "bottle of soy sauce-f9c400376933e0f2", "small cup-9c3bcbf40b2d6e87", "knife-37f2303d21d59017", "garlic cloves-a6635f799069d2db", "cloth-ab363aebeb8a4521", "coffee filter stand-87876f19c710b2fb", "coffee filter-98e44309a4afddda", "new coffee filter cone-b101b208b921ce07", "red spoon-591d984a7e12315e", "frozen dumplings-6afa13ccb0f4fb62", "second set of frozen dumplings-25135ae7dded01a8", "last frozen dumpling-d74a8290a9239a04", "empty plastic bag-63a325a5cdbccba8", "orange cloth-5ebfaaee70535f6a", "first chopsticks-1e42f51bdc8c76e7", "second chopsticks-9fb14c743e71e22e", "first red chopstick-e4688d3a0dd2b34e", "second red chopstick-5bf110b7efab0714", "other pink butterfly shaped chopstick holder-ab75d44751329343", "blue butterfly shaped chopstick holder-abe050773fd1b530", "red chopsticks-3a4b1d0fbf79b413", "folded white cloth-9eb79751f6a7636e", "top empty plastic bag-8125f8410a873826", "other empty food bag-25b1b286ba900bf6", "dishwasher brush-a28fbab556e8b019", "small sponge-13244022c8cacc49", "washing up liquid-a06b408a43cc43ab", "new small bowl-ebb4474d35cc2b7c", "second bowl in stack of bowls-c270ecc940081bc8", "top bowl in stack of bowls-7b4f34b9a92e4a1f", "stack of small bowls-a160bfdbc189c8c4", "bowl2-1eef02d7d93c4931", "bottom two big bowls-eaed4b7ecc151b96", "top two big bowls-1244e9499e09d47b", "new bottom bowls-5c0ad1805c5cc614", "glass-8ce3e24bf8ab5c6c", "new bottom bowl-b9e128f5249ea986", "new bottom bowl 2-4c5a23d029c68fc1", "first big bowl-e3bdac110de97f87", "plastic container-8dfb8583a0fea3b6", "box of grapes-d58b9f1d65ebe5b2", "plastic sleeve-3e3ccd99ab3c7c95", "grapes-83d33d9598b3cc68", "new track 6-5bcf3facf279db57", "small bunch of grapes-4db9de2cf57b75a4", "bunch of grapes-39380d796a05df50", "last bunch of grapes-612ff9ffd3a0b645", "plastic box-fc168a7d19f5f915", "small plastic colander-a0ac641b94e24899", "colander-156217f7f4400618", "new sieve-573cb366ca5a60dc", "plate-d707b77979daed6b", "new small bottom bowl-c13da7e8da35f780", "small stack of bowls-d1891513ba1b5022", "final bowl-5b35fac800bf9fa5", "bowl of grapes-3f45f17ad1296808" ], "reasoning": "1. I watched for the user to touch the 1st chair (chair-2fefa7fb254fe497), which occurs at 1:37.\n2. I looked to see what was on the opposite wall to the right of the chair (chair-2fefa7fb254fe497).\n3. There is a table and a refrigerator against the wall, and the object in between is a silver trash can , which is directly opposite the 1st chair (chair-2fefa7fb254fe497).", "referenced_object_masks": [ "chair-2fefa7fb254fe497" ] }, { "key": "P03-20240218-140054:fb211cff1e39f2b2d5158f1d45671756155b00ae", "video_id": "P03-20240218-140054", "question": "What's the purpose of putting the contents of the food bag into the water?", "answer_choice_0": "To Blanch them", "answer_choice_1": "To boil them", "answer_choice_2": "To poach them", "answer_choice_3": "To soak them", "answer_choice_4": "To wash them", "answer_id": 1, "answer": "To boil them", "question_type": [ "Goal Reasoning", "Action Recognition", "State Changes", "Temporal Reasoning", "Object Recognition" ], "objects_of_interest": [ "container-3db98c624e413b2d", "hoover-fa42a926c802af23", "hoover charger-a0ed1d356c17b389", "chair-2fefa7fb254fe497", "second chair-11232e9af4b48fc0", "pack of taiwanese pancakes-ee302825e73eb626", "second saucepan-ffe93f5370033200", "new track 1-11a097c1ffc7cd41", "third saucepan-a3508894581f53d4", "larger saucepan-d632760b19d7e71d", "new track 3-5cafbfb12a904c77", "container box-23ea8486f458f6ac", "dirty mug-dbbc1376dee81466", "rice cooker bowl-02ad871e2228da09", "rice cooker tray-4dd2f4ce6ec011bf", "right placemat-5f3795bbcbf3af68", "left placemat-22e05f03d8cfa22f", "water filter tank-002884c4d601ff0e", "laundry detergent cap-b8cdcbc511ace0b1", "plastic bag-a7907e8b1b98e5b2", "box-7292668b779034a7", "scissors-64d7d9c0d7b996fb", "safety cover of scissors-989a23c52aedebd5", "dry wooden spoon-b3f394aa2d811358", "small bowl-141df35b7a42703a", "other food bag-3f630662c6d6df73", "bottle of sesame oil-933272b80328c99f", "another smaller bottle-cecf7711b49e5304", "another bottle-27d3b981e862da6c", "bottle of soy sauce-f9c400376933e0f2", "small cup-9c3bcbf40b2d6e87", "knife-37f2303d21d59017", "garlic cloves-a6635f799069d2db", "cloth-ab363aebeb8a4521", "coffee filter stand-87876f19c710b2fb", "coffee filter-98e44309a4afddda", "new coffee filter cone-b101b208b921ce07", "red spoon-591d984a7e12315e", "frozen dumplings-6afa13ccb0f4fb62", "second set of frozen dumplings-25135ae7dded01a8", "last frozen dumpling-d74a8290a9239a04", "empty plastic bag-63a325a5cdbccba8", "orange cloth-5ebfaaee70535f6a", "first chopsticks-1e42f51bdc8c76e7", "second chopsticks-9fb14c743e71e22e", "first red chopstick-e4688d3a0dd2b34e", "second red chopstick-5bf110b7efab0714", "other pink butterfly shaped chopstick holder-ab75d44751329343", "blue butterfly shaped chopstick holder-abe050773fd1b530", "red chopsticks-3a4b1d0fbf79b413", "folded white cloth-9eb79751f6a7636e", "top empty plastic bag-8125f8410a873826", "other empty food bag-25b1b286ba900bf6", "dishwasher brush-a28fbab556e8b019", "small sponge-13244022c8cacc49", "washing up liquid-a06b408a43cc43ab", "new small bowl-ebb4474d35cc2b7c", "second bowl in stack of bowls-c270ecc940081bc8", "top bowl in stack of bowls-7b4f34b9a92e4a1f", "stack of small bowls-a160bfdbc189c8c4", "bowl2-1eef02d7d93c4931", "bottom two big bowls-eaed4b7ecc151b96", "top two big bowls-1244e9499e09d47b", "new bottom bowls-5c0ad1805c5cc614", "glass-8ce3e24bf8ab5c6c", "new bottom bowl-b9e128f5249ea986", "new bottom bowl 2-4c5a23d029c68fc1", "first big bowl-e3bdac110de97f87", "plastic container-8dfb8583a0fea3b6", "box of grapes-d58b9f1d65ebe5b2", "plastic sleeve-3e3ccd99ab3c7c95", "grapes-83d33d9598b3cc68", "new track 6-5bcf3facf279db57", "small bunch of grapes-4db9de2cf57b75a4", "bunch of grapes-39380d796a05df50", "last bunch of grapes-612ff9ffd3a0b645", "plastic box-fc168a7d19f5f915", "small plastic colander-a0ac641b94e24899", "colander-156217f7f4400618", "new sieve-573cb366ca5a60dc", "plate-d707b77979daed6b", "new small bottom bowl-c13da7e8da35f780", "small stack of bowls-d1891513ba1b5022", "final bowl-5b35fac800bf9fa5", "bowl of grapes-3f45f17ad1296808" ], "reasoning": "1. I found the food bag (other empty food bag-25b1b286ba900bf6) and located the moment its contents (frozen dumplings-6afa13ccb0f4fb62) were about to be placed in water at 10:47.\n2. I saw the object holding the water that the food (frozen dumplings-6afa13ccb0f4fb62) was about to be placed into (larger saucepan-d632760b19d7e71d), had steam rising off of the surface of the water as the stove burner was on. This series of events means that the purpose of the process couldn't be washing, blanching, or simply soaking.\n3. As the user came back to the stove at 16:34, I saw that the pan began to boil, with froth and bubbles indicating the nature of the process.", "referenced_object_masks": [ "larger saucepan-d632760b19d7e71d", "frozen dumplings-6afa13ccb0f4fb62", "other empty food bag-25b1b286ba900bf6" ] }, { "key": "P03-20240218-144430:02ab04998479ed848214019671d915eaf8c6d84a", "video_id": "P03-20240218-144430", "question": "If I move the food plate to the left of the top shelf, and put the beer under it on the second shelf before I put the bowl inside the fridge, which item would be the closest to the beer when I close the refrigerator door?", "answer_choice_0": "Milk", "answer_choice_1": "Eggs", "answer_choice_2": "Tomato Sauce", "answer_choice_3": "Yogurt", "answer_choice_4": "Tomatoes", "answer_id": 4, "answer": "Tomatoes", "question_type": [ "Counterfactual", "Object Recognition", "Egocentric & Spatial Perception", "Action Recognition" ], "objects_of_interest": [ "plate-68995d6c413d74c5", "rubbish-9649e6ab70961ab0", "plastic pot-816832240200f827", "piece of plastic-6d067e4665e159b4", "tiramisu cardboard packaging-c58a2d9619da6e30", "yellow choptsticks-49bb73368f3966cf", "red chopsticks-6ef279c277ffe646", "bowl-10f8a36128219ceb", "bowl2-124c9011e840f379", "dishwasher brush-5e57d282ad52352f", "used tea bag-5652afaed2cc2207", "knife-12bbfe97ac901ef0", "knife2-d7572b114bdf92cb", "two teaspoons-65d088d7f1fe24c7", "something-58516898e60db1b3", "dirty colander-d4110985f9b794cf", "tiramisu pot-9483b029e2f22ab7", "little sponge-e1496936a7a97ae9", "nutella jar-47a08c96fa652b11", "bowl of blueberries-cd53f3e3b92dae7c", "plate of food-cf24e75a0d522ba0", "saucepan-753239c8f0fe610d", "bowl3-059f5c0929704800", "food waste bin-9f90a617696ab8fd", "food piece-c600a020a6156e49", "food piece2-d5aa8ff2fac206f7", "food piece3-6e1160b422c1279c", "food piece4-7d982122a6d5cdca", "food piece5-d86c7e7b7fd39041", "food piece6-12ade4051c667b2b", "cloth-9ed6f0a16ffe9e9e", "sink plug-315fa893832b0b4c", "orange cloth-a7b43b712724e7bf", "soapbox-e8261f061e675cd8", "kettle-f11decbf61d52dd5", "cloth2-efabc2cdf90d3873", "coffee beans bag-358c69ed9f089fa0", "clip-55ce6862e2e52622", "coffee grinder-4a91c606d89e676e", "coffee grinder lid-c3812ad474e0c3c8", "red spoon-a1dbdd5a1c9ea824", "coffee jug-c178d5dd69f7efbe", "coffee grinder box-4805534cd413ce90", "coffee filter-0e3196c308e70981", "coffee grinder box lid-e5d47850c26d3025", "coffee jug and filter combined-d16ee991ea8edb45", "butterfly shaped chopsticks holder-a965014b8eedcd9e", "other butterfly shaped chopsticks holder-892a72dcd7f3073a", "place mat-d505e6b37b624959", "placemat2-a9ecbbfd81c12f99", "coffee mug-a148820d6c69b32d", "coffee mug2-e671c8375892665c", "smaller coffee mug-e43fa4c8a2c70d3f", "teapot-17c15612da40f459" ], "reasoning": "1. I watched the video to determine when the user opened the refigerator , which occurs at 02:49.\n2. After the user opened the refrigerator, I paused the video to examine the contents of the fridge and locate the items listed in the question: the food plate (plate of food-cf24e75a0d522ba0) and the beer . I noticed the food plate (plate of food-cf24e75a0d522ba0) is on the third shelf, and the beer is on the top shelf to the right. \n3. From 02:49-02:55, the refrigerator door is open. I examined which items are on the second shelf since this would be the closest to the beer shelf when the user closes the refrigerator. From left to right, there's a carton of eggs , cheese , and a yogurt container .\n3. I reasoned that if the user puts the food plate on the left of the top shelf and moves the beer right under it on the second shelf, the beer would be on the left side of the second drawer. Therefore, when the user closes the fridge, the closest item to the beer would be the yogurt container , which would be right in front of it.", "referenced_object_masks": [ "plate of food-cf24e75a0d522ba0" ] }, { "key": "P03-20240218-144430:61b7fb4e2f76fc3bb4174d7141c8df5658005e72", "video_id": "P03-20240218-144430", "question": "What is the second item I touch with my left hand after filling the electric kettle with water?", "answer_choice_0": "The stove light", "answer_choice_1": "The kettle lid", "answer_choice_2": "The orange towel", "answer_choice_3": "The dishwasher", "answer_choice_4": "The kitchen faucet", "answer_id": 4, "answer": "The kitchen faucet", "question_type": [ "Egocentric & Spatial Perception", "Event Occurence", "Situational Awareness", "Object Recognition" ], "objects_of_interest": [ "plate-68995d6c413d74c5", "rubbish-9649e6ab70961ab0", "plastic pot-816832240200f827", "piece of plastic-6d067e4665e159b4", "tiramisu cardboard packaging-c58a2d9619da6e30", "yellow choptsticks-49bb73368f3966cf", "red chopsticks-6ef279c277ffe646", "bowl-10f8a36128219ceb", "bowl2-124c9011e840f379", "dishwasher brush-5e57d282ad52352f", "used tea bag-5652afaed2cc2207", "knife-12bbfe97ac901ef0", "knife2-d7572b114bdf92cb", "two teaspoons-65d088d7f1fe24c7", "something-58516898e60db1b3", "dirty colander-d4110985f9b794cf", "tiramisu pot-9483b029e2f22ab7", "little sponge-e1496936a7a97ae9", "nutella jar-47a08c96fa652b11", "bowl of blueberries-cd53f3e3b92dae7c", "plate of food-cf24e75a0d522ba0", "saucepan-753239c8f0fe610d", "bowl3-059f5c0929704800", "food waste bin-9f90a617696ab8fd", "food piece-c600a020a6156e49", "food piece2-d5aa8ff2fac206f7", "food piece3-6e1160b422c1279c", "food piece4-7d982122a6d5cdca", "food piece5-d86c7e7b7fd39041", "food piece6-12ade4051c667b2b", "cloth-9ed6f0a16ffe9e9e", "sink plug-315fa893832b0b4c", "orange cloth-a7b43b712724e7bf", "soapbox-e8261f061e675cd8", "kettle-f11decbf61d52dd5", "cloth2-efabc2cdf90d3873", "coffee beans bag-358c69ed9f089fa0", "clip-55ce6862e2e52622", "coffee grinder-4a91c606d89e676e", "coffee grinder lid-c3812ad474e0c3c8", "red spoon-a1dbdd5a1c9ea824", "coffee jug-c178d5dd69f7efbe", "coffee grinder box-4805534cd413ce90", "coffee filter-0e3196c308e70981", "coffee grinder box lid-e5d47850c26d3025", "coffee jug and filter combined-d16ee991ea8edb45", "butterfly shaped chopsticks holder-a965014b8eedcd9e", "other butterfly shaped chopsticks holder-892a72dcd7f3073a", "place mat-d505e6b37b624959", "placemat2-a9ecbbfd81c12f99", "coffee mug-a148820d6c69b32d", "coffee mug2-e671c8375892665c", "smaller coffee mug-e43fa4c8a2c70d3f", "teapot-17c15612da40f459" ], "reasoning": "1. I watched the video to see when the user fills the electric kettle (kettle-f11decbf61d52dd5) with water, which occurs from 07:26-07:32.\n2. I noted that the user holds the kettle (kettle-f11decbf61d52dd5) with the left hand until setting it down - after filling the kettle (kettle-f11decbf61d52dd5), the first thing I notice the user do with their right hand is flip the kettle's lid down at 07:32.\n3. The second thing I noticed the user touch with their right hand is the orange towel (orange cloth-a7b43b712724e7bf) at 07:35. The user then proceeds to clean the countertop with the orange towel (orange cloth-a7b43b712724e7bf) from 07:35-08:03.\n4. I count the third thing that the user touches with their right hand as the tap on the kitchen faucet at 08:04, making E the correct answer.", "referenced_object_masks": [ "orange cloth-a7b43b712724e7bf", "kettle-f11decbf61d52dd5" ] }, { "key": "P03-20240218-144430:6de9c6dfe15cc35663809d2702161469adcf9317", "video_id": "P03-20240218-144430", "question": "How many knives do I clean before putting them in the dishwasher?", "answer_choice_0": "2", "answer_choice_1": "3", "answer_choice_2": "0", "answer_choice_3": "4", "answer_choice_4": "1", "answer_id": 4, "answer": "1", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Counting", "Action Recognition" ], "objects_of_interest": [ "plate-68995d6c413d74c5", "rubbish-9649e6ab70961ab0", "plastic pot-816832240200f827", "piece of plastic-6d067e4665e159b4", "tiramisu cardboard packaging-c58a2d9619da6e30", "yellow choptsticks-49bb73368f3966cf", "red chopsticks-6ef279c277ffe646", "bowl-10f8a36128219ceb", "bowl2-124c9011e840f379", "dishwasher brush-5e57d282ad52352f", "used tea bag-5652afaed2cc2207", "knife-12bbfe97ac901ef0", "knife2-d7572b114bdf92cb", "two teaspoons-65d088d7f1fe24c7", "something-58516898e60db1b3", "dirty colander-d4110985f9b794cf", "tiramisu pot-9483b029e2f22ab7", "little sponge-e1496936a7a97ae9", "nutella jar-47a08c96fa652b11", "bowl of blueberries-cd53f3e3b92dae7c", "plate of food-cf24e75a0d522ba0", "saucepan-753239c8f0fe610d", "bowl3-059f5c0929704800", "food waste bin-9f90a617696ab8fd", "food piece-c600a020a6156e49", "food piece2-d5aa8ff2fac206f7", "food piece3-6e1160b422c1279c", "food piece4-7d982122a6d5cdca", "food piece5-d86c7e7b7fd39041", "food piece6-12ade4051c667b2b", "cloth-9ed6f0a16ffe9e9e", "sink plug-315fa893832b0b4c", "orange cloth-a7b43b712724e7bf", "soapbox-e8261f061e675cd8", "kettle-f11decbf61d52dd5", "cloth2-efabc2cdf90d3873", "coffee beans bag-358c69ed9f089fa0", "clip-55ce6862e2e52622", "coffee grinder-4a91c606d89e676e", "coffee grinder lid-c3812ad474e0c3c8", "red spoon-a1dbdd5a1c9ea824", "coffee jug-c178d5dd69f7efbe", "coffee grinder box-4805534cd413ce90", "coffee filter-0e3196c308e70981", "coffee grinder box lid-e5d47850c26d3025", "coffee jug and filter combined-d16ee991ea8edb45", "butterfly shaped chopsticks holder-a965014b8eedcd9e", "other butterfly shaped chopsticks holder-892a72dcd7f3073a", "place mat-d505e6b37b624959", "placemat2-a9ecbbfd81c12f99", "coffee mug-a148820d6c69b32d", "coffee mug2-e671c8375892665c", "smaller coffee mug-e43fa4c8a2c70d3f", "teapot-17c15612da40f459" ], "reasoning": "1. I watched the video to see when the user opened the dishwasher , which occurs from 00:49-00:50.\n2. I then watched the user put dishes in the dishwasher, waiting to see the first knife (knife-12bbfe97ac901ef0), which the user grabs at 01:20.\n3. After grabbing the knife (knife-12bbfe97ac901ef0), the user cleans it from 01:21-01:27 before putting it in the dishwasher from 01:27-01:29.\n4. As the video continued, I saw the user grab a second knife (knife2-d7572b114bdf92cb), putting it in the dishwasher from 01:29-01:31 without washing it.\n5. I continued to watch the user load the dishwasher, and did not observe any other knives being placed in it, meaning only one knife was washed before being placed in.", "referenced_object_masks": [ "knife-12bbfe97ac901ef0", "knife2-d7572b114bdf92cb" ] }, { "key": "P03-20240218-144430:d794ce5b452d553af8c1b708501b5a0963982bc5", "video_id": "P03-20240218-144430", "question": "What is the third to last item I thouch with my right hand?", "answer_choice_0": "The kitchen sink", "answer_choice_1": "Refrigerator door", "answer_choice_2": "Chopsticks", "answer_choice_3": "Dishwasher door", "answer_choice_4": "The coffee filter", "answer_id": 4, "answer": "The coffee filter", "question_type": [ "Egocentric & Spatial Perception", "Goal Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "plate-68995d6c413d74c5", "rubbish-9649e6ab70961ab0", "plastic pot-816832240200f827", "piece of plastic-6d067e4665e159b4", "tiramisu cardboard packaging-c58a2d9619da6e30", "yellow choptsticks-49bb73368f3966cf", "red chopsticks-6ef279c277ffe646", "bowl-10f8a36128219ceb", "bowl2-124c9011e840f379", "dishwasher brush-5e57d282ad52352f", "used tea bag-5652afaed2cc2207", "knife-12bbfe97ac901ef0", "knife2-d7572b114bdf92cb", "two teaspoons-65d088d7f1fe24c7", "something-58516898e60db1b3", "dirty colander-d4110985f9b794cf", "tiramisu pot-9483b029e2f22ab7", "little sponge-e1496936a7a97ae9", "nutella jar-47a08c96fa652b11", "bowl of blueberries-cd53f3e3b92dae7c", "plate of food-cf24e75a0d522ba0", "saucepan-753239c8f0fe610d", "bowl3-059f5c0929704800", "food waste bin-9f90a617696ab8fd", "food piece-c600a020a6156e49", "food piece2-d5aa8ff2fac206f7", "food piece3-6e1160b422c1279c", "food piece4-7d982122a6d5cdca", "food piece5-d86c7e7b7fd39041", "food piece6-12ade4051c667b2b", "cloth-9ed6f0a16ffe9e9e", "sink plug-315fa893832b0b4c", "orange cloth-a7b43b712724e7bf", "soapbox-e8261f061e675cd8", "kettle-f11decbf61d52dd5", "cloth2-efabc2cdf90d3873", "coffee beans bag-358c69ed9f089fa0", "clip-55ce6862e2e52622", "coffee grinder-4a91c606d89e676e", "coffee grinder lid-c3812ad474e0c3c8", "red spoon-a1dbdd5a1c9ea824", "coffee jug-c178d5dd69f7efbe", "coffee grinder box-4805534cd413ce90", "coffee filter-0e3196c308e70981", "coffee grinder box lid-e5d47850c26d3025", "coffee jug and filter combined-d16ee991ea8edb45", "butterfly shaped chopsticks holder-a965014b8eedcd9e", "other butterfly shaped chopsticks holder-892a72dcd7f3073a", "place mat-d505e6b37b624959", "placemat2-a9ecbbfd81c12f99", "coffee mug-a148820d6c69b32d", "coffee mug2-e671c8375892665c", "smaller coffee mug-e43fa4c8a2c70d3f", "teapot-17c15612da40f459" ], "reasoning": "1. I watched to the end of the video to see that the last item the user touches is the second coffee mug (coffee mug2-e671c8375892665c), touched from 14:23-14:24.\n2. I then went backwards to see that the user previously touches the first coffee mug (coffee mug-a148820d6c69b32d) at 14:19.\n3. Looking before this action, I saw that the user previously grabs and touches the coffee filter (coffee filter-0e3196c308e70981) from 14:09-14:16.", "referenced_object_masks": [ "coffee mug-a148820d6c69b32d", "coffee mug2-e671c8375892665c", "coffee filter-0e3196c308e70981" ] }, { "key": "P03-20240218-144430:da4e365f731314481d84b1dd3200acfbaeee35a2", "video_id": "P03-20240218-144430", "question": "What is the product of the number of scoops of coffee I put into the coffee grinder and the number of colored coasters on the table to the left of the refigerator?", "answer_choice_0": "15", "answer_choice_1": "6", "answer_choice_2": "12", "answer_choice_3": "9", "answer_choice_4": "4", "answer_id": 1, "answer": "6", "question_type": [ "Numerical Reasoning", "Counting", "Object Recognition", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "plate-68995d6c413d74c5", "rubbish-9649e6ab70961ab0", "plastic pot-816832240200f827", "piece of plastic-6d067e4665e159b4", "tiramisu cardboard packaging-c58a2d9619da6e30", "yellow choptsticks-49bb73368f3966cf", "red chopsticks-6ef279c277ffe646", "bowl-10f8a36128219ceb", "bowl2-124c9011e840f379", "dishwasher brush-5e57d282ad52352f", "used tea bag-5652afaed2cc2207", "knife-12bbfe97ac901ef0", "knife2-d7572b114bdf92cb", "two teaspoons-65d088d7f1fe24c7", "something-58516898e60db1b3", "dirty colander-d4110985f9b794cf", "tiramisu pot-9483b029e2f22ab7", "little sponge-e1496936a7a97ae9", "nutella jar-47a08c96fa652b11", "bowl of blueberries-cd53f3e3b92dae7c", "plate of food-cf24e75a0d522ba0", "saucepan-753239c8f0fe610d", "bowl3-059f5c0929704800", "food waste bin-9f90a617696ab8fd", "food piece-c600a020a6156e49", "food piece2-d5aa8ff2fac206f7", "food piece3-6e1160b422c1279c", "food piece4-7d982122a6d5cdca", "food piece5-d86c7e7b7fd39041", "food piece6-12ade4051c667b2b", "cloth-9ed6f0a16ffe9e9e", "sink plug-315fa893832b0b4c", "orange cloth-a7b43b712724e7bf", "soapbox-e8261f061e675cd8", "kettle-f11decbf61d52dd5", "cloth2-efabc2cdf90d3873", "coffee beans bag-358c69ed9f089fa0", "clip-55ce6862e2e52622", "coffee grinder-4a91c606d89e676e", "coffee grinder lid-c3812ad474e0c3c8", "red spoon-a1dbdd5a1c9ea824", "coffee jug-c178d5dd69f7efbe", "coffee grinder box-4805534cd413ce90", "coffee filter-0e3196c308e70981", "coffee grinder box lid-e5d47850c26d3025", "coffee jug and filter combined-d16ee991ea8edb45", "butterfly shaped chopsticks holder-a965014b8eedcd9e", "other butterfly shaped chopsticks holder-892a72dcd7f3073a", "place mat-d505e6b37b624959", "placemat2-a9ecbbfd81c12f99", "coffee mug-a148820d6c69b32d", "coffee mug2-e671c8375892665c", "smaller coffee mug-e43fa4c8a2c70d3f", "teapot-17c15612da40f459" ], "reasoning": "1. I watched the video to see when the user scoops coffee into the coffee grinder (coffee grinder-4a91c606d89e676e), which occurs from 09:00-09:21.\n2. During this time span, I watched to count how many scoops the user puts into the grinder (coffee grinder-4a91c606d89e676e)\n3. From 09:09-09:12, the user scoops the first scoop of coffee (coffee beans bag-358c69ed9f089fa0).\n4. Then, the user scoops the second scoop of coffee (coffee beans bag-358c69ed9f089fa0) from 09:16-09:18.\n5. I then noted that the user scoops 2 scoops of coffee into the grinder.\n6. Next, I continued to watch the video to identify when the table to the left of the fridge comes into view, which occurs at 10:03.\n7. At this time, I looked at the table and counted 3 coasters .\n8. Lastly, I did the final math of 2 x 3 = 6.", "referenced_object_masks": [ "coffee grinder-4a91c606d89e676e", "coffee beans bag-358c69ed9f089fa0" ] }, { "key": "P03-20240218-190556:03683f745f33441e239a2d7888b8121f2184cec8", "video_id": "P03-20240218-190556", "question": "After washing the third mushroom and then turning to look in another direction, what color is the farthest object away from me on my right?", "answer_choice_0": "Yellow", "answer_choice_1": "Red", "answer_choice_2": "Black", "answer_choice_3": "Blue", "answer_choice_4": "Green", "answer_id": 1, "answer": "Red", "question_type": [ "Action Recognition", "Object Recognition", "Counting", "Egocentric & Spatial Perception", "Situational Awareness" ], "objects_of_interest": [ "recipe card-75931342cb8e90e7", "recipe card2-a931478e0377d548", "bag of potatoes-2094c7856761ecff", "box of mushrooms-d34c859d4fdd04c7", "bowl-eaa20b0ad329270f", "scale-26730edecfc504e5", "chopping board-116f0066f17df5bf", "one potato-b8c21cbc3dfcb948", "another potato-c8ead991588f3249", "third potato-9275b300c513dbaa", "fourth potato-e2ad6ce07bf58571", "fifth potato-2c3eb1393e6bb23c", "sixth potato-38d6166b22ad78b3", "big potato-d1f760f1b2b4fd58", "large bowl-dedd4a9a84e5af46", "large bowl2-b8d7c59dc22726e1", "used v60 coffee filter-75e6c42365e15da1", "cup-63f296c573fe7524", "smaller cup-af1bed8f6d7c0e3b", "potato peeler-379c1ff72ea5901f", "potato7-bf62354ddd692a8c", "potato8-b2c78291e683fa86", "potato9-d2fb164598e2263c", "potato10-66ded1b6c76d0ad7", "potato11-f144cbd6bed62072", "potato12-d2eb08a038025fb0", "large saucepan-aa2d9122dc0ded15", "knife-8cf58492cde33b7f", "potato chunks-f615ebb909ffcf40", "tea towel-b46c32f1499be80c", "plastic colander-4400d2bf8c957e7a", "plant-based burger-c55227d550b2d835", "mushroom1-f97cfbfa6c2750c6", "mushroom2-4f2442a9f0ca0a2e", "mushroom3-d0890334025a816a", "mushroom4-6949b206ccb3d811", "mushroom5-2fee8f74c638d686", "mushroom6-06454cfb785f7732", "box of mushrooms2-41245785705ef10f", "mushroom7-235726fa159c36c9", "mushroom8-49d2cf2e8d84b728", "plastic wrapping-83697ef0d4046828", "mushroom10-a131d4607e0720e8", "orange cloth-22f1fd13931806e3", "bag of onions-0d93a8546a5edeeb", "onion-381959e3b0582059", "half onion-753c79e6ec54a8af", "half onion2-641875f8152a79fc", "cloth-a8e98474cb456d87", "wooden spoon-43e48cdcb1c4655e", "tomato puree tube-ee35a7d1de4001e7", "butter package-ef891def0a8f3b64", "small bowl-c51f838b88d1a4c3", "food waste bin-92d10b34b87b6fa3", "potato peel-f6c4d7af162cba5f", "second batch of peels-365ac50e045900fe", "v60 filter stand-2dc06c7e6ec02b4d", "plastic sleeve-50e8f31d5cd16fe4", "burger packaging2-ed8e0567384a0caa", "burger packaging-ad683c3dd3413267", "stack of small frying pan-7a6c65d1934c50d5", "large frying pan-6988e61572bf0e6e", "pan padding-c89a7e0b03d0211e", "recipe kit box-bf8b96fd9cc309fd", "recipe kit box2-01983ad6eabd69c7", "yellow tab-540164355061d556", "plate-90b91fc4d2b70284", "one strip of plant-based burger-09ea87a42b1e082b", "strip of plant-based burger2-079638a955ae5eea", "strip of plant-based burgers3-9c36085b2a6e3cec", "four last strips of plant-based burger-315383ee03035bd2", "sunflower oil bottle-b2ff5cefc018b27c", "sunflower oil bottle cap-dde4e81960836607", "teaspoon-87d8e02e6e646653", "plastic spatula-829e8a803d099511", "knife2-5389ba9dd0e6ec0d", "sponge-e8ca10bbf9926f86", "slab of batter-d327ec37289dd20b", "sour cream pot-62823ab54aaa6baf", "pink recipe tub-4cb42f6264a765d8", "tomato puree tube cap-86038f3975d839c3", "spoon-31c17545b55b8c73", "foil sleeve of sour cream-1413ebfec624d11d", "plate2-b1be62d80b83f854", "metal colander-e7f16d9bb8f8bb30", "slab of butter2-3a4bf10f5bd1db39", "potato masher-4eb23c6ecef251f2", "green recipe tub-c6eb1b8f8c86989f", "tea towel2-6c26adf3d0c0f78b", "tea towel3-619eab3ed44e2662", "ebook reader-d87491ccb26b19c3", "red glass-267e58100f235154", "green glass-4e481882ef31c08f", "water plastic bottle-ef19eb66b1ab3e0a", "rice cooker bowl-5d43487d17ad8afc", "dishwasher brush-b52601e9a67279d3", "plate3-d503d334731b0c4b", "plate4-0c626a74bc928970", "peeler2-44c2d131e6cf215f", "small bowl2-341c1d019798633d", "small bowl3-c86250647498d9be", "washing up liquid bottle-0535bb06843e8f28", "plastic bag-fc523455845d004a", "bowl4-3376447088c8fb27", "fork-04835a7e627a700e", "fork2-eb4cf8a453098c76", "chair-388a550480f274cd", "chair2-df116c6b9c7bb0b1", "coffee jug-feae76b47e550c46" ], "reasoning": "1. I watched the video to find the moment when the user was washing mushrooms. I found that this segment began at 13:19, when the user grabbed a colander (plastic colander-4400d2bf8c957e7a) full of mushrooms (mushroom1-f97cfbfa6c2750c6, mushroom2-4f2442a9f0ca0a2e, mushroom3-d0890334025a816a, mushroom4-6949b206ccb3d811, mushroom5-2fee8f74c638d686, mushroom6-06454cfb785f7732, mushroom7-235726fa159c36c9, mushroom8-49d2cf2e8d84b728).\n2. At 13:20, the user placed the colander (plastic colander-4400d2bf8c957e7a) full of mushrooms (mushroom1-f97cfbfa6c2750c6, mushroom2-4f2442a9f0ca0a2e, mushroom3-d0890334025a816a, mushroom4-6949b206ccb3d811, mushroom5-2fee8f74c638d686, mushroom6-06454cfb785f7732, mushroom7-235726fa159c36c9, mushroom8-49d2cf2e8d84b728) into the sink and turned on the water.\n3. From 13:22-13:26, the user washed a mushroom (mushroom7-235726fa159c36c9).\n4. From 13:27-13:30, the user washed a 2nd mushroom (mushroom8-49d2cf2e8d84b728)\n5. From 13:31-13:33, the user washed a 3rd mushroom (mushroom6-06454cfb785f7732).\n6. Then, I watched the user turn to the right from 13:33-13:34.\n7. At this point, I looked for the object that was farthest away from the user on the right. I found the object, which resembles a box next to the wall to the right of the oven in the corner. The box is red, making that the final answer.", "referenced_object_masks": [ "mushroom6-06454cfb785f7732", "plastic colander-4400d2bf8c957e7a", "mushroom1-f97cfbfa6c2750c6", "mushroom2-4f2442a9f0ca0a2e", "mushroom3-d0890334025a816a", "mushroom4-6949b206ccb3d811", "mushroom5-2fee8f74c638d686", "mushroom7-235726fa159c36c9", "mushroom8-49d2cf2e8d84b728" ] }, { "key": "P03-20240218-190556:2e62a41a632aa30aef84cea463bcdf1bcaee5153", "video_id": "P03-20240218-190556", "question": "In terms of a clock, where is the cutting board in relation to me as I add the cut burgers to the pan?", "answer_choice_0": "8 o'clock", "answer_choice_1": "4 o'clock", "answer_choice_2": "6 o'clock", "answer_choice_3": "2 o'clock", "answer_choice_4": "10 o'clock", "answer_id": 0, "answer": "8 o'clock", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Situational Awareness", "Object Recognition" ], "objects_of_interest": [ "recipe card-75931342cb8e90e7", "recipe card2-a931478e0377d548", "bag of potatoes-2094c7856761ecff", "box of mushrooms-d34c859d4fdd04c7", "bowl-eaa20b0ad329270f", "scale-26730edecfc504e5", "chopping board-116f0066f17df5bf", "one potato-b8c21cbc3dfcb948", "another potato-c8ead991588f3249", "third potato-9275b300c513dbaa", "fourth potato-e2ad6ce07bf58571", "fifth potato-2c3eb1393e6bb23c", "sixth potato-38d6166b22ad78b3", "big potato-d1f760f1b2b4fd58", "large bowl-dedd4a9a84e5af46", "large bowl2-b8d7c59dc22726e1", "used v60 coffee filter-75e6c42365e15da1", "cup-63f296c573fe7524", "smaller cup-af1bed8f6d7c0e3b", "potato peeler-379c1ff72ea5901f", "potato7-bf62354ddd692a8c", "potato8-b2c78291e683fa86", "potato9-d2fb164598e2263c", "potato10-66ded1b6c76d0ad7", "potato11-f144cbd6bed62072", "potato12-d2eb08a038025fb0", "large saucepan-aa2d9122dc0ded15", "knife-8cf58492cde33b7f", "potato chunks-f615ebb909ffcf40", "tea towel-b46c32f1499be80c", "plastic colander-4400d2bf8c957e7a", "plant-based burger-c55227d550b2d835", "mushroom1-f97cfbfa6c2750c6", "mushroom2-4f2442a9f0ca0a2e", "mushroom3-d0890334025a816a", "mushroom4-6949b206ccb3d811", "mushroom5-2fee8f74c638d686", "mushroom6-06454cfb785f7732", "box of mushrooms2-41245785705ef10f", "mushroom7-235726fa159c36c9", "mushroom8-49d2cf2e8d84b728", "plastic wrapping-83697ef0d4046828", "mushroom10-a131d4607e0720e8", "orange cloth-22f1fd13931806e3", "bag of onions-0d93a8546a5edeeb", "onion-381959e3b0582059", "half onion-753c79e6ec54a8af", "half onion2-641875f8152a79fc", "cloth-a8e98474cb456d87", "wooden spoon-43e48cdcb1c4655e", "tomato puree tube-ee35a7d1de4001e7", "butter package-ef891def0a8f3b64", "small bowl-c51f838b88d1a4c3", "food waste bin-92d10b34b87b6fa3", "potato peel-f6c4d7af162cba5f", "second batch of peels-365ac50e045900fe", "v60 filter stand-2dc06c7e6ec02b4d", "plastic sleeve-50e8f31d5cd16fe4", "burger packaging2-ed8e0567384a0caa", "burger packaging-ad683c3dd3413267", "stack of small frying pan-7a6c65d1934c50d5", "large frying pan-6988e61572bf0e6e", "pan padding-c89a7e0b03d0211e", "recipe kit box-bf8b96fd9cc309fd", "recipe kit box2-01983ad6eabd69c7", "yellow tab-540164355061d556", "plate-90b91fc4d2b70284", "one strip of plant-based burger-09ea87a42b1e082b", "strip of plant-based burger2-079638a955ae5eea", "strip of plant-based burgers3-9c36085b2a6e3cec", "four last strips of plant-based burger-315383ee03035bd2", "sunflower oil bottle-b2ff5cefc018b27c", "sunflower oil bottle cap-dde4e81960836607", "teaspoon-87d8e02e6e646653", "plastic spatula-829e8a803d099511", "knife2-5389ba9dd0e6ec0d", "sponge-e8ca10bbf9926f86", "slab of batter-d327ec37289dd20b", "sour cream pot-62823ab54aaa6baf", "pink recipe tub-4cb42f6264a765d8", "tomato puree tube cap-86038f3975d839c3", "spoon-31c17545b55b8c73", "foil sleeve of sour cream-1413ebfec624d11d", "plate2-b1be62d80b83f854", "metal colander-e7f16d9bb8f8bb30", "slab of butter2-3a4bf10f5bd1db39", "potato masher-4eb23c6ecef251f2", "green recipe tub-c6eb1b8f8c86989f", "tea towel2-6c26adf3d0c0f78b", "tea towel3-619eab3ed44e2662", "ebook reader-d87491ccb26b19c3", "red glass-267e58100f235154", "green glass-4e481882ef31c08f", "water plastic bottle-ef19eb66b1ab3e0a", "rice cooker bowl-5d43487d17ad8afc", "dishwasher brush-b52601e9a67279d3", "plate3-d503d334731b0c4b", "plate4-0c626a74bc928970", "peeler2-44c2d131e6cf215f", "small bowl2-341c1d019798633d", "small bowl3-c86250647498d9be", "washing up liquid bottle-0535bb06843e8f28", "plastic bag-fc523455845d004a", "bowl4-3376447088c8fb27", "fork-04835a7e627a700e", "fork2-eb4cf8a453098c76", "chair-388a550480f274cd", "chair2-df116c6b9c7bb0b1", "coffee jug-feae76b47e550c46" ], "reasoning": "1. I watched the video to look for where the cutting board (chopping board-116f0066f17df5bf) was located as the user added the cut burgers (one strip of plant-based burger-09ea87a42b1e082b, strip of plant-based burger2-079638a955ae5eea, strip of plant-based burgers3-9c36085b2a6e3cec, four last strips of plant-based burger-315383ee03035bd2) to the pan (large frying pan-6988e61572bf0e6e).\n2. I found the moment in question at 25:52. The user adds the cut burgers (one strip of plant-based burger-09ea87a42b1e082b, strip of plant-based burger2-079638a955ae5eea, strip of plant-based burgers3-9c36085b2a6e3cec, four last strips of plant-based burger-315383ee03035bd2) to the pan (large frying pan-6988e61572bf0e6e) quickly at that time.\n3. Then, I looked for the cutting board (chopping board-116f0066f17df5bf). I found it at 26:30 to the left of the sink. The user turned to the left quickly at this time, and that is when the cutting board (chopping board-116f0066f17df5bf) came into view.\n4. Imagining the hands of a clock, if the oven represented the user's position as 12 o'clock before they turned, that means the cutting board (chopping board-116f0066f17df5bf) would be approximately at the 8 o'clock position.", "referenced_object_masks": [ "one strip of plant-based burger-09ea87a42b1e082b", "strip of plant-based burger2-079638a955ae5eea", "strip of plant-based burgers3-9c36085b2a6e3cec", "four last strips of plant-based burger-315383ee03035bd2", "chopping board-116f0066f17df5bf", "large frying pan-6988e61572bf0e6e" ] }, { "key": "P03-20240218-190556:374979c4dfde57927661661119df9d47ddab754d", "video_id": "P03-20240218-190556", "question": "If the yellow packaging with bottles inside it on the top shelf of the fridge were a twelve pack instead of a four pack, and if it contained the same number of absent bottles that are currently missing from the four pack, how many bottles would there be?", "answer_choice_0": "9", "answer_choice_1": "11", "answer_choice_2": "3", "answer_choice_3": "8", "answer_choice_4": "7", "answer_id": 1, "answer": "11", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Counting", "Counterfactual" ], "objects_of_interest": [ "recipe card-75931342cb8e90e7", "recipe card2-a931478e0377d548", "bag of potatoes-2094c7856761ecff", "box of mushrooms-d34c859d4fdd04c7", "bowl-eaa20b0ad329270f", "scale-26730edecfc504e5", "chopping board-116f0066f17df5bf", "one potato-b8c21cbc3dfcb948", "another potato-c8ead991588f3249", "third potato-9275b300c513dbaa", "fourth potato-e2ad6ce07bf58571", "fifth potato-2c3eb1393e6bb23c", "sixth potato-38d6166b22ad78b3", "big potato-d1f760f1b2b4fd58", "large bowl-dedd4a9a84e5af46", "large bowl2-b8d7c59dc22726e1", "used v60 coffee filter-75e6c42365e15da1", "cup-63f296c573fe7524", "smaller cup-af1bed8f6d7c0e3b", "potato peeler-379c1ff72ea5901f", "potato7-bf62354ddd692a8c", "potato8-b2c78291e683fa86", "potato9-d2fb164598e2263c", "potato10-66ded1b6c76d0ad7", "potato11-f144cbd6bed62072", "potato12-d2eb08a038025fb0", "large saucepan-aa2d9122dc0ded15", "knife-8cf58492cde33b7f", "potato chunks-f615ebb909ffcf40", "tea towel-b46c32f1499be80c", "plastic colander-4400d2bf8c957e7a", "plant-based burger-c55227d550b2d835", "mushroom1-f97cfbfa6c2750c6", "mushroom2-4f2442a9f0ca0a2e", "mushroom3-d0890334025a816a", "mushroom4-6949b206ccb3d811", "mushroom5-2fee8f74c638d686", "mushroom6-06454cfb785f7732", "box of mushrooms2-41245785705ef10f", "mushroom7-235726fa159c36c9", "mushroom8-49d2cf2e8d84b728", "plastic wrapping-83697ef0d4046828", "mushroom10-a131d4607e0720e8", "orange cloth-22f1fd13931806e3", "bag of onions-0d93a8546a5edeeb", "onion-381959e3b0582059", "half onion-753c79e6ec54a8af", "half onion2-641875f8152a79fc", "cloth-a8e98474cb456d87", "wooden spoon-43e48cdcb1c4655e", "tomato puree tube-ee35a7d1de4001e7", "butter package-ef891def0a8f3b64", "small bowl-c51f838b88d1a4c3", "food waste bin-92d10b34b87b6fa3", "potato peel-f6c4d7af162cba5f", "second batch of peels-365ac50e045900fe", "v60 filter stand-2dc06c7e6ec02b4d", "plastic sleeve-50e8f31d5cd16fe4", "burger packaging2-ed8e0567384a0caa", "burger packaging-ad683c3dd3413267", "stack of small frying pan-7a6c65d1934c50d5", "large frying pan-6988e61572bf0e6e", "pan padding-c89a7e0b03d0211e", "recipe kit box-bf8b96fd9cc309fd", "recipe kit box2-01983ad6eabd69c7", "yellow tab-540164355061d556", "plate-90b91fc4d2b70284", "one strip of plant-based burger-09ea87a42b1e082b", "strip of plant-based burger2-079638a955ae5eea", "strip of plant-based burgers3-9c36085b2a6e3cec", "four last strips of plant-based burger-315383ee03035bd2", "sunflower oil bottle-b2ff5cefc018b27c", "sunflower oil bottle cap-dde4e81960836607", "teaspoon-87d8e02e6e646653", "plastic spatula-829e8a803d099511", "knife2-5389ba9dd0e6ec0d", "sponge-e8ca10bbf9926f86", "slab of batter-d327ec37289dd20b", "sour cream pot-62823ab54aaa6baf", "pink recipe tub-4cb42f6264a765d8", "tomato puree tube cap-86038f3975d839c3", "spoon-31c17545b55b8c73", "foil sleeve of sour cream-1413ebfec624d11d", "plate2-b1be62d80b83f854", "metal colander-e7f16d9bb8f8bb30", "slab of butter2-3a4bf10f5bd1db39", "potato masher-4eb23c6ecef251f2", "green recipe tub-c6eb1b8f8c86989f", "tea towel2-6c26adf3d0c0f78b", "tea towel3-619eab3ed44e2662", "ebook reader-d87491ccb26b19c3", "red glass-267e58100f235154", "green glass-4e481882ef31c08f", "water plastic bottle-ef19eb66b1ab3e0a", "rice cooker bowl-5d43487d17ad8afc", "dishwasher brush-b52601e9a67279d3", "plate3-d503d334731b0c4b", "plate4-0c626a74bc928970", "peeler2-44c2d131e6cf215f", "small bowl2-341c1d019798633d", "small bowl3-c86250647498d9be", "washing up liquid bottle-0535bb06843e8f28", "plastic bag-fc523455845d004a", "bowl4-3376447088c8fb27", "fork-04835a7e627a700e", "fork2-eb4cf8a453098c76", "chair-388a550480f274cd", "chair2-df116c6b9c7bb0b1", "coffee jug-feae76b47e550c46" ], "reasoning": "1. I found the fridge at 00:40, and watched for the door to open.\n2. I then looked for the top shelf once the door opens, and located anything with yellow packaging and bottles .\n3. The packaging is only large enough to contain four items, and only one is gone.\n4. I then imposed an imaginary twelve pack in place of the pack of four, and did the simple subtraction to find that I would have eleven bottles in place of three, in this scenario.", "referenced_object_masks": [] }, { "key": "P03-20240218-190556:732033ef9db9fc325bd3fca54010d03d6eec93dc", "video_id": "P03-20240218-190556", "question": "When I touch the stove hood between 17:08-17:09, where is the onion in relation to me?", "answer_choice_0": "To the right", "answer_choice_1": "Behind", "answer_choice_2": "To the left", "answer_choice_3": "Diagonally across", "answer_choice_4": "In front", "answer_id": 1, "answer": "Behind", "question_type": [ "Action Recognition", "Object Recognition", "Temporal Reasoning", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "recipe card-75931342cb8e90e7", "recipe card2-a931478e0377d548", "bag of potatoes-2094c7856761ecff", "box of mushrooms-d34c859d4fdd04c7", "bowl-eaa20b0ad329270f", "scale-26730edecfc504e5", "chopping board-116f0066f17df5bf", "one potato-b8c21cbc3dfcb948", "another potato-c8ead991588f3249", "third potato-9275b300c513dbaa", "fourth potato-e2ad6ce07bf58571", "fifth potato-2c3eb1393e6bb23c", "sixth potato-38d6166b22ad78b3", "big potato-d1f760f1b2b4fd58", "large bowl-dedd4a9a84e5af46", "large bowl2-b8d7c59dc22726e1", "used v60 coffee filter-75e6c42365e15da1", "cup-63f296c573fe7524", "smaller cup-af1bed8f6d7c0e3b", "potato peeler-379c1ff72ea5901f", "potato7-bf62354ddd692a8c", "potato8-b2c78291e683fa86", "potato9-d2fb164598e2263c", "potato10-66ded1b6c76d0ad7", "potato11-f144cbd6bed62072", "potato12-d2eb08a038025fb0", "large saucepan-aa2d9122dc0ded15", "knife-8cf58492cde33b7f", "potato chunks-f615ebb909ffcf40", "tea towel-b46c32f1499be80c", "plastic colander-4400d2bf8c957e7a", "plant-based burger-c55227d550b2d835", "mushroom1-f97cfbfa6c2750c6", "mushroom2-4f2442a9f0ca0a2e", "mushroom3-d0890334025a816a", "mushroom4-6949b206ccb3d811", "mushroom5-2fee8f74c638d686", "mushroom6-06454cfb785f7732", "box of mushrooms2-41245785705ef10f", "mushroom7-235726fa159c36c9", "mushroom8-49d2cf2e8d84b728", "plastic wrapping-83697ef0d4046828", "mushroom10-a131d4607e0720e8", "orange cloth-22f1fd13931806e3", "bag of onions-0d93a8546a5edeeb", "onion-381959e3b0582059", "half onion-753c79e6ec54a8af", "half onion2-641875f8152a79fc", "cloth-a8e98474cb456d87", "wooden spoon-43e48cdcb1c4655e", "tomato puree tube-ee35a7d1de4001e7", "butter package-ef891def0a8f3b64", "small bowl-c51f838b88d1a4c3", "food waste bin-92d10b34b87b6fa3", "potato peel-f6c4d7af162cba5f", "second batch of peels-365ac50e045900fe", "v60 filter stand-2dc06c7e6ec02b4d", "plastic sleeve-50e8f31d5cd16fe4", "burger packaging2-ed8e0567384a0caa", "burger packaging-ad683c3dd3413267", "stack of small frying pan-7a6c65d1934c50d5", "large frying pan-6988e61572bf0e6e", "pan padding-c89a7e0b03d0211e", "recipe kit box-bf8b96fd9cc309fd", "recipe kit box2-01983ad6eabd69c7", "yellow tab-540164355061d556", "plate-90b91fc4d2b70284", "one strip of plant-based burger-09ea87a42b1e082b", "strip of plant-based burger2-079638a955ae5eea", "strip of plant-based burgers3-9c36085b2a6e3cec", "four last strips of plant-based burger-315383ee03035bd2", "sunflower oil bottle-b2ff5cefc018b27c", "sunflower oil bottle cap-dde4e81960836607", "teaspoon-87d8e02e6e646653", "plastic spatula-829e8a803d099511", "knife2-5389ba9dd0e6ec0d", "sponge-e8ca10bbf9926f86", "slab of batter-d327ec37289dd20b", "sour cream pot-62823ab54aaa6baf", "pink recipe tub-4cb42f6264a765d8", "tomato puree tube cap-86038f3975d839c3", "spoon-31c17545b55b8c73", "foil sleeve of sour cream-1413ebfec624d11d", "plate2-b1be62d80b83f854", "metal colander-e7f16d9bb8f8bb30", "slab of butter2-3a4bf10f5bd1db39", "potato masher-4eb23c6ecef251f2", "green recipe tub-c6eb1b8f8c86989f", "tea towel2-6c26adf3d0c0f78b", "tea towel3-619eab3ed44e2662", "ebook reader-d87491ccb26b19c3", "red glass-267e58100f235154", "green glass-4e481882ef31c08f", "water plastic bottle-ef19eb66b1ab3e0a", "rice cooker bowl-5d43487d17ad8afc", "dishwasher brush-b52601e9a67279d3", "plate3-d503d334731b0c4b", "plate4-0c626a74bc928970", "peeler2-44c2d131e6cf215f", "small bowl2-341c1d019798633d", "small bowl3-c86250647498d9be", "washing up liquid bottle-0535bb06843e8f28", "plastic bag-fc523455845d004a", "bowl4-3376447088c8fb27", "fork-04835a7e627a700e", "fork2-eb4cf8a453098c76", "chair-388a550480f274cd", "chair2-df116c6b9c7bb0b1", "coffee jug-feae76b47e550c46" ], "reasoning": "1. First, I looked for the user to retrieve onion and saw them grab an onion (onion-381959e3b0582059) at 16:28. \n2. Next, I saw the user cut the onion into two halves (half onion-753c79e6ec54a8af, half onion2-641875f8152a79fc) at 16:35. \n3. At 17:03, I saw the user turn from the counter with the onion halves (half onion-753c79e6ec54a8af, half onion2-641875f8152a79fc) towards the stove and noticed that the stove is across from the onion (half onion-753c79e6ec54a8af, half onion2-641875f8152a79fc). \n4. At 17:08, I saw the user reach up to the stove hood and twist a knob. I noticed that at this moment the user has their back to the onion (half onion-753c79e6ec54a8af, half onion2-641875f8152a79fc) as they are facing the stove. \n5. Reasoning about the scene, I concluded that when the user touches the stove hood between 17:08-17:09, the onion (half onion-753c79e6ec54a8af, half onion2-641875f8152a79fc) is directly behind them.", "referenced_object_masks": [ "onion-381959e3b0582059", "half onion-753c79e6ec54a8af", "half onion2-641875f8152a79fc" ] }, { "key": "P03-20240218-190556:9c8c92b27aa40a7b7304b08216132fcc00ce5b28", "video_id": "P03-20240218-190556", "question": "When I put the potato masher in the sink, where is the object I used to scrape potato off it in relation to me?", "answer_choice_0": "2 o'clock", "answer_choice_1": "10 o'clock", "answer_choice_2": "8 o'clock", "answer_choice_3": "6 o'clock", "answer_choice_4": "4 o'clock", "answer_id": 2, "answer": "8 o'clock", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Object Recognition" ], "objects_of_interest": [ "recipe card-75931342cb8e90e7", "recipe card2-a931478e0377d548", "bag of potatoes-2094c7856761ecff", "box of mushrooms-d34c859d4fdd04c7", "bowl-eaa20b0ad329270f", "scale-26730edecfc504e5", "chopping board-116f0066f17df5bf", "one potato-b8c21cbc3dfcb948", "another potato-c8ead991588f3249", "third potato-9275b300c513dbaa", "fourth potato-e2ad6ce07bf58571", "fifth potato-2c3eb1393e6bb23c", "sixth potato-38d6166b22ad78b3", "big potato-d1f760f1b2b4fd58", "large bowl-dedd4a9a84e5af46", "large bowl2-b8d7c59dc22726e1", "used v60 coffee filter-75e6c42365e15da1", "cup-63f296c573fe7524", "smaller cup-af1bed8f6d7c0e3b", "potato peeler-379c1ff72ea5901f", "potato7-bf62354ddd692a8c", "potato8-b2c78291e683fa86", "potato9-d2fb164598e2263c", "potato10-66ded1b6c76d0ad7", "potato11-f144cbd6bed62072", "potato12-d2eb08a038025fb0", "large saucepan-aa2d9122dc0ded15", "knife-8cf58492cde33b7f", "potato chunks-f615ebb909ffcf40", "tea towel-b46c32f1499be80c", "plastic colander-4400d2bf8c957e7a", "plant-based burger-c55227d550b2d835", "mushroom1-f97cfbfa6c2750c6", "mushroom2-4f2442a9f0ca0a2e", "mushroom3-d0890334025a816a", "mushroom4-6949b206ccb3d811", "mushroom5-2fee8f74c638d686", "mushroom6-06454cfb785f7732", "box of mushrooms2-41245785705ef10f", "mushroom7-235726fa159c36c9", "mushroom8-49d2cf2e8d84b728", "plastic wrapping-83697ef0d4046828", "mushroom10-a131d4607e0720e8", "orange cloth-22f1fd13931806e3", "bag of onions-0d93a8546a5edeeb", "onion-381959e3b0582059", "half onion-753c79e6ec54a8af", "half onion2-641875f8152a79fc", "cloth-a8e98474cb456d87", "wooden spoon-43e48cdcb1c4655e", "tomato puree tube-ee35a7d1de4001e7", "butter package-ef891def0a8f3b64", "small bowl-c51f838b88d1a4c3", "food waste bin-92d10b34b87b6fa3", "potato peel-f6c4d7af162cba5f", "second batch of peels-365ac50e045900fe", "v60 filter stand-2dc06c7e6ec02b4d", "plastic sleeve-50e8f31d5cd16fe4", "burger packaging2-ed8e0567384a0caa", "burger packaging-ad683c3dd3413267", "stack of small frying pan-7a6c65d1934c50d5", "large frying pan-6988e61572bf0e6e", "pan padding-c89a7e0b03d0211e", "recipe kit box-bf8b96fd9cc309fd", "recipe kit box2-01983ad6eabd69c7", "yellow tab-540164355061d556", "plate-90b91fc4d2b70284", "one strip of plant-based burger-09ea87a42b1e082b", "strip of plant-based burger2-079638a955ae5eea", "strip of plant-based burgers3-9c36085b2a6e3cec", "four last strips of plant-based burger-315383ee03035bd2", "sunflower oil bottle-b2ff5cefc018b27c", "sunflower oil bottle cap-dde4e81960836607", "teaspoon-87d8e02e6e646653", "plastic spatula-829e8a803d099511", "knife2-5389ba9dd0e6ec0d", "sponge-e8ca10bbf9926f86", "slab of batter-d327ec37289dd20b", "sour cream pot-62823ab54aaa6baf", "pink recipe tub-4cb42f6264a765d8", "tomato puree tube cap-86038f3975d839c3", "spoon-31c17545b55b8c73", "foil sleeve of sour cream-1413ebfec624d11d", "plate2-b1be62d80b83f854", "metal colander-e7f16d9bb8f8bb30", "slab of butter2-3a4bf10f5bd1db39", "potato masher-4eb23c6ecef251f2", "green recipe tub-c6eb1b8f8c86989f", "tea towel2-6c26adf3d0c0f78b", "tea towel3-619eab3ed44e2662", "ebook reader-d87491ccb26b19c3", "red glass-267e58100f235154", "green glass-4e481882ef31c08f", "water plastic bottle-ef19eb66b1ab3e0a", "rice cooker bowl-5d43487d17ad8afc", "dishwasher brush-b52601e9a67279d3", "plate3-d503d334731b0c4b", "plate4-0c626a74bc928970", "peeler2-44c2d131e6cf215f", "small bowl2-341c1d019798633d", "small bowl3-c86250647498d9be", "washing up liquid bottle-0535bb06843e8f28", "plastic bag-fc523455845d004a", "bowl4-3376447088c8fb27", "fork-04835a7e627a700e", "fork2-eb4cf8a453098c76", "chair-388a550480f274cd", "chair2-df116c6b9c7bb0b1", "coffee jug-feae76b47e550c46" ], "reasoning": "1. I started by looking for the moment when the user put the potato masher (potato masher-4eb23c6ecef251f2) in the sink, and found that this occurs at 45:50. \n2. Next, I went back and looked for the user to scrape potato off the potato masher (potato masher-4eb23c6ecef251f2) and found that they use a wooden spoon (wooden spoon-43e48cdcb1c4655e) to do this at 45:30-45:48. I saw the user set the wooden spoon (wooden spoon-43e48cdcb1c4655e) in a bowl on the counter at 45:48 and noticed that the counter is to the left of the sink. \n3. Finally, I reasoned about the scene and determined that when the user puts the potato masher (potato masher-4eb23c6ecef251f2) into the sink, the wooden spoon (wooden spoon-43e48cdcb1c4655e) is at their 8 o'clock.", "referenced_object_masks": [ "potato masher-4eb23c6ecef251f2", "wooden spoon-43e48cdcb1c4655e" ] }, { "key": "P03-20240218-190556:a7abc43f3de0000264249151b3fb15f0e6a485f6", "video_id": "P03-20240218-190556", "question": "How many times do I shake one, or both of my hands before drying them on the tea towel at 09:13?", "answer_choice_0": "5", "answer_choice_1": "0", "answer_choice_2": "2", "answer_choice_3": "1", "answer_choice_4": "4", "answer_id": 4, "answer": "4", "question_type": [ "Action Recognition", "Counting", "Object Recognition" ], "objects_of_interest": [ "recipe card-75931342cb8e90e7", "recipe card2-a931478e0377d548", "bag of potatoes-2094c7856761ecff", "box of mushrooms-d34c859d4fdd04c7", "bowl-eaa20b0ad329270f", "scale-26730edecfc504e5", "chopping board-116f0066f17df5bf", "one potato-b8c21cbc3dfcb948", "another potato-c8ead991588f3249", "third potato-9275b300c513dbaa", "fourth potato-e2ad6ce07bf58571", "fifth potato-2c3eb1393e6bb23c", "sixth potato-38d6166b22ad78b3", "big potato-d1f760f1b2b4fd58", "large bowl-dedd4a9a84e5af46", "large bowl2-b8d7c59dc22726e1", "used v60 coffee filter-75e6c42365e15da1", "cup-63f296c573fe7524", "smaller cup-af1bed8f6d7c0e3b", "potato peeler-379c1ff72ea5901f", "potato7-bf62354ddd692a8c", "potato8-b2c78291e683fa86", "potato9-d2fb164598e2263c", "potato10-66ded1b6c76d0ad7", "potato11-f144cbd6bed62072", "potato12-d2eb08a038025fb0", "large saucepan-aa2d9122dc0ded15", "knife-8cf58492cde33b7f", "potato chunks-f615ebb909ffcf40", "tea towel-b46c32f1499be80c", "plastic colander-4400d2bf8c957e7a", "plant-based burger-c55227d550b2d835", "mushroom1-f97cfbfa6c2750c6", "mushroom2-4f2442a9f0ca0a2e", "mushroom3-d0890334025a816a", "mushroom4-6949b206ccb3d811", "mushroom5-2fee8f74c638d686", "mushroom6-06454cfb785f7732", "box of mushrooms2-41245785705ef10f", "mushroom7-235726fa159c36c9", "mushroom8-49d2cf2e8d84b728", "plastic wrapping-83697ef0d4046828", "mushroom10-a131d4607e0720e8", "orange cloth-22f1fd13931806e3", "bag of onions-0d93a8546a5edeeb", "onion-381959e3b0582059", "half onion-753c79e6ec54a8af", "half onion2-641875f8152a79fc", "cloth-a8e98474cb456d87", "wooden spoon-43e48cdcb1c4655e", "tomato puree tube-ee35a7d1de4001e7", "butter package-ef891def0a8f3b64", "small bowl-c51f838b88d1a4c3", "food waste bin-92d10b34b87b6fa3", "potato peel-f6c4d7af162cba5f", "second batch of peels-365ac50e045900fe", "v60 filter stand-2dc06c7e6ec02b4d", "plastic sleeve-50e8f31d5cd16fe4", "burger packaging2-ed8e0567384a0caa", "burger packaging-ad683c3dd3413267", "stack of small frying pan-7a6c65d1934c50d5", "large frying pan-6988e61572bf0e6e", "pan padding-c89a7e0b03d0211e", "recipe kit box-bf8b96fd9cc309fd", "recipe kit box2-01983ad6eabd69c7", "yellow tab-540164355061d556", "plate-90b91fc4d2b70284", "one strip of plant-based burger-09ea87a42b1e082b", "strip of plant-based burger2-079638a955ae5eea", "strip of plant-based burgers3-9c36085b2a6e3cec", "four last strips of plant-based burger-315383ee03035bd2", "sunflower oil bottle-b2ff5cefc018b27c", "sunflower oil bottle cap-dde4e81960836607", "teaspoon-87d8e02e6e646653", "plastic spatula-829e8a803d099511", "knife2-5389ba9dd0e6ec0d", "sponge-e8ca10bbf9926f86", "slab of batter-d327ec37289dd20b", "sour cream pot-62823ab54aaa6baf", "pink recipe tub-4cb42f6264a765d8", "tomato puree tube cap-86038f3975d839c3", "spoon-31c17545b55b8c73", "foil sleeve of sour cream-1413ebfec624d11d", "plate2-b1be62d80b83f854", "metal colander-e7f16d9bb8f8bb30", "slab of butter2-3a4bf10f5bd1db39", "potato masher-4eb23c6ecef251f2", "green recipe tub-c6eb1b8f8c86989f", "tea towel2-6c26adf3d0c0f78b", "tea towel3-619eab3ed44e2662", "ebook reader-d87491ccb26b19c3", "red glass-267e58100f235154", "green glass-4e481882ef31c08f", "water plastic bottle-ef19eb66b1ab3e0a", "rice cooker bowl-5d43487d17ad8afc", "dishwasher brush-b52601e9a67279d3", "plate3-d503d334731b0c4b", "plate4-0c626a74bc928970", "peeler2-44c2d131e6cf215f", "small bowl2-341c1d019798633d", "small bowl3-c86250647498d9be", "washing up liquid bottle-0535bb06843e8f28", "plastic bag-fc523455845d004a", "bowl4-3376447088c8fb27", "fork-04835a7e627a700e", "fork2-eb4cf8a453098c76", "chair-388a550480f274cd", "chair2-df116c6b9c7bb0b1", "coffee jug-feae76b47e550c46" ], "reasoning": "1. I identified when the user begins to wash their hands at 09:06.\n2. After washing their hands for approximately three seconds, the user begins to shake their hands dry.\n3. The user shakes both hands three times, then shakes their left hand one additional time between 09:09-09:11 before reaching for a tea towel (tea towel2-6c26adf3d0c0f78b) at 09:13.", "referenced_object_masks": [ "tea towel2-6c26adf3d0c0f78b" ] }, { "key": "P03-20240218-190556:ce3821ebb8b122cb9b3cda568fd1fa4f81cc14d7", "video_id": "P03-20240218-190556", "question": "What object is to the immediate right of the 6th item that I placed in the dishwasher?", "answer_choice_0": "A plate", "answer_choice_1": "A mug", "answer_choice_2": "A bowl", "answer_choice_3": "A pot", "answer_choice_4": "A plastic container", "answer_id": 2, "answer": "A bowl", "question_type": [ "Action Recognition", "Counting", "Object Recognition", "Situational Awareness", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "recipe card-75931342cb8e90e7", "recipe card2-a931478e0377d548", "bag of potatoes-2094c7856761ecff", "box of mushrooms-d34c859d4fdd04c7", "bowl-eaa20b0ad329270f", "scale-26730edecfc504e5", "chopping board-116f0066f17df5bf", "one potato-b8c21cbc3dfcb948", "another potato-c8ead991588f3249", "third potato-9275b300c513dbaa", "fourth potato-e2ad6ce07bf58571", "fifth potato-2c3eb1393e6bb23c", "sixth potato-38d6166b22ad78b3", "big potato-d1f760f1b2b4fd58", "large bowl-dedd4a9a84e5af46", "large bowl2-b8d7c59dc22726e1", "used v60 coffee filter-75e6c42365e15da1", "cup-63f296c573fe7524", "smaller cup-af1bed8f6d7c0e3b", "potato peeler-379c1ff72ea5901f", "potato7-bf62354ddd692a8c", "potato8-b2c78291e683fa86", "potato9-d2fb164598e2263c", "potato10-66ded1b6c76d0ad7", "potato11-f144cbd6bed62072", "potato12-d2eb08a038025fb0", "large saucepan-aa2d9122dc0ded15", "knife-8cf58492cde33b7f", "potato chunks-f615ebb909ffcf40", "tea towel-b46c32f1499be80c", "plastic colander-4400d2bf8c957e7a", "plant-based burger-c55227d550b2d835", "mushroom1-f97cfbfa6c2750c6", "mushroom2-4f2442a9f0ca0a2e", "mushroom3-d0890334025a816a", "mushroom4-6949b206ccb3d811", "mushroom5-2fee8f74c638d686", "mushroom6-06454cfb785f7732", "box of mushrooms2-41245785705ef10f", "mushroom7-235726fa159c36c9", "mushroom8-49d2cf2e8d84b728", "plastic wrapping-83697ef0d4046828", "mushroom10-a131d4607e0720e8", "orange cloth-22f1fd13931806e3", "bag of onions-0d93a8546a5edeeb", "onion-381959e3b0582059", "half onion-753c79e6ec54a8af", "half onion2-641875f8152a79fc", "cloth-a8e98474cb456d87", "wooden spoon-43e48cdcb1c4655e", "tomato puree tube-ee35a7d1de4001e7", "butter package-ef891def0a8f3b64", "small bowl-c51f838b88d1a4c3", "food waste bin-92d10b34b87b6fa3", "potato peel-f6c4d7af162cba5f", "second batch of peels-365ac50e045900fe", "v60 filter stand-2dc06c7e6ec02b4d", "plastic sleeve-50e8f31d5cd16fe4", "burger packaging2-ed8e0567384a0caa", "burger packaging-ad683c3dd3413267", "stack of small frying pan-7a6c65d1934c50d5", "large frying pan-6988e61572bf0e6e", "pan padding-c89a7e0b03d0211e", "recipe kit box-bf8b96fd9cc309fd", "recipe kit box2-01983ad6eabd69c7", "yellow tab-540164355061d556", "plate-90b91fc4d2b70284", "one strip of plant-based burger-09ea87a42b1e082b", "strip of plant-based burger2-079638a955ae5eea", "strip of plant-based burgers3-9c36085b2a6e3cec", "four last strips of plant-based burger-315383ee03035bd2", "sunflower oil bottle-b2ff5cefc018b27c", "sunflower oil bottle cap-dde4e81960836607", "teaspoon-87d8e02e6e646653", "plastic spatula-829e8a803d099511", "knife2-5389ba9dd0e6ec0d", "sponge-e8ca10bbf9926f86", "slab of batter-d327ec37289dd20b", "sour cream pot-62823ab54aaa6baf", "pink recipe tub-4cb42f6264a765d8", "tomato puree tube cap-86038f3975d839c3", "spoon-31c17545b55b8c73", "foil sleeve of sour cream-1413ebfec624d11d", "plate2-b1be62d80b83f854", "metal colander-e7f16d9bb8f8bb30", "slab of butter2-3a4bf10f5bd1db39", "potato masher-4eb23c6ecef251f2", "green recipe tub-c6eb1b8f8c86989f", "tea towel2-6c26adf3d0c0f78b", "tea towel3-619eab3ed44e2662", "ebook reader-d87491ccb26b19c3", "red glass-267e58100f235154", "green glass-4e481882ef31c08f", "water plastic bottle-ef19eb66b1ab3e0a", "rice cooker bowl-5d43487d17ad8afc", "dishwasher brush-b52601e9a67279d3", "plate3-d503d334731b0c4b", "plate4-0c626a74bc928970", "peeler2-44c2d131e6cf215f", "small bowl2-341c1d019798633d", "small bowl3-c86250647498d9be", "washing up liquid bottle-0535bb06843e8f28", "plastic bag-fc523455845d004a", "bowl4-3376447088c8fb27", "fork-04835a7e627a700e", "fork2-eb4cf8a453098c76", "chair-388a550480f274cd", "chair2-df116c6b9c7bb0b1", "coffee jug-feae76b47e550c46" ], "reasoning": "1. I watched the video to determine when the user placed the 6th item into the dishwasher . I first noticed that the user opened the dishwasher at 48:20.\n2. The 1st item the user placed into the dishwasher was the knife (knife-8cf58492cde33b7f) at 48:33.\n3. The 2nd item the user placed into the dishwasher was the cutting board (chopping board-116f0066f17df5bf) at 48:45.\n4. The 3rd and 4th items the user placed into the dishwasher were the spoon (spoon-31c17545b55b8c73) and the teaspoon (teaspoon-87d8e02e6e646653) at 49:07.\n5. The 5th item the user placed into the dishwasher was the peeler (potato peeler-379c1ff72ea5901f) at 49:12.\n6. The 6th item the user placed into the dishwasher was the masher (potato masher-4eb23c6ecef251f2) at 49:34.\n7. Since the masher (potato masher-4eb23c6ecef251f2) was the 6th item, I looked to see what was to its immediate right. I found a small white bowl .", "referenced_object_masks": [ "potato masher-4eb23c6ecef251f2", "potato peeler-379c1ff72ea5901f", "knife-8cf58492cde33b7f", "spoon-31c17545b55b8c73", "teaspoon-87d8e02e6e646653", "chopping board-116f0066f17df5bf" ] }, { "key": "P03-20240218-190556:d26a3bbed35fc6746ad026f87f7e309a1de4e19b", "video_id": "P03-20240218-190556", "question": "What takes the place of the sour cream pot in the fridge?", "answer_choice_0": "A package of burgers", "answer_choice_1": "A butter package", "answer_choice_2": "A tomato puree tube", "answer_choice_3": "A jar of jelly", "answer_choice_4": "A box of mushrooms", "answer_id": 2, "answer": "A tomato puree tube", "question_type": [ "Object Recognition", "State Changes", "Situational Awareness", "Action Recognition", "Event Occurence" ], "objects_of_interest": [ "recipe card-75931342cb8e90e7", "recipe card2-a931478e0377d548", "bag of potatoes-2094c7856761ecff", "box of mushrooms-d34c859d4fdd04c7", "bowl-eaa20b0ad329270f", "scale-26730edecfc504e5", "chopping board-116f0066f17df5bf", "one potato-b8c21cbc3dfcb948", "another potato-c8ead991588f3249", "third potato-9275b300c513dbaa", "fourth potato-e2ad6ce07bf58571", "fifth potato-2c3eb1393e6bb23c", "sixth potato-38d6166b22ad78b3", "big potato-d1f760f1b2b4fd58", "large bowl-dedd4a9a84e5af46", "large bowl2-b8d7c59dc22726e1", "used v60 coffee filter-75e6c42365e15da1", "cup-63f296c573fe7524", "smaller cup-af1bed8f6d7c0e3b", "potato peeler-379c1ff72ea5901f", "potato7-bf62354ddd692a8c", "potato8-b2c78291e683fa86", "potato9-d2fb164598e2263c", "potato10-66ded1b6c76d0ad7", "potato11-f144cbd6bed62072", "potato12-d2eb08a038025fb0", "large saucepan-aa2d9122dc0ded15", "knife-8cf58492cde33b7f", "potato chunks-f615ebb909ffcf40", "tea towel-b46c32f1499be80c", "plastic colander-4400d2bf8c957e7a", "plant-based burger-c55227d550b2d835", "mushroom1-f97cfbfa6c2750c6", "mushroom2-4f2442a9f0ca0a2e", "mushroom3-d0890334025a816a", "mushroom4-6949b206ccb3d811", "mushroom5-2fee8f74c638d686", "mushroom6-06454cfb785f7732", "box of mushrooms2-41245785705ef10f", "mushroom7-235726fa159c36c9", "mushroom8-49d2cf2e8d84b728", "plastic wrapping-83697ef0d4046828", "mushroom10-a131d4607e0720e8", "orange cloth-22f1fd13931806e3", "bag of onions-0d93a8546a5edeeb", "onion-381959e3b0582059", "half onion-753c79e6ec54a8af", "half onion2-641875f8152a79fc", "cloth-a8e98474cb456d87", "wooden spoon-43e48cdcb1c4655e", "tomato puree tube-ee35a7d1de4001e7", "butter package-ef891def0a8f3b64", "small bowl-c51f838b88d1a4c3", "food waste bin-92d10b34b87b6fa3", "potato peel-f6c4d7af162cba5f", "second batch of peels-365ac50e045900fe", "v60 filter stand-2dc06c7e6ec02b4d", "plastic sleeve-50e8f31d5cd16fe4", "burger packaging2-ed8e0567384a0caa", "burger packaging-ad683c3dd3413267", "stack of small frying pan-7a6c65d1934c50d5", "large frying pan-6988e61572bf0e6e", "pan padding-c89a7e0b03d0211e", "recipe kit box-bf8b96fd9cc309fd", "recipe kit box2-01983ad6eabd69c7", "yellow tab-540164355061d556", "plate-90b91fc4d2b70284", "one strip of plant-based burger-09ea87a42b1e082b", "strip of plant-based burger2-079638a955ae5eea", "strip of plant-based burgers3-9c36085b2a6e3cec", "four last strips of plant-based burger-315383ee03035bd2", "sunflower oil bottle-b2ff5cefc018b27c", "sunflower oil bottle cap-dde4e81960836607", "teaspoon-87d8e02e6e646653", "plastic spatula-829e8a803d099511", "knife2-5389ba9dd0e6ec0d", "sponge-e8ca10bbf9926f86", "slab of batter-d327ec37289dd20b", "sour cream pot-62823ab54aaa6baf", "pink recipe tub-4cb42f6264a765d8", "tomato puree tube cap-86038f3975d839c3", "spoon-31c17545b55b8c73", "foil sleeve of sour cream-1413ebfec624d11d", "plate2-b1be62d80b83f854", "metal colander-e7f16d9bb8f8bb30", "slab of butter2-3a4bf10f5bd1db39", "potato masher-4eb23c6ecef251f2", "green recipe tub-c6eb1b8f8c86989f", "tea towel2-6c26adf3d0c0f78b", "tea towel3-619eab3ed44e2662", "ebook reader-d87491ccb26b19c3", "red glass-267e58100f235154", "green glass-4e481882ef31c08f", "water plastic bottle-ef19eb66b1ab3e0a", "rice cooker bowl-5d43487d17ad8afc", "dishwasher brush-b52601e9a67279d3", "plate3-d503d334731b0c4b", "plate4-0c626a74bc928970", "peeler2-44c2d131e6cf215f", "small bowl2-341c1d019798633d", "small bowl3-c86250647498d9be", "washing up liquid bottle-0535bb06843e8f28", "plastic bag-fc523455845d004a", "bowl4-3376447088c8fb27", "fork-04835a7e627a700e", "fork2-eb4cf8a453098c76", "chair-388a550480f274cd", "chair2-df116c6b9c7bb0b1", "coffee jug-feae76b47e550c46" ], "reasoning": "1. I watched the video in order to see what takes the place of the sour cream pot (sour cream pot-62823ab54aaa6baf) inside the fridge . I found the user first interacted with the sour cream pot (sour cream pot-62823ab54aaa6baf) at 32:15, when they removed it from the fridge .\n2. At 32:15, I noted that the sour cream pot (sour cream pot-62823ab54aaa6baf) was located in the upper right corner of the fridge on the top shelf on the door. It was right next to a jar of jelly .\n3. Then, I watched to figure out what object took the place of the sour cream pot (sour cream pot-62823ab54aaa6baf). From 42:40-42:41, I noticed the user opened the fridge and placed a tomato puree tube (tomato puree tube-ee35a7d1de4001e7) in that exact same location, right next to the jar of jelly . \n4. The user closed the fridge at 42:45 and did not interact again with the tomato puree tube (tomato puree tube-ee35a7d1de4001e7). That makes the tomato puree tube (tomato puree tube-ee35a7d1de4001e7) the final answer.", "referenced_object_masks": [ "tomato puree tube-ee35a7d1de4001e7", "sour cream pot-62823ab54aaa6baf" ] }, { "key": "P03-20240218-190556:dc708f639a4da11beaf0fb50262e0f62ee8d887c", "video_id": "P03-20240218-190556", "question": "After the first two instances of meat slices being placed into the bowl, which of the following slices of meat left on the cutting board is the shortest?", "answer_choice_0": "Second from the left", "answer_choice_1": "Third from the right", "answer_choice_2": "Third from the left", "answer_choice_3": "Second from the right", "answer_choice_4": "The farthest left", "answer_id": 1, "answer": "Third from the right", "question_type": [ "Action Recognition", "Object Recognition", "State Changes", "Temporal Reasoning" ], "objects_of_interest": [ "recipe card-75931342cb8e90e7", "recipe card2-a931478e0377d548", "bag of potatoes-2094c7856761ecff", "box of mushrooms-d34c859d4fdd04c7", "bowl-eaa20b0ad329270f", "scale-26730edecfc504e5", "chopping board-116f0066f17df5bf", "one potato-b8c21cbc3dfcb948", "another potato-c8ead991588f3249", "third potato-9275b300c513dbaa", "fourth potato-e2ad6ce07bf58571", "fifth potato-2c3eb1393e6bb23c", "sixth potato-38d6166b22ad78b3", "big potato-d1f760f1b2b4fd58", "large bowl-dedd4a9a84e5af46", "large bowl2-b8d7c59dc22726e1", "used v60 coffee filter-75e6c42365e15da1", "cup-63f296c573fe7524", "smaller cup-af1bed8f6d7c0e3b", "potato peeler-379c1ff72ea5901f", "potato7-bf62354ddd692a8c", "potato8-b2c78291e683fa86", "potato9-d2fb164598e2263c", "potato10-66ded1b6c76d0ad7", "potato11-f144cbd6bed62072", "potato12-d2eb08a038025fb0", "large saucepan-aa2d9122dc0ded15", "knife-8cf58492cde33b7f", "potato chunks-f615ebb909ffcf40", "tea towel-b46c32f1499be80c", "plastic colander-4400d2bf8c957e7a", "plant-based burger-c55227d550b2d835", "mushroom1-f97cfbfa6c2750c6", "mushroom2-4f2442a9f0ca0a2e", "mushroom3-d0890334025a816a", "mushroom4-6949b206ccb3d811", "mushroom5-2fee8f74c638d686", "mushroom6-06454cfb785f7732", "box of mushrooms2-41245785705ef10f", "mushroom7-235726fa159c36c9", "mushroom8-49d2cf2e8d84b728", "plastic wrapping-83697ef0d4046828", "mushroom10-a131d4607e0720e8", "orange cloth-22f1fd13931806e3", "bag of onions-0d93a8546a5edeeb", "onion-381959e3b0582059", "half onion-753c79e6ec54a8af", "half onion2-641875f8152a79fc", "cloth-a8e98474cb456d87", "wooden spoon-43e48cdcb1c4655e", "tomato puree tube-ee35a7d1de4001e7", "butter package-ef891def0a8f3b64", "small bowl-c51f838b88d1a4c3", "food waste bin-92d10b34b87b6fa3", "potato peel-f6c4d7af162cba5f", "second batch of peels-365ac50e045900fe", "v60 filter stand-2dc06c7e6ec02b4d", "plastic sleeve-50e8f31d5cd16fe4", "burger packaging2-ed8e0567384a0caa", "burger packaging-ad683c3dd3413267", "stack of small frying pan-7a6c65d1934c50d5", "large frying pan-6988e61572bf0e6e", "pan padding-c89a7e0b03d0211e", "recipe kit box-bf8b96fd9cc309fd", "recipe kit box2-01983ad6eabd69c7", "yellow tab-540164355061d556", "plate-90b91fc4d2b70284", "one strip of plant-based burger-09ea87a42b1e082b", "strip of plant-based burger2-079638a955ae5eea", "strip of plant-based burgers3-9c36085b2a6e3cec", "four last strips of plant-based burger-315383ee03035bd2", "sunflower oil bottle-b2ff5cefc018b27c", "sunflower oil bottle cap-dde4e81960836607", "teaspoon-87d8e02e6e646653", "plastic spatula-829e8a803d099511", "knife2-5389ba9dd0e6ec0d", "sponge-e8ca10bbf9926f86", "slab of batter-d327ec37289dd20b", "sour cream pot-62823ab54aaa6baf", "pink recipe tub-4cb42f6264a765d8", "tomato puree tube cap-86038f3975d839c3", "spoon-31c17545b55b8c73", "foil sleeve of sour cream-1413ebfec624d11d", "plate2-b1be62d80b83f854", "metal colander-e7f16d9bb8f8bb30", "slab of butter2-3a4bf10f5bd1db39", "potato masher-4eb23c6ecef251f2", "green recipe tub-c6eb1b8f8c86989f", "tea towel2-6c26adf3d0c0f78b", "tea towel3-619eab3ed44e2662", "ebook reader-d87491ccb26b19c3", "red glass-267e58100f235154", "green glass-4e481882ef31c08f", "water plastic bottle-ef19eb66b1ab3e0a", "rice cooker bowl-5d43487d17ad8afc", "dishwasher brush-b52601e9a67279d3", "plate3-d503d334731b0c4b", "plate4-0c626a74bc928970", "peeler2-44c2d131e6cf215f", "small bowl2-341c1d019798633d", "small bowl3-c86250647498d9be", "washing up liquid bottle-0535bb06843e8f28", "plastic bag-fc523455845d004a", "bowl4-3376447088c8fb27", "fork-04835a7e627a700e", "fork2-eb4cf8a453098c76", "chair-388a550480f274cd", "chair2-df116c6b9c7bb0b1", "coffee jug-feae76b47e550c46" ], "reasoning": "1. I start by looking for the point in which the full range of burger strips is cut on the cutting board (chopping board-116f0066f17df5bf), and found it at 23:43.\n2. I then watch as the user takes handfuls of the burger strips and places them in the bowl (plate-90b91fc4d2b70284) - I observe the user place the first handful of burger strips, in this case one strip (one strip of plant-based burger-09ea87a42b1e082b) into the bowl (plate-90b91fc4d2b70284) at 23:49-23:50, and the second handful (strip of plant-based burger2-079638a955ae5eea) into the bowl (plate-90b91fc4d2b70284) from 23:50-23:52. \n3. I take note that there are 7 strips of meat (strip of plant-based burgers3-9c36085b2a6e3cec, four last strips of plant-based burger-315383ee03035bd2) left on the cutting board (chopping board-116f0066f17df5bf) at 23:51, and observe them to see which looks the shortest. I observe at 23:51 that the shortest piece of meat appears to be the one third from the right, and compare this observation to the answer choices, getting the desired answer of B.", "referenced_object_masks": [ "strip of plant-based burgers3-9c36085b2a6e3cec", "four last strips of plant-based burger-315383ee03035bd2", "strip of plant-based burger2-079638a955ae5eea", "chopping board-116f0066f17df5bf", "plate-90b91fc4d2b70284", "one strip of plant-based burger-09ea87a42b1e082b" ] }, { "key": "P03-20240218-190556:ea5cfec212c63e823fbcccfea5152f0bf0a15c4c", "video_id": "P03-20240218-190556", "question": "How long does it take me to reach my 10th mash with the potato masher, counting from the 1st mash to the 10th mash?", "answer_choice_0": "25 seconds", "answer_choice_1": "29 seconds", "answer_choice_2": "22 seconds", "answer_choice_3": "17 seconds", "answer_choice_4": "34 seconds", "answer_id": 2, "answer": "22 seconds", "question_type": [ "Action Recognition", "Counting", "Numerical Reasoning", "Temporal Reasoning", "Object Recognition" ], "objects_of_interest": [ "recipe card-75931342cb8e90e7", "recipe card2-a931478e0377d548", "bag of potatoes-2094c7856761ecff", "box of mushrooms-d34c859d4fdd04c7", "bowl-eaa20b0ad329270f", "scale-26730edecfc504e5", "chopping board-116f0066f17df5bf", "one potato-b8c21cbc3dfcb948", "another potato-c8ead991588f3249", "third potato-9275b300c513dbaa", "fourth potato-e2ad6ce07bf58571", "fifth potato-2c3eb1393e6bb23c", "sixth potato-38d6166b22ad78b3", "big potato-d1f760f1b2b4fd58", "large bowl-dedd4a9a84e5af46", "large bowl2-b8d7c59dc22726e1", "used v60 coffee filter-75e6c42365e15da1", "cup-63f296c573fe7524", "smaller cup-af1bed8f6d7c0e3b", "potato peeler-379c1ff72ea5901f", "potato7-bf62354ddd692a8c", "potato8-b2c78291e683fa86", "potato9-d2fb164598e2263c", "potato10-66ded1b6c76d0ad7", "potato11-f144cbd6bed62072", "potato12-d2eb08a038025fb0", "large saucepan-aa2d9122dc0ded15", "knife-8cf58492cde33b7f", "potato chunks-f615ebb909ffcf40", "tea towel-b46c32f1499be80c", "plastic colander-4400d2bf8c957e7a", "plant-based burger-c55227d550b2d835", "mushroom1-f97cfbfa6c2750c6", "mushroom2-4f2442a9f0ca0a2e", "mushroom3-d0890334025a816a", "mushroom4-6949b206ccb3d811", "mushroom5-2fee8f74c638d686", "mushroom6-06454cfb785f7732", "box of mushrooms2-41245785705ef10f", "mushroom7-235726fa159c36c9", "mushroom8-49d2cf2e8d84b728", "plastic wrapping-83697ef0d4046828", "mushroom10-a131d4607e0720e8", "orange cloth-22f1fd13931806e3", "bag of onions-0d93a8546a5edeeb", "onion-381959e3b0582059", "half onion-753c79e6ec54a8af", "half onion2-641875f8152a79fc", "cloth-a8e98474cb456d87", "wooden spoon-43e48cdcb1c4655e", "tomato puree tube-ee35a7d1de4001e7", "butter package-ef891def0a8f3b64", "small bowl-c51f838b88d1a4c3", "food waste bin-92d10b34b87b6fa3", "potato peel-f6c4d7af162cba5f", "second batch of peels-365ac50e045900fe", "v60 filter stand-2dc06c7e6ec02b4d", "plastic sleeve-50e8f31d5cd16fe4", "burger packaging2-ed8e0567384a0caa", "burger packaging-ad683c3dd3413267", "stack of small frying pan-7a6c65d1934c50d5", "large frying pan-6988e61572bf0e6e", "pan padding-c89a7e0b03d0211e", "recipe kit box-bf8b96fd9cc309fd", "recipe kit box2-01983ad6eabd69c7", "yellow tab-540164355061d556", "plate-90b91fc4d2b70284", "one strip of plant-based burger-09ea87a42b1e082b", "strip of plant-based burger2-079638a955ae5eea", "strip of plant-based burgers3-9c36085b2a6e3cec", "four last strips of plant-based burger-315383ee03035bd2", "sunflower oil bottle-b2ff5cefc018b27c", "sunflower oil bottle cap-dde4e81960836607", "teaspoon-87d8e02e6e646653", "plastic spatula-829e8a803d099511", "knife2-5389ba9dd0e6ec0d", "sponge-e8ca10bbf9926f86", "slab of batter-d327ec37289dd20b", "sour cream pot-62823ab54aaa6baf", "pink recipe tub-4cb42f6264a765d8", "tomato puree tube cap-86038f3975d839c3", "spoon-31c17545b55b8c73", "foil sleeve of sour cream-1413ebfec624d11d", "plate2-b1be62d80b83f854", "metal colander-e7f16d9bb8f8bb30", "slab of butter2-3a4bf10f5bd1db39", "potato masher-4eb23c6ecef251f2", "green recipe tub-c6eb1b8f8c86989f", "tea towel2-6c26adf3d0c0f78b", "tea towel3-619eab3ed44e2662", "ebook reader-d87491ccb26b19c3", "red glass-267e58100f235154", "green glass-4e481882ef31c08f", "water plastic bottle-ef19eb66b1ab3e0a", "rice cooker bowl-5d43487d17ad8afc", "dishwasher brush-b52601e9a67279d3", "plate3-d503d334731b0c4b", "plate4-0c626a74bc928970", "peeler2-44c2d131e6cf215f", "small bowl2-341c1d019798633d", "small bowl3-c86250647498d9be", "washing up liquid bottle-0535bb06843e8f28", "plastic bag-fc523455845d004a", "bowl4-3376447088c8fb27", "fork-04835a7e627a700e", "fork2-eb4cf8a453098c76", "chair-388a550480f274cd", "chair2-df116c6b9c7bb0b1", "coffee jug-feae76b47e550c46" ], "reasoning": "1. I watched the video in order to figure out how long it took the user to go from the 1st mash to the 10th mash with the potatoes. I saw the user begin to work with the bowl (bowl-eaa20b0ad329270f) of potatoes (potato chunks-f615ebb909ffcf40) at 43:45.\n2. Then, I saw the user grab a potato masher (potato masher-4eb23c6ecef251f2) at 43:57.\n3. Then, I counted the number of mashes the user completed with the potato masher (potato masher-4eb23c6ecef251f2). Counting up to 10, I saw them mash the potatoes (potato chunks-f615ebb909ffcf40) at the following times: 43:58, 44:00, 44:02, 44:04, 44:09, 44:11, 44:12, 44:16, 44:18, and 44:19. The 10th instance ended at 44:20 as the user pulled the potato masher (potato masher-4eb23c6ecef251f2) from the bowl (bowl-eaa20b0ad329270f), so I counted that as the end of the final instance.\n4. I subtracted 43:58 from 44:20 in order to get a final answer of 22 seconds.", "referenced_object_masks": [ "potato masher-4eb23c6ecef251f2", "bowl-eaa20b0ad329270f", "potato chunks-f615ebb909ffcf40" ] }, { "key": "P03-20240218-202148:27061ccdc87249ae4b5c7a2c9b89770a17ec68fe", "video_id": "P03-20240218-202148", "question": "What is the second unique item that I removed from the dining table, after I finished wiping down the kitchen counters?", "answer_choice_0": "The second yellow tablecloth", "answer_choice_1": "The water pitcher", "answer_choice_2": "The food container", "answer_choice_3": "The first yellow table tablecloth", "answer_choice_4": "The red cup", "answer_id": 3, "answer": "The first yellow table tablecloth", "question_type": [ "Counting", "Temporal Reasoning", "Object Recognition" ], "objects_of_interest": [ "plastic lid-1ad4f0e031583ec9", "sweet's cardboard box-8b85a5c121ffeb5c", "chair-b96a4b62ec67708f", "bowl-ec6245bb483a76bd", "bowl2-65b37a9944ec5fcd", "bowl3-3174eb14aa62bba3", "food container-4a0f7c4c87986741", "plastic divider-fe9778bd17abfbca", "food container lid-df378746da1b3c99", "plate-d346c0670b42ac01", "wooden spoon-7e2f403d5418a7be", "plate2-cf637da0c337fe30", "spoon-08e4df2e032a9992", "fork-e438a2f116194cf6", "fork2-9edc57fcbab5dfa6", "dishwasher brush-8aab966836ceaa19", "metal colander-68225a30d851781e", "saucepan-228e129c57faa3ed", "frying pan-d81c115d8897c156", "spatula-e2554ea76cd17d7c", "plate3-5788bd76576d1aba", "plate4-6a4f8e0d2f166ee0", "plate5-95a6568eecbe8d95", "plate6-f3f1cc579c5dfe0a", "plastic colander-9b637d44c7320fc9", "small bowl-132c80ddea020751", "inner part of plastic colander-1548fa7e46aa453f", "potato masher-e60a90cb2fa480bc", "sponge-b6005169ad6d7947", "knife-ca783bd14caa0758", "cloth-b8b2761f480df219", "dishwasher tablet-a71c608c9a0ea13b", "orange cloth-4be439448637e0a2", "kitchen paper roll-71a58efa2d5440a7", "one tissue-375ebdaa6b88c9d3", "recipe card-bd1ebd1a27285bcf", "cupboard-8656dfe5ae2f7005", "hand wash bottle-31dd0c473bd89df5", "soap-530a33b5da765fc7", "strainer-57361e5359f59b6e", "red glass-55433dba5093f062", "tea towel-7c4b79e876002e0c", "tea towel2-07f6e004ce4b18fa" ], "reasoning": "1. I watched the video for the end of the kitchen counter wipedown, which I observed from about 12:35-12:45, when the user finishes wiping the sink and hangs up the orange cloth (orange cloth-4be439448637e0a2) on the faucet.\n2. I then watched the video to determine when the user returns to the dining table , which they do from 13:00-13:03.\n3. I then counted the unique items that the user removes from the dining table, and the order in which they remove them: the first of these was the food container (food container-4a0f7c4c87986741) with its lid (food container lid-df378746da1b3c99) - this item is first removed from the table at 13:12, placed back on it at 13:18, and then removed again and placed in the fridge at 13:20. Though it is removed from the table twice, it is a unique item and thus only counts once.\n4. At 13:26, I observed the red cup (red glass-55433dba5093f062) being moved across the tabletop: though it is moved across the table, it is not fully removed out and over from the table, so I did not count it as an item removed.\n5. I then observed the second item removed from the table: this was the first yellow tablecloth (tea towel-7c4b79e876002e0c) at 13:30, when it is walked over to the counter, making this the correct answer.", "referenced_object_masks": [ "food container lid-df378746da1b3c99", "tea towel-7c4b79e876002e0c", "food container-4a0f7c4c87986741", "orange cloth-4be439448637e0a2", "red glass-55433dba5093f062" ] }, { "key": "P03-20240218-202148:30dff34a24c3759ada66bec2170a5db169670e67", "video_id": "P03-20240218-202148", "question": "Where is the second item removed from the dishwasher placed on the counter, in relation to the first item removed from the dishwasher and the bowls from the dining table?", "answer_choice_0": "On top of the first dishwasher item, to the left of the bowls", "answer_choice_1": "To the right of the first dishwasher item, and to the left of the bowls", "answer_choice_2": "To the left of both the first dishwasher item and the bowls", "answer_choice_3": "To the right of both the first dishwasher item and the bowls", "answer_choice_4": "To the left of the first dishwasher item, and to the right of the bowls", "answer_id": 1, "answer": "To the right of the first dishwasher item, and to the left of the bowls", "question_type": [ "Spatial Perception", "Counting" ], "objects_of_interest": [ "plastic lid-1ad4f0e031583ec9", "sweet's cardboard box-8b85a5c121ffeb5c", "chair-b96a4b62ec67708f", "bowl-ec6245bb483a76bd", "bowl2-65b37a9944ec5fcd", "bowl3-3174eb14aa62bba3", "food container-4a0f7c4c87986741", "plastic divider-fe9778bd17abfbca", "food container lid-df378746da1b3c99", "plate-d346c0670b42ac01", "wooden spoon-7e2f403d5418a7be", "plate2-cf637da0c337fe30", "spoon-08e4df2e032a9992", "fork-e438a2f116194cf6", "fork2-9edc57fcbab5dfa6", "dishwasher brush-8aab966836ceaa19", "metal colander-68225a30d851781e", "saucepan-228e129c57faa3ed", "frying pan-d81c115d8897c156", "spatula-e2554ea76cd17d7c", "plate3-5788bd76576d1aba", "plate4-6a4f8e0d2f166ee0", "plate5-95a6568eecbe8d95", "plate6-f3f1cc579c5dfe0a", "plastic colander-9b637d44c7320fc9", "small bowl-132c80ddea020751", "inner part of plastic colander-1548fa7e46aa453f", "potato masher-e60a90cb2fa480bc", "sponge-b6005169ad6d7947", "knife-ca783bd14caa0758", "cloth-b8b2761f480df219", "dishwasher tablet-a71c608c9a0ea13b", "orange cloth-4be439448637e0a2", "kitchen paper roll-71a58efa2d5440a7", "one tissue-375ebdaa6b88c9d3", "recipe card-bd1ebd1a27285bcf", "cupboard-8656dfe5ae2f7005", "hand wash bottle-31dd0c473bd89df5", "soap-530a33b5da765fc7", "strainer-57361e5359f59b6e", "red glass-55433dba5093f062", "tea towel-7c4b79e876002e0c", "tea towel2-07f6e004ce4b18fa" ], "reasoning": "1. I watched the video to determine where the bowls from the dining table were placed on the countertop: first, at 00:33, 2 bowls from the table (bowl2-65b37a9944ec5fcd, bowl3-3174eb14aa62bba3) were placed at the right-side corner on the left side of the counter. At 2:24, 1 of the larger bowls (plate-d346c0670b42ac01) is placed to the right of the first 2 bowls, and at 2:41, the other large bowl (plate2-cf637da0c337fe30) is placed to the left of the other 3, but still on the right side of the countertop.\n2. I then watched the video to observe when items were taken out of the dishwasher, and in what order: I watched the first item (metal colander-68225a30d851781e) be taken out at 2:36, and placed on the left corner of the counter, far to the left of the bowls from the dining table.\n3. I then observed and counted the second item, a metal saucepan (saucepan-228e129c57faa3ed) be removed from the dishwasher at 02:40.\n4. At 02:41, I watched the metal saucepan (saucepan-228e129c57faa3ed) be placed on the countertop, and took note of its location: it was planed to the right of the first dishwasher item (metal colander-68225a30d851781e) and to the left of the dining table bowls (bowl2-65b37a9944ec5fcd, bowl3-3174eb14aa62bba3, plate-d346c0670b42ac01, plate2-cf637da0c337fe30), making B the correct answer.", "referenced_object_masks": [ "saucepan-228e129c57faa3ed", "plate2-cf637da0c337fe30", "bowl2-65b37a9944ec5fcd", "bowl3-3174eb14aa62bba3", "plate-d346c0670b42ac01", "metal colander-68225a30d851781e" ] }, { "key": "P03-20240218-202148:941b08f6f7f43583f1912aebf2d85379153f70f2", "video_id": "P03-20240218-202148", "question": "During the span of time I spend wiping down the counter with the orange cloth and spraying it with liquid from the spray bottle, what is the first item I touch with my left hand?", "answer_choice_0": "The papertowels", "answer_choice_1": "The oven handle", "answer_choice_2": "The ceramic candle holder", "answer_choice_3": "The coffee maker", "answer_choice_4": "The soap container", "answer_id": 2, "answer": "The ceramic candle holder", "question_type": [ "Counting", "Event Occurence", "Temporal Reasoning", "Object Recognition" ], "objects_of_interest": [ "plastic lid-1ad4f0e031583ec9", "sweet's cardboard box-8b85a5c121ffeb5c", "chair-b96a4b62ec67708f", "bowl-ec6245bb483a76bd", "bowl2-65b37a9944ec5fcd", "bowl3-3174eb14aa62bba3", "food container-4a0f7c4c87986741", "plastic divider-fe9778bd17abfbca", "food container lid-df378746da1b3c99", "plate-d346c0670b42ac01", "wooden spoon-7e2f403d5418a7be", "plate2-cf637da0c337fe30", "spoon-08e4df2e032a9992", "fork-e438a2f116194cf6", "fork2-9edc57fcbab5dfa6", "dishwasher brush-8aab966836ceaa19", "metal colander-68225a30d851781e", "saucepan-228e129c57faa3ed", "frying pan-d81c115d8897c156", "spatula-e2554ea76cd17d7c", "plate3-5788bd76576d1aba", "plate4-6a4f8e0d2f166ee0", "plate5-95a6568eecbe8d95", "plate6-f3f1cc579c5dfe0a", "plastic colander-9b637d44c7320fc9", "small bowl-132c80ddea020751", "inner part of plastic colander-1548fa7e46aa453f", "potato masher-e60a90cb2fa480bc", "sponge-b6005169ad6d7947", "knife-ca783bd14caa0758", "cloth-b8b2761f480df219", "dishwasher tablet-a71c608c9a0ea13b", "orange cloth-4be439448637e0a2", "kitchen paper roll-71a58efa2d5440a7", "one tissue-375ebdaa6b88c9d3", "recipe card-bd1ebd1a27285bcf", "cupboard-8656dfe5ae2f7005", "hand wash bottle-31dd0c473bd89df5", "soap-530a33b5da765fc7", "strainer-57361e5359f59b6e", "red glass-55433dba5093f062", "tea towel-7c4b79e876002e0c", "tea towel2-07f6e004ce4b18fa" ], "reasoning": "1. I watched the video to determine when the countertop was sprayed down with liquid from the spray bottle (cupboard-8656dfe5ae2f7005). This occurred from 09:42-09:51.\n2. I then paid attention to the counter wipe down that occurred after the user sprayed the counter: this counter wipe down began at 10:06, with the user using the orange cloth (orange cloth-4be439448637e0a2) in their right hand.\n3. During the counter wipedown, I paid attention to what the user touched with their left hand, and the order in which they did so: I noted that the first thing they touched with their left hand during the counter wipedown was the orange cloth itself (orange cloth-4be439448637e0a2), briefly running their fingers on it at 10:32.\n4. I then watched for the second thing that the user would touch with their left hand: this was the ceramic candle holder from 10:33-10:34, which they slid from left to right across the counter. Therefore, the ceramic candle holder is the answer.", "referenced_object_masks": [ "cupboard-8656dfe5ae2f7005", "orange cloth-4be439448637e0a2" ] }, { "key": "P03-20240218-202148:c8da55a06bd2e23619fc0869fb4b877d8f342733", "video_id": "P03-20240218-202148", "question": "If I multiplied the number of food bowls resting on the table at the start of the video with the number of metal utensils in those bowls, what would be the product of this equation?", "answer_choice_0": "15", "answer_choice_1": "8", "answer_choice_2": "9", "answer_choice_3": "16", "answer_choice_4": "20", "answer_id": 0, "answer": "15", "question_type": [ "Numerical Reasoning", "Object Recognition" ], "objects_of_interest": [ "plastic lid-1ad4f0e031583ec9", "sweet's cardboard box-8b85a5c121ffeb5c", "chair-b96a4b62ec67708f", "bowl-ec6245bb483a76bd", "bowl2-65b37a9944ec5fcd", "bowl3-3174eb14aa62bba3", "food container-4a0f7c4c87986741", "plastic divider-fe9778bd17abfbca", "food container lid-df378746da1b3c99", "plate-d346c0670b42ac01", "wooden spoon-7e2f403d5418a7be", "plate2-cf637da0c337fe30", "spoon-08e4df2e032a9992", "fork-e438a2f116194cf6", "fork2-9edc57fcbab5dfa6", "dishwasher brush-8aab966836ceaa19", "metal colander-68225a30d851781e", "saucepan-228e129c57faa3ed", "frying pan-d81c115d8897c156", "spatula-e2554ea76cd17d7c", "plate3-5788bd76576d1aba", "plate4-6a4f8e0d2f166ee0", "plate5-95a6568eecbe8d95", "plate6-f3f1cc579c5dfe0a", "plastic colander-9b637d44c7320fc9", "small bowl-132c80ddea020751", "inner part of plastic colander-1548fa7e46aa453f", "potato masher-e60a90cb2fa480bc", "sponge-b6005169ad6d7947", "knife-ca783bd14caa0758", "cloth-b8b2761f480df219", "dishwasher tablet-a71c608c9a0ea13b", "orange cloth-4be439448637e0a2", "kitchen paper roll-71a58efa2d5440a7", "one tissue-375ebdaa6b88c9d3", "recipe card-bd1ebd1a27285bcf", "cupboard-8656dfe5ae2f7005", "hand wash bottle-31dd0c473bd89df5", "soap-530a33b5da765fc7", "strainer-57361e5359f59b6e", "red glass-55433dba5093f062", "tea towel-7c4b79e876002e0c", "tea towel2-07f6e004ce4b18fa" ], "reasoning": "1. At the start of the video, I count the food bowls on the counter: these are a bowl with black olives (bowl-ec6245bb483a76bd), an empty bowl on the far left (bowl2-65b37a9944ec5fcd), and empty bowl in the lower center (bowl3-3174eb14aa62bba3), a large bowl with potatoes (plate-d346c0670b42ac01), and a large bowl (plate2-cf637da0c337fe30) with meat, for a total of 5 bowls.\n2. I then make note of which bowls have utensils - the bowl with black olives (bowl-ec6245bb483a76bd) has none, the empty bowl (bowl2-65b37a9944ec5fcd) on the far left has a fork (fork2-9edc57fcbab5dfa6), the empty bowl (bowl3-3174eb14aa62bba3) in the lower center has a fork (fork-e438a2f116194cf6), the bowl (plate-d346c0670b42ac01) with potatoes has a wooden spoon (wooden spoon-7e2f403d5418a7be) and the bowl (plate2-cf637da0c337fe30) with meat has a spoon (spoon-08e4df2e032a9992) for a total of 4 utensils.\n3. Of these 4 utensils, I then made note of how many were metal. I counted a total of 3: the 2 forks (fork2-9edc57fcbab5dfa6, fork-e438a2f116194cf6, spoon-08e4df2e032a9992), omitting the wooden spoon (wooden spoon-7e2f403d5418a7be).\n4. I multiplied the total number of bowls, 5, with the total number of metal utensils, 3, to get a product of 15, making 15 the correct answer.", "referenced_object_masks": [ "bowl2-65b37a9944ec5fcd", "bowl3-3174eb14aa62bba3", "fork2-9edc57fcbab5dfa6", "plate2-cf637da0c337fe30", "bowl-ec6245bb483a76bd", "wooden spoon-7e2f403d5418a7be", "plate-d346c0670b42ac01", "fork-e438a2f116194cf6", "spoon-08e4df2e032a9992" ] }, { "key": "P03-20240218-202148:f9cfa3ab4a443655b5b0d6e6f5153552c1db9930", "video_id": "P03-20240218-202148", "question": "From first to last, in which order do I perform the following actions - take the spray bottle out of the cabinet, turn on the stove light, wipe off the recipe card, hang the cloth on its hook?", "answer_choice_0": "Hang the cloth on its hook, take the spray bottle out of the cabinet, turn on the stove light, wipe off the recipe card", "answer_choice_1": "Hang the cloth on its hook, wipe off the recipe card, take the spray bottle out of the cabinet, turn on the stove light", "answer_choice_2": "Hang the cloth on its hook, turn on the stove light, wipe off the recipe card, take the spray bottle out of the cabinet", "answer_choice_3": "Hang the cloth on its hook, wipe off the recipe card, turn on the stove light, take the spray bottle out of the cabinet", "answer_choice_4": "Hang the cloth on its hook, take the spray bottle out of the cabinet, wipe off the recipe card, turn on the stove light", "answer_id": 3, "answer": "Hang the cloth on its hook, wipe off the recipe card, turn on the stove light, take the spray bottle out of the cabinet", "question_type": [ "Temporal Reasoning", "Event Occurence", "Object Recognition" ], "objects_of_interest": [ "plastic lid-1ad4f0e031583ec9", "sweet's cardboard box-8b85a5c121ffeb5c", "chair-b96a4b62ec67708f", "bowl-ec6245bb483a76bd", "bowl2-65b37a9944ec5fcd", "bowl3-3174eb14aa62bba3", "food container-4a0f7c4c87986741", "plastic divider-fe9778bd17abfbca", "food container lid-df378746da1b3c99", "plate-d346c0670b42ac01", "wooden spoon-7e2f403d5418a7be", "plate2-cf637da0c337fe30", "spoon-08e4df2e032a9992", "fork-e438a2f116194cf6", "fork2-9edc57fcbab5dfa6", "dishwasher brush-8aab966836ceaa19", "metal colander-68225a30d851781e", "saucepan-228e129c57faa3ed", "frying pan-d81c115d8897c156", "spatula-e2554ea76cd17d7c", "plate3-5788bd76576d1aba", "plate4-6a4f8e0d2f166ee0", "plate5-95a6568eecbe8d95", "plate6-f3f1cc579c5dfe0a", "plastic colander-9b637d44c7320fc9", "small bowl-132c80ddea020751", "inner part of plastic colander-1548fa7e46aa453f", "potato masher-e60a90cb2fa480bc", "sponge-b6005169ad6d7947", "knife-ca783bd14caa0758", "cloth-b8b2761f480df219", "dishwasher tablet-a71c608c9a0ea13b", "orange cloth-4be439448637e0a2", "kitchen paper roll-71a58efa2d5440a7", "one tissue-375ebdaa6b88c9d3", "recipe card-bd1ebd1a27285bcf", "cupboard-8656dfe5ae2f7005", "hand wash bottle-31dd0c473bd89df5", "soap-530a33b5da765fc7", "strainer-57361e5359f59b6e", "red glass-55433dba5093f062", "tea towel-7c4b79e876002e0c", "tea towel2-07f6e004ce4b18fa" ], "reasoning": "1. I watched the video to determine when the first of these chronological events happens: this occurs from 08:22-08:29, when the cloth (cloth-b8b2761f480df219) is picked up by the user and hung on its hook below the sink.\n2. I then watched for the second of the listed events to occur chronologically: this happened from 09:07-09:13, when the recipe card (recipe card-bd1ebd1a27285bcf) is wiped off with the orange cloth (orange cloth-4be439448637e0a2).\n3. I then watched for the third of these events to occur chronologically: this happened at 09:21, when the user turned on the stovetop light, illuminating the stovetop.\n4. I then observed the final chronological event occur at 09:36, when the user opens the cabinet and removes the spray bottle (cupboard-8656dfe5ae2f7005), making this the fourth and final event.", "referenced_object_masks": [ "cupboard-8656dfe5ae2f7005", "orange cloth-4be439448637e0a2", "cloth-b8b2761f480df219", "recipe card-bd1ebd1a27285bcf" ] }, { "key": "P04-20240413-142619:05f168a169506482ee6bb67e5a2e89ee407cf26c", "video_id": "P04-20240413-142619", "question": "When I bring the bowl to the sink to rinse the chicken, where would I have to look to find the bowl's original location?", "answer_choice_0": "Above and to the left", "answer_choice_1": "Above and to the right", "answer_choice_2": "Below and to the right", "answer_choice_3": "Behind", "answer_choice_4": "Below and to the left", "answer_id": 0, "answer": "Above and to the left", "question_type": [ "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counterfactual", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "measuring cup-24a180dd40d2d5f1", "soap dispenser-eefb4cf7b9db6b17", "black tea towel-dc0c6dd9834d5b91", "box of chicken-ed756fd25589eb34", "cooling rack-4fd9f5ad5b8fb6ca", "chopping board-f696c813f774c077", "herb jar1-e07e73b7e5188d9c", "herb jar2-a192a0897d1b82b4", "spice jar1-95dc26bbfd03ff97", "spice jar2-3f070f24df5ec151", "spice jar3-b6a5cedf2cf5ebcd", "bottle of oil-a6d945305f4c8100", "black jar1-a956ffc59c8a15da", "black jar2-7457a62c58c523e8", "herb jar3-c91abd3e2d1f2844", "condiment jar-3ffec4b711151386", "weighing scales-5d8e61b3f5c17472", "both black jars-bc93b3e14eadbede", "bag of lemons-3046e9fbf1b2446c", "spoon-af9ea4b03e35e603", "knife2-1a331567dfde7af4", "knife-0c07a8f376c06a09", "spoon2-8a765b15984b0e82", "spoon3-98ffa9ff7f44fe0d", "metallic mixing bowl-4759eb889afe2c0e", "plastic bowl-38495d6ed4b2bfb9", "stack of three bowls-1ce6d503e005c036", "bowl1-690c3c19c6730040", "lid of herb jar2-b9ce8575c6884125", "lid of herb jar1-2e31d2e0d94e6f96", "lid of spice jar1-714d8e2f724cbccf", "lid of black jar2-21856974749fa4cf", "lid of black jar1-a4ada65674ce97ae", "scissors-5cba3c3bf478d2ca", "lemon1-5c34b77a865fc36a", "second lemon half-3b9775645dbb370f", "small plate-b75595f5fafbba17", "notebook-1ba13250e37bdea7", "orange cloths-a6f77263ac20e348", "pen-3b3bf5e7017de656", "cap of oil-f7cb09968b725109", "lid of spice jar2-6de97d74ac133206", "piece of foil-43d5d571967bb574", "onion-b5e721c1aea4354f", "pack of fresh romaine lettuce-664fca351401b7a2", "bunch of tomatoes-f7c244f011e6ccec", "tomato1-6d7d35e441dcaff2", "tomato2-20ea5aaa1c44e820", "lemon2-71b8cd847b03e841", "lettuce1-7cf75be46024f30f", "piece of onion1-0d7b5aa98932ae85", "half of onion-acc1bd58c616a8b0", "second half of onion-65bfa2bbccf98c46", "bowl2-0a1262c86f71ccf0", "spoons-ccd2081d4e8cc69a", "bowl3-6ce463b7689badb4" ], "reasoning": "1. I locate the bowl (bowl1-690c3c19c6730040) on a stack of bowls (stack of three bowls-1ce6d503e005c036) in a cabinet in front and above the user at 03:38.\n2. At 05:06, I observe the user at the sink below and to the right of the cabinet , preparing to add water to the bowl (bowl1-690c3c19c6730040) filled with the chicken from the package (box of chicken-ed756fd25589eb34).\n3. I determine that at 05:06, the user would have to look to the cabinet above them to the left to identify the original location of the bowl (bowl1-690c3c19c6730040).", "referenced_object_masks": [ "stack of three bowls-1ce6d503e005c036", "bowl1-690c3c19c6730040", "box of chicken-ed756fd25589eb34" ] }, { "key": "P04-20240413-142619:42744b0397f8c10b083c80a7a2ca62021298db8c", "video_id": "P04-20240413-142619", "question": "When the towel reaches its third location, where is the towel's second location in relation to me?", "answer_choice_0": "Above and to the right", "answer_choice_1": "Above and to the left", "answer_choice_2": "Behind", "answer_choice_3": "Below", "answer_choice_4": "Directly ahead", "answer_id": 2, "answer": "Behind", "question_type": [ "Object Recognition", "Action Recognition", "Temporal Reasoning", "Egocentric & Spatial Perception", "Counting" ], "objects_of_interest": [ "measuring cup-24a180dd40d2d5f1", "soap dispenser-eefb4cf7b9db6b17", "black tea towel-dc0c6dd9834d5b91", "box of chicken-ed756fd25589eb34", "cooling rack-4fd9f5ad5b8fb6ca", "chopping board-f696c813f774c077", "herb jar1-e07e73b7e5188d9c", "herb jar2-a192a0897d1b82b4", "spice jar1-95dc26bbfd03ff97", "spice jar2-3f070f24df5ec151", "spice jar3-b6a5cedf2cf5ebcd", "bottle of oil-a6d945305f4c8100", "black jar1-a956ffc59c8a15da", "black jar2-7457a62c58c523e8", "herb jar3-c91abd3e2d1f2844", "condiment jar-3ffec4b711151386", "weighing scales-5d8e61b3f5c17472", "both black jars-bc93b3e14eadbede", "bag of lemons-3046e9fbf1b2446c", "spoon-af9ea4b03e35e603", "knife2-1a331567dfde7af4", "knife-0c07a8f376c06a09", "spoon2-8a765b15984b0e82", "spoon3-98ffa9ff7f44fe0d", "metallic mixing bowl-4759eb889afe2c0e", "plastic bowl-38495d6ed4b2bfb9", "stack of three bowls-1ce6d503e005c036", "bowl1-690c3c19c6730040", "lid of herb jar2-b9ce8575c6884125", "lid of herb jar1-2e31d2e0d94e6f96", "lid of spice jar1-714d8e2f724cbccf", "lid of black jar2-21856974749fa4cf", "lid of black jar1-a4ada65674ce97ae", "scissors-5cba3c3bf478d2ca", "lemon1-5c34b77a865fc36a", "second lemon half-3b9775645dbb370f", "small plate-b75595f5fafbba17", "notebook-1ba13250e37bdea7", "orange cloths-a6f77263ac20e348", "pen-3b3bf5e7017de656", "cap of oil-f7cb09968b725109", "lid of spice jar2-6de97d74ac133206", "piece of foil-43d5d571967bb574", "onion-b5e721c1aea4354f", "pack of fresh romaine lettuce-664fca351401b7a2", "bunch of tomatoes-f7c244f011e6ccec", "tomato1-6d7d35e441dcaff2", "tomato2-20ea5aaa1c44e820", "lemon2-71b8cd847b03e841", "lettuce1-7cf75be46024f30f", "piece of onion1-0d7b5aa98932ae85", "half of onion-acc1bd58c616a8b0", "second half of onion-65bfa2bbccf98c46", "bowl2-0a1262c86f71ccf0", "spoons-ccd2081d4e8cc69a", "bowl3-6ce463b7689badb4" ], "reasoning": "1. I locate the towel (black tea towel-dc0c6dd9834d5b91) at 00:45 draped over a metal bar in front of the stove .\n2. At 0:55, I observe the user place the towel (black tea towel-dc0c6dd9834d5b91) onto the floating table directly across from the stove .\n3. At 04:58, I observe the user reach for the towel (black tea towel-dc0c6dd9834d5b91) with their left hand.\n4. At 05:02, I observe the user place the towel (black tea towel-dc0c6dd9834d5b91) down behind the white bowl (bowl1-690c3c19c6730040) on the counter.\n5. I determine that the second location of the towel (black tea towel-dc0c6dd9834d5b91) is behind the user at 05:02.", "referenced_object_masks": [ "black tea towel-dc0c6dd9834d5b91", "bowl1-690c3c19c6730040" ] }, { "key": "P04-20240413-142619:54e737b58160651a74ce4abcf0500100708b7caf", "video_id": "P04-20240413-142619", "question": "How did the weight scale's reading change as the user added all the chicken?", "answer_choice_0": "By increasing from 0.0 to 3.0", "answer_choice_1": "By increasing from 0.0 to 2.8", "answer_choice_2": "By increasing from 0.0 to 25.3", "answer_choice_3": "By increasing from 0.0 to 40.4", "answer_choice_4": "By increasing from 0.0 to 24.2", "answer_id": 3, "answer": "By increasing from 0.0 to 40.4", "question_type": [ "Object Recognition", "State Changes", "Action Recognition", "Temporal Reasoning", "Reading" ], "objects_of_interest": [ "measuring cup-24a180dd40d2d5f1", "soap dispenser-eefb4cf7b9db6b17", "black tea towel-dc0c6dd9834d5b91", "box of chicken-ed756fd25589eb34", "cooling rack-4fd9f5ad5b8fb6ca", "chopping board-f696c813f774c077", "herb jar1-e07e73b7e5188d9c", "herb jar2-a192a0897d1b82b4", "spice jar1-95dc26bbfd03ff97", "spice jar2-3f070f24df5ec151", "spice jar3-b6a5cedf2cf5ebcd", "bottle of oil-a6d945305f4c8100", "black jar1-a956ffc59c8a15da", "black jar2-7457a62c58c523e8", "herb jar3-c91abd3e2d1f2844", "condiment jar-3ffec4b711151386", "weighing scales-5d8e61b3f5c17472", "both black jars-bc93b3e14eadbede", "bag of lemons-3046e9fbf1b2446c", "spoon-af9ea4b03e35e603", "knife2-1a331567dfde7af4", "knife-0c07a8f376c06a09", "spoon2-8a765b15984b0e82", "spoon3-98ffa9ff7f44fe0d", "metallic mixing bowl-4759eb889afe2c0e", "plastic bowl-38495d6ed4b2bfb9", "stack of three bowls-1ce6d503e005c036", "bowl1-690c3c19c6730040", "lid of herb jar2-b9ce8575c6884125", "lid of herb jar1-2e31d2e0d94e6f96", "lid of spice jar1-714d8e2f724cbccf", "lid of black jar2-21856974749fa4cf", "lid of black jar1-a4ada65674ce97ae", "scissors-5cba3c3bf478d2ca", "lemon1-5c34b77a865fc36a", "second lemon half-3b9775645dbb370f", "small plate-b75595f5fafbba17", "notebook-1ba13250e37bdea7", "orange cloths-a6f77263ac20e348", "pen-3b3bf5e7017de656", "cap of oil-f7cb09968b725109", "lid of spice jar2-6de97d74ac133206", "piece of foil-43d5d571967bb574", "onion-b5e721c1aea4354f", "pack of fresh romaine lettuce-664fca351401b7a2", "bunch of tomatoes-f7c244f011e6ccec", "tomato1-6d7d35e441dcaff2", "tomato2-20ea5aaa1c44e820", "lemon2-71b8cd847b03e841", "lettuce1-7cf75be46024f30f", "piece of onion1-0d7b5aa98932ae85", "half of onion-acc1bd58c616a8b0", "second half of onion-65bfa2bbccf98c46", "bowl2-0a1262c86f71ccf0", "spoons-ccd2081d4e8cc69a", "bowl3-6ce463b7689badb4" ], "reasoning": "1. I watched the video and found the packaged chicken (box of chicken-ed756fd25589eb34) at 01:01.\n2. I found the weight scale (weighing scales-5d8e61b3f5c17472) at 01:58.\n3. At 04:40, before the user added any of the packaged chicken (box of chicken-ed756fd25589eb34) to the scale (weighing scales-5d8e61b3f5c17472), the scale (weighing scales-5d8e61b3f5c17472) read \"0.0\".\n4. After the user added some of the packaged chicken (box of chicken-ed756fd25589eb34) to the scale (weighing scales-5d8e61b3f5c17472) at 04:43, the scale (weighing scales-5d8e61b3f5c17472) read \"2.8\".\n5. After the user added some more of the packaged chicken (box of chicken-ed756fd25589eb34) to the scale (weighing scales-5d8e61b3f5c17472) at 04:45, the scale (weighing scales-5d8e61b3f5c17472) read \"24.2\".\n6. After the user added the rest of the packaged chicken (box of chicken-ed756fd25589eb34) to the scale (weighing scales-5d8e61b3f5c17472) at 04:47, the scale (weighing scales-5d8e61b3f5c17472) read \"40.4\".\n7. Therefore, the reading of the weight scale (weighing scales-5d8e61b3f5c17472) changed as the user added all the chicken (box of chicken-ed756fd25589eb34) by increasing from 0.0 to 40.4.", "referenced_object_masks": [ "weighing scales-5d8e61b3f5c17472", "box of chicken-ed756fd25589eb34" ] }, { "key": "P04-20240413-142619:7b3ab2ad9fe1e47de1c93b132194a56f0b9aadd9", "video_id": "P04-20240413-142619", "question": "After using a spoon handle to loosen the spice in the spice jar, how many times do I tap the jar with my left pointer finger as I pour a spoonful?", "answer_choice_0": "25", "answer_choice_1": "22", "answer_choice_2": "24", "answer_choice_3": "23", "answer_choice_4": "26", "answer_id": 0, "answer": "25", "question_type": [ "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counting" ], "objects_of_interest": [ "measuring cup-24a180dd40d2d5f1", "soap dispenser-eefb4cf7b9db6b17", "black tea towel-dc0c6dd9834d5b91", "box of chicken-ed756fd25589eb34", "cooling rack-4fd9f5ad5b8fb6ca", "chopping board-f696c813f774c077", "herb jar1-e07e73b7e5188d9c", "herb jar2-a192a0897d1b82b4", "spice jar1-95dc26bbfd03ff97", "spice jar2-3f070f24df5ec151", "spice jar3-b6a5cedf2cf5ebcd", "bottle of oil-a6d945305f4c8100", "black jar1-a956ffc59c8a15da", "black jar2-7457a62c58c523e8", "herb jar3-c91abd3e2d1f2844", "condiment jar-3ffec4b711151386", "weighing scales-5d8e61b3f5c17472", "both black jars-bc93b3e14eadbede", "bag of lemons-3046e9fbf1b2446c", "spoon-af9ea4b03e35e603", "knife2-1a331567dfde7af4", "knife-0c07a8f376c06a09", "spoon2-8a765b15984b0e82", "spoon3-98ffa9ff7f44fe0d", "metallic mixing bowl-4759eb889afe2c0e", "plastic bowl-38495d6ed4b2bfb9", "stack of three bowls-1ce6d503e005c036", "bowl1-690c3c19c6730040", "lid of herb jar2-b9ce8575c6884125", "lid of herb jar1-2e31d2e0d94e6f96", "lid of spice jar1-714d8e2f724cbccf", "lid of black jar2-21856974749fa4cf", "lid of black jar1-a4ada65674ce97ae", "scissors-5cba3c3bf478d2ca", "lemon1-5c34b77a865fc36a", "second lemon half-3b9775645dbb370f", "small plate-b75595f5fafbba17", "notebook-1ba13250e37bdea7", "orange cloths-a6f77263ac20e348", "pen-3b3bf5e7017de656", "cap of oil-f7cb09968b725109", "lid of spice jar2-6de97d74ac133206", "piece of foil-43d5d571967bb574", "onion-b5e721c1aea4354f", "pack of fresh romaine lettuce-664fca351401b7a2", "bunch of tomatoes-f7c244f011e6ccec", "tomato1-6d7d35e441dcaff2", "tomato2-20ea5aaa1c44e820", "lemon2-71b8cd847b03e841", "lettuce1-7cf75be46024f30f", "piece of onion1-0d7b5aa98932ae85", "half of onion-acc1bd58c616a8b0", "second half of onion-65bfa2bbccf98c46", "bowl2-0a1262c86f71ccf0", "spoons-ccd2081d4e8cc69a", "bowl3-6ce463b7689badb4" ], "reasoning": "1. At 06:24, I locate five spices (spice jar1-95dc26bbfd03ff97, spice jar2-3f070f24df5ec151, spice jar3-b6a5cedf2cf5ebcd, herb jar1-e07e73b7e5188d9c, herb jar2-a192a0897d1b82b4) in front of the user on the counter with the bowl (bowl1-690c3c19c6730040).\n2. From 07:38 to 07:40, I observe the user attempting to pour spice (spice jar2-3f070f24df5ec151) into a spoon (spoon-af9ea4b03e35e603).\n3. From 07:46 to 08:00, I observe the user loosen the spice in the jar (spice jar2-3f070f24df5ec151) with the handle of another spoon (spoon3-98ffa9ff7f44fe0d).\n4. From 08:03 to 08:09, I observe the user pour the spice (spice jar2-3f070f24df5ec151) into the spoon (spoon-af9ea4b03e35e603).\n5. From 08:03 to 08:09, I count the user tap the spice jar (spice jar2-3f070f24df5ec151) with their left pointer finger 25 times.", "referenced_object_masks": [ "spice jar1-95dc26bbfd03ff97", "spice jar2-3f070f24df5ec151", "spice jar3-b6a5cedf2cf5ebcd", "herb jar1-e07e73b7e5188d9c", "herb jar2-a192a0897d1b82b4", "bowl1-690c3c19c6730040", "spoon3-98ffa9ff7f44fe0d", "spoon-af9ea4b03e35e603" ] }, { "key": "P04-20240413-142619:afa92ba834336304ded02a2ef46a7909bef271d7", "video_id": "P04-20240413-142619", "question": "How many items did I take out of the cabinet after taking the packaged chicken out of the refrigerator?", "answer_choice_0": "9", "answer_choice_1": "7", "answer_choice_2": "10", "answer_choice_3": "11", "answer_choice_4": "8", "answer_id": 2, "answer": "10", "question_type": [ "Action Recognition", "Temporal Reasoning", "Situational Awareness", "Object Recognition", "Counting" ], "objects_of_interest": [ "measuring cup-24a180dd40d2d5f1", "soap dispenser-eefb4cf7b9db6b17", "black tea towel-dc0c6dd9834d5b91", "box of chicken-ed756fd25589eb34", "cooling rack-4fd9f5ad5b8fb6ca", "chopping board-f696c813f774c077", "herb jar1-e07e73b7e5188d9c", "herb jar2-a192a0897d1b82b4", "spice jar1-95dc26bbfd03ff97", "spice jar2-3f070f24df5ec151", "spice jar3-b6a5cedf2cf5ebcd", "bottle of oil-a6d945305f4c8100", "black jar1-a956ffc59c8a15da", "black jar2-7457a62c58c523e8", "herb jar3-c91abd3e2d1f2844", "condiment jar-3ffec4b711151386", "weighing scales-5d8e61b3f5c17472", "both black jars-bc93b3e14eadbede", "bag of lemons-3046e9fbf1b2446c", "spoon-af9ea4b03e35e603", "knife2-1a331567dfde7af4", "knife-0c07a8f376c06a09", "spoon2-8a765b15984b0e82", "spoon3-98ffa9ff7f44fe0d", "metallic mixing bowl-4759eb889afe2c0e", "plastic bowl-38495d6ed4b2bfb9", "stack of three bowls-1ce6d503e005c036", "bowl1-690c3c19c6730040", "lid of herb jar2-b9ce8575c6884125", "lid of herb jar1-2e31d2e0d94e6f96", "lid of spice jar1-714d8e2f724cbccf", "lid of black jar2-21856974749fa4cf", "lid of black jar1-a4ada65674ce97ae", "scissors-5cba3c3bf478d2ca", "lemon1-5c34b77a865fc36a", "second lemon half-3b9775645dbb370f", "small plate-b75595f5fafbba17", "notebook-1ba13250e37bdea7", "orange cloths-a6f77263ac20e348", "pen-3b3bf5e7017de656", "cap of oil-f7cb09968b725109", "lid of spice jar2-6de97d74ac133206", "piece of foil-43d5d571967bb574", "onion-b5e721c1aea4354f", "pack of fresh romaine lettuce-664fca351401b7a2", "bunch of tomatoes-f7c244f011e6ccec", "tomato1-6d7d35e441dcaff2", "tomato2-20ea5aaa1c44e820", "lemon2-71b8cd847b03e841", "lettuce1-7cf75be46024f30f", "piece of onion1-0d7b5aa98932ae85", "half of onion-acc1bd58c616a8b0", "second half of onion-65bfa2bbccf98c46", "bowl2-0a1262c86f71ccf0", "spoons-ccd2081d4e8cc69a", "bowl3-6ce463b7689badb4" ], "reasoning": "1. I watched the video and saw the user take the packaged chicken (box of chicken-ed756fd25589eb34) out of the refrigerator at 01:01.\n2. After 01:01, I saw the user open a cabinet at 01:17.\n3. I counted each item the user took out of the cabinet, starting at 01:17, calculating a total of 10, including the bottle of oil (bottle of oil-a6d945305f4c8100) at 01:34 and the kitchen scale at 01:57.\n4. Therefore, the user took 10 items out of the cabinet after taking the packaged chicken out of the refrigerator.", "referenced_object_masks": [ "bottle of oil-a6d945305f4c8100", "box of chicken-ed756fd25589eb34" ] }, { "key": "P04-20240413-142619:b2d9c3f2bd163aadc6b22d5d304fcb538abec6cb", "video_id": "P04-20240413-142619", "question": "What was the second item I took out of the refrigerator?", "answer_choice_0": "A condiment jar", "answer_choice_1": "A packaged chicken", "answer_choice_2": "A bottle of olive oil", "answer_choice_3": "A bag of lemons", "answer_choice_4": "A spice canister", "answer_id": 3, "answer": "A bag of lemons", "question_type": [ "Action Recognition", "Counting", "Situational Awareness", "Object Recognition" ], "objects_of_interest": [ "measuring cup-24a180dd40d2d5f1", "soap dispenser-eefb4cf7b9db6b17", "black tea towel-dc0c6dd9834d5b91", "box of chicken-ed756fd25589eb34", "cooling rack-4fd9f5ad5b8fb6ca", "chopping board-f696c813f774c077", "herb jar1-e07e73b7e5188d9c", "herb jar2-a192a0897d1b82b4", "spice jar1-95dc26bbfd03ff97", "spice jar2-3f070f24df5ec151", "spice jar3-b6a5cedf2cf5ebcd", "bottle of oil-a6d945305f4c8100", "black jar1-a956ffc59c8a15da", "black jar2-7457a62c58c523e8", "herb jar3-c91abd3e2d1f2844", "condiment jar-3ffec4b711151386", "weighing scales-5d8e61b3f5c17472", "both black jars-bc93b3e14eadbede", "bag of lemons-3046e9fbf1b2446c", "spoon-af9ea4b03e35e603", "knife2-1a331567dfde7af4", "knife-0c07a8f376c06a09", "spoon2-8a765b15984b0e82", "spoon3-98ffa9ff7f44fe0d", "metallic mixing bowl-4759eb889afe2c0e", "plastic bowl-38495d6ed4b2bfb9", "stack of three bowls-1ce6d503e005c036", "bowl1-690c3c19c6730040", "lid of herb jar2-b9ce8575c6884125", "lid of herb jar1-2e31d2e0d94e6f96", "lid of spice jar1-714d8e2f724cbccf", "lid of black jar2-21856974749fa4cf", "lid of black jar1-a4ada65674ce97ae", "scissors-5cba3c3bf478d2ca", "lemon1-5c34b77a865fc36a", "second lemon half-3b9775645dbb370f", "small plate-b75595f5fafbba17", "notebook-1ba13250e37bdea7", "orange cloths-a6f77263ac20e348", "pen-3b3bf5e7017de656", "cap of oil-f7cb09968b725109", "lid of spice jar2-6de97d74ac133206", "piece of foil-43d5d571967bb574", "onion-b5e721c1aea4354f", "pack of fresh romaine lettuce-664fca351401b7a2", "bunch of tomatoes-f7c244f011e6ccec", "tomato1-6d7d35e441dcaff2", "tomato2-20ea5aaa1c44e820", "lemon2-71b8cd847b03e841", "lettuce1-7cf75be46024f30f", "piece of onion1-0d7b5aa98932ae85", "half of onion-acc1bd58c616a8b0", "second half of onion-65bfa2bbccf98c46", "bowl2-0a1262c86f71ccf0", "spoons-ccd2081d4e8cc69a", "bowl3-6ce463b7689badb4" ], "reasoning": "1. I watched the video and counted each item the user took out of the refrigerator.\n2. I saw the user take the first item out of the refrigerator, which was packaged chicken (box of chicken-ed756fd25589eb34) at 01:01.\n3. I saw the user take the second item out of the refrigerator, which was a bag of lemons (bag of lemons-3046e9fbf1b2446c) at 02:29.\n4. Therefore, the second item the user took out of the refrigerator was a bag of lemons (bag of lemons-3046e9fbf1b2446c).", "referenced_object_masks": [ "box of chicken-ed756fd25589eb34", "bag of lemons-3046e9fbf1b2446c" ] }, { "key": "P04-20240413-142619:b3c5227c6cb2375a9b4c88ac57b31625c5d5a01f", "video_id": "P04-20240413-142619", "question": "Where did I move the measuring cup to before washing my hands?", "answer_choice_0": "From the right side of the left side of the sink", "answer_choice_1": "From the left side of the right side of the sink", "answer_choice_2": "From the left side of the right side of the stove", "answer_choice_3": "From inside the sink to on top of the stove", "answer_choice_4": "From the right side to the left side of the stove", "answer_id": 4, "answer": "From the right side to the left side of the stove", "question_type": [ "Object Recognition", "Action Recognition", "Temporal Reasoning", "Situational Awareness", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "measuring cup-24a180dd40d2d5f1", "soap dispenser-eefb4cf7b9db6b17", "black tea towel-dc0c6dd9834d5b91", "box of chicken-ed756fd25589eb34", "cooling rack-4fd9f5ad5b8fb6ca", "chopping board-f696c813f774c077", "herb jar1-e07e73b7e5188d9c", "herb jar2-a192a0897d1b82b4", "spice jar1-95dc26bbfd03ff97", "spice jar2-3f070f24df5ec151", "spice jar3-b6a5cedf2cf5ebcd", "bottle of oil-a6d945305f4c8100", "black jar1-a956ffc59c8a15da", "black jar2-7457a62c58c523e8", "herb jar3-c91abd3e2d1f2844", "condiment jar-3ffec4b711151386", "weighing scales-5d8e61b3f5c17472", "both black jars-bc93b3e14eadbede", "bag of lemons-3046e9fbf1b2446c", "spoon-af9ea4b03e35e603", "knife2-1a331567dfde7af4", "knife-0c07a8f376c06a09", "spoon2-8a765b15984b0e82", "spoon3-98ffa9ff7f44fe0d", "metallic mixing bowl-4759eb889afe2c0e", "plastic bowl-38495d6ed4b2bfb9", "stack of three bowls-1ce6d503e005c036", "bowl1-690c3c19c6730040", "lid of herb jar2-b9ce8575c6884125", "lid of herb jar1-2e31d2e0d94e6f96", "lid of spice jar1-714d8e2f724cbccf", "lid of black jar2-21856974749fa4cf", "lid of black jar1-a4ada65674ce97ae", "scissors-5cba3c3bf478d2ca", "lemon1-5c34b77a865fc36a", "second lemon half-3b9775645dbb370f", "small plate-b75595f5fafbba17", "notebook-1ba13250e37bdea7", "orange cloths-a6f77263ac20e348", "pen-3b3bf5e7017de656", "cap of oil-f7cb09968b725109", "lid of spice jar2-6de97d74ac133206", "piece of foil-43d5d571967bb574", "onion-b5e721c1aea4354f", "pack of fresh romaine lettuce-664fca351401b7a2", "bunch of tomatoes-f7c244f011e6ccec", "tomato1-6d7d35e441dcaff2", "tomato2-20ea5aaa1c44e820", "lemon2-71b8cd847b03e841", "lettuce1-7cf75be46024f30f", "piece of onion1-0d7b5aa98932ae85", "half of onion-acc1bd58c616a8b0", "second half of onion-65bfa2bbccf98c46", "bowl2-0a1262c86f71ccf0", "spoons-ccd2081d4e8cc69a", "bowl3-6ce463b7689badb4" ], "reasoning": "1. I watched the video and found the measuring cup (measuring cup-24a180dd40d2d5f1) at 00:05.\n2. I saw the user wash their hands at 00:14.\n3. Before 00:14, I saw the user move the measuring cup (measuring cup-24a180dd40d2d5f1) from the right side of the stove to the left side at 00:06.\n4. Therefore, the user moved the measuring cup (measuring cup-24a180dd40d2d5f1) from the right side to the left side of the stove before washing their hands.", "referenced_object_masks": [ "measuring cup-24a180dd40d2d5f1" ] }, { "key": "P04-20240413-142619:d123f7f12fa9f08d3b9ce682ef591948c1326d1b", "video_id": "P04-20240413-142619", "question": "When I open the cabinet for the second time, how many bowls are stacked on the middle shelf on the right?", "answer_choice_0": "5", "answer_choice_1": "6", "answer_choice_2": "3", "answer_choice_3": "7", "answer_choice_4": "4", "answer_id": 0, "answer": "5", "question_type": [ "Object Recognition", "Action Recognition", "Temporal Reasoning", "Egocentric & Spatial Perception", "Counting" ], "objects_of_interest": [ "measuring cup-24a180dd40d2d5f1", "soap dispenser-eefb4cf7b9db6b17", "black tea towel-dc0c6dd9834d5b91", "box of chicken-ed756fd25589eb34", "cooling rack-4fd9f5ad5b8fb6ca", "chopping board-f696c813f774c077", "herb jar1-e07e73b7e5188d9c", "herb jar2-a192a0897d1b82b4", "spice jar1-95dc26bbfd03ff97", "spice jar2-3f070f24df5ec151", "spice jar3-b6a5cedf2cf5ebcd", "bottle of oil-a6d945305f4c8100", "black jar1-a956ffc59c8a15da", "black jar2-7457a62c58c523e8", "herb jar3-c91abd3e2d1f2844", "condiment jar-3ffec4b711151386", "weighing scales-5d8e61b3f5c17472", "both black jars-bc93b3e14eadbede", "bag of lemons-3046e9fbf1b2446c", "spoon-af9ea4b03e35e603", "knife2-1a331567dfde7af4", "knife-0c07a8f376c06a09", "spoon2-8a765b15984b0e82", "spoon3-98ffa9ff7f44fe0d", "metallic mixing bowl-4759eb889afe2c0e", "plastic bowl-38495d6ed4b2bfb9", "stack of three bowls-1ce6d503e005c036", "bowl1-690c3c19c6730040", "lid of herb jar2-b9ce8575c6884125", "lid of herb jar1-2e31d2e0d94e6f96", "lid of spice jar1-714d8e2f724cbccf", "lid of black jar2-21856974749fa4cf", "lid of black jar1-a4ada65674ce97ae", "scissors-5cba3c3bf478d2ca", "lemon1-5c34b77a865fc36a", "second lemon half-3b9775645dbb370f", "small plate-b75595f5fafbba17", "notebook-1ba13250e37bdea7", "orange cloths-a6f77263ac20e348", "pen-3b3bf5e7017de656", "cap of oil-f7cb09968b725109", "lid of spice jar2-6de97d74ac133206", "piece of foil-43d5d571967bb574", "onion-b5e721c1aea4354f", "pack of fresh romaine lettuce-664fca351401b7a2", "bunch of tomatoes-f7c244f011e6ccec", "tomato1-6d7d35e441dcaff2", "tomato2-20ea5aaa1c44e820", "lemon2-71b8cd847b03e841", "lettuce1-7cf75be46024f30f", "piece of onion1-0d7b5aa98932ae85", "half of onion-acc1bd58c616a8b0", "second half of onion-65bfa2bbccf98c46", "bowl2-0a1262c86f71ccf0", "spoons-ccd2081d4e8cc69a", "bowl3-6ce463b7689badb4" ], "reasoning": "1. At 03:14, I observe the user open the cabinet above the counter for the first time.\n2. I locate one mixing bowl (metallic mixing bowl-4759eb889afe2c0e), one metal bowl , and two white serving bowls on the right side.\n3. At 03:28, I see a translucent bowl (plastic bowl-38495d6ed4b2bfb9) within the mixing bowl (metallic mixing bowl-4759eb889afe2c0e).\n4. At 03:52, I observe the user close the cabinet .\n5. At 09:33, I observe the user open the cabinet .\n6. I determine that at 09:33, there are 5 bowls stacked on the middle right shelf.", "referenced_object_masks": [ "metallic mixing bowl-4759eb889afe2c0e", "plastic bowl-38495d6ed4b2bfb9" ] }, { "key": "P04-20240413-142619:e472d2d98a18c6307d6dd0f8db86548cc6148349", "video_id": "P04-20240413-142619", "question": "If I reversed the order in which I added the first four spices to the bowl of chicken, what would the sequence of lid colors be?", "answer_choice_0": "Green, Orange, Yellow, Green", "answer_choice_1": "Yellow, Green, Yellow, Orange", "answer_choice_2": "Yellow, Green, Orange, Yellow", "answer_choice_3": "Yellow, Orange, Green, Green", "answer_choice_4": "Green, Green, Yellow, Orange", "answer_id": 3, "answer": "Yellow, Orange, Green, Green", "question_type": [ "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counting", "Counterfactual", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "measuring cup-24a180dd40d2d5f1", "soap dispenser-eefb4cf7b9db6b17", "black tea towel-dc0c6dd9834d5b91", "box of chicken-ed756fd25589eb34", "cooling rack-4fd9f5ad5b8fb6ca", "chopping board-f696c813f774c077", "herb jar1-e07e73b7e5188d9c", "herb jar2-a192a0897d1b82b4", "spice jar1-95dc26bbfd03ff97", "spice jar2-3f070f24df5ec151", "spice jar3-b6a5cedf2cf5ebcd", "bottle of oil-a6d945305f4c8100", "black jar1-a956ffc59c8a15da", "black jar2-7457a62c58c523e8", "herb jar3-c91abd3e2d1f2844", "condiment jar-3ffec4b711151386", "weighing scales-5d8e61b3f5c17472", "both black jars-bc93b3e14eadbede", "bag of lemons-3046e9fbf1b2446c", "spoon-af9ea4b03e35e603", "knife2-1a331567dfde7af4", "knife-0c07a8f376c06a09", "spoon2-8a765b15984b0e82", "spoon3-98ffa9ff7f44fe0d", "metallic mixing bowl-4759eb889afe2c0e", "plastic bowl-38495d6ed4b2bfb9", "stack of three bowls-1ce6d503e005c036", "bowl1-690c3c19c6730040", "lid of herb jar2-b9ce8575c6884125", "lid of herb jar1-2e31d2e0d94e6f96", "lid of spice jar1-714d8e2f724cbccf", "lid of black jar2-21856974749fa4cf", "lid of black jar1-a4ada65674ce97ae", "scissors-5cba3c3bf478d2ca", "lemon1-5c34b77a865fc36a", "second lemon half-3b9775645dbb370f", "small plate-b75595f5fafbba17", "notebook-1ba13250e37bdea7", "orange cloths-a6f77263ac20e348", "pen-3b3bf5e7017de656", "cap of oil-f7cb09968b725109", "lid of spice jar2-6de97d74ac133206", "piece of foil-43d5d571967bb574", "onion-b5e721c1aea4354f", "pack of fresh romaine lettuce-664fca351401b7a2", "bunch of tomatoes-f7c244f011e6ccec", "tomato1-6d7d35e441dcaff2", "tomato2-20ea5aaa1c44e820", "lemon2-71b8cd847b03e841", "lettuce1-7cf75be46024f30f", "piece of onion1-0d7b5aa98932ae85", "half of onion-acc1bd58c616a8b0", "second half of onion-65bfa2bbccf98c46", "bowl2-0a1262c86f71ccf0", "spoons-ccd2081d4e8cc69a", "bowl3-6ce463b7689badb4" ], "reasoning": "1. I observe the user placing the contents of the package of chicken (box of chicken-ed756fd25589eb34) into the bowl (bowl1-690c3c19c6730040) in front of them from 04:41 to 04:47.\n2. At 06:24, I locate one spice jar with an orange lid (spice jar1-95dc26bbfd03ff97), two spice jars with yellow lids (spice jar2-3f070f24df5ec151, spice jar3-b6a5cedf2cf5ebcd), and two spice jars with green lids (herb jar1-e07e73b7e5188d9c, herb jar2-a192a0897d1b82b4).\n3. Between 06:24 and 08:12, I observe the user adding spices from the 5 jars in the following order of lid color: green, green, orange, yellow.\n4. I determine that the reverse order of the spice jar lid covers is yellow, orange, green, green.", "referenced_object_masks": [ "box of chicken-ed756fd25589eb34", "herb jar1-e07e73b7e5188d9c", "herb jar2-a192a0897d1b82b4", "spice jar2-3f070f24df5ec151", "spice jar3-b6a5cedf2cf5ebcd", "bowl1-690c3c19c6730040", "spice jar1-95dc26bbfd03ff97" ] }, { "key": "P04-20240413-142619:ea278a47af5965ce302ff91a82f790c42c0c00f6", "video_id": "P04-20240413-142619", "question": "If I doubled the amount of time I spent washing my hands, approximately how long would I have washed them?", "answer_choice_0": "40 seconds", "answer_choice_1": "20 seconds", "answer_choice_2": "10 seconds", "answer_choice_3": "30 seconds", "answer_choice_4": "50 seconds", "answer_id": 4, "answer": "50 seconds", "question_type": [ "Action Recognition", "Temporal Reasoning", "Counting", "Counterfactual", "Numerical Reasoning" ], "objects_of_interest": [ "measuring cup-24a180dd40d2d5f1", "soap dispenser-eefb4cf7b9db6b17", "black tea towel-dc0c6dd9834d5b91", "box of chicken-ed756fd25589eb34", "cooling rack-4fd9f5ad5b8fb6ca", "chopping board-f696c813f774c077", "herb jar1-e07e73b7e5188d9c", "herb jar2-a192a0897d1b82b4", "spice jar1-95dc26bbfd03ff97", "spice jar2-3f070f24df5ec151", "spice jar3-b6a5cedf2cf5ebcd", "bottle of oil-a6d945305f4c8100", "black jar1-a956ffc59c8a15da", "black jar2-7457a62c58c523e8", "herb jar3-c91abd3e2d1f2844", "condiment jar-3ffec4b711151386", "weighing scales-5d8e61b3f5c17472", "both black jars-bc93b3e14eadbede", "bag of lemons-3046e9fbf1b2446c", "spoon-af9ea4b03e35e603", "knife2-1a331567dfde7af4", "knife-0c07a8f376c06a09", "spoon2-8a765b15984b0e82", "spoon3-98ffa9ff7f44fe0d", "metallic mixing bowl-4759eb889afe2c0e", "plastic bowl-38495d6ed4b2bfb9", "stack of three bowls-1ce6d503e005c036", "bowl1-690c3c19c6730040", "lid of herb jar2-b9ce8575c6884125", "lid of herb jar1-2e31d2e0d94e6f96", "lid of spice jar1-714d8e2f724cbccf", "lid of black jar2-21856974749fa4cf", "lid of black jar1-a4ada65674ce97ae", "scissors-5cba3c3bf478d2ca", "lemon1-5c34b77a865fc36a", "second lemon half-3b9775645dbb370f", "small plate-b75595f5fafbba17", "notebook-1ba13250e37bdea7", "orange cloths-a6f77263ac20e348", "pen-3b3bf5e7017de656", "cap of oil-f7cb09968b725109", "lid of spice jar2-6de97d74ac133206", "piece of foil-43d5d571967bb574", "onion-b5e721c1aea4354f", "pack of fresh romaine lettuce-664fca351401b7a2", "bunch of tomatoes-f7c244f011e6ccec", "tomato1-6d7d35e441dcaff2", "tomato2-20ea5aaa1c44e820", "lemon2-71b8cd847b03e841", "lettuce1-7cf75be46024f30f", "piece of onion1-0d7b5aa98932ae85", "half of onion-acc1bd58c616a8b0", "second half of onion-65bfa2bbccf98c46", "bowl2-0a1262c86f71ccf0", "spoons-ccd2081d4e8cc69a", "bowl3-6ce463b7689badb4" ], "reasoning": "1. I watched the video and saw the user wash their hands from 00:14-00:38.\n2. I calculated the duration from 00:14-00:38, calculating a total of 24 seconds.\n3. I calculated 24*2=48, which approximates to 50 seconds.\n4. Therefore, if the user doubled the amount of time they spent washing their hands, they would have washed them for approximately 50 seconds.", "referenced_object_masks": [] }, { "key": "P04-20240413-145245:0afc0e83007237cf9d206f399b36d06f08fdaf5f", "video_id": "P04-20240413-145245", "question": "What is the order of the following actions that I performed, if I were to reverse those actions chronologically: move salt shaker, dry hand with scrunched towel, place bowl on food scale, turn food scale on?", "answer_choice_0": "Turn on food scale, place bowl on food scale, move salt shaker, dry hand with scrunched towel", "answer_choice_1": "Move salt shaker, dry hand with scrunched towel, turn food scale on, place bowl on food scale", "answer_choice_2": "Place bowl on food scale, dry hand with scrunched towel, turn on food scale, move salt shaker", "answer_choice_3": "Dry hand with scrunched towel, turn on food scale, place bowl on food scale, move salt shaker", "answer_choice_4": "Move salt shaker, turn on food scale, dry hand with scrunched towel, place bowl on food scale", "answer_id": 4, "answer": "Move salt shaker, turn on food scale, dry hand with scrunched towel, place bowl on food scale", "question_type": [ "Temporal Reasoning", "Counting", "Event Occurence", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "frying pan-921d7b4d7ac74b8f", "plate-015ea96430f9db8a", "plug-5519ea4633309ddf", "bowl-9b730e42be90b7a9", "wooden spoon-12da575b18f1206d", "black slotted spatula-f48cc487bb8b3431", "tongs-ed72550f8b816bf6", "kitchen tongs-3452eded0581664d", "head of romaine lettuce-3cd25bfad2750162", "sieve-0f067a401ba67ef6", "conical shaped white bowl-1dcd01dc28b4ede0", "knife-954a8f4896f9a220", "piece of lettuce-f4dd1727d9984294", "fork-7a3bbf9f9b31a389", "larger white bowl-5b2d36d76fa615ff", "white bowl-56b27e4eb887bbe1", "kitchen rag-1d9fd887b4e77523", "black marker-3bf0e009f0b82cb0" ], "reasoning": "1. I watched for the first event in chronological order of the listed events to occur: this happens at 11:53, when the user places a white bowl (conical shaped white bowl-1dcd01dc28b4ede0) on the food scale .\n2. I continued watching the video for the second chronological event, which I observed at 11:54, when the user reaches across and dries their hand by scrunching it on the scrunched towel (kitchen rag-1d9fd887b4e77523) that rests on a large wooden cutting board .\n3. I continued watching for the third chronological event of those listed, which I observed at 11:57 when the user turned on the food scale by pressing a button and making its screen light up.\n4. I continued watching for the fourth chronological event, which I observed at 11:58 when the user used their left hand to shift the salt shaker leftward on the counter, away from the food scale .\n5. Having observed these events in chronological order, I reversed the order I watched them occur to create the following list: move salt shaker, turn on food scale, dry hand with scrunched towel, and place bowl on food scale, making E the correct answer.", "referenced_object_masks": [ "conical shaped white bowl-1dcd01dc28b4ede0", "kitchen rag-1d9fd887b4e77523" ] }, { "key": "P04-20240413-145245:228debcf4110f45be80ffd9c9f6482780cd0ab7f", "video_id": "P04-20240413-145245", "question": "What is the fifth thing I touched with my right hand after plugging in the portable burner?", "answer_choice_0": "The bowl of chicken", "answer_choice_1": "My own forehead", "answer_choice_2": "A button on the portable burner", "answer_choice_3": "My own left hand", "answer_choice_4": "An electrical outlet switch", "answer_id": 2, "answer": "A button on the portable burner", "question_type": [ "Counting", "Event Occurence", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "frying pan-921d7b4d7ac74b8f", "plate-015ea96430f9db8a", "plug-5519ea4633309ddf", "bowl-9b730e42be90b7a9", "wooden spoon-12da575b18f1206d", "black slotted spatula-f48cc487bb8b3431", "tongs-ed72550f8b816bf6", "kitchen tongs-3452eded0581664d", "head of romaine lettuce-3cd25bfad2750162", "sieve-0f067a401ba67ef6", "conical shaped white bowl-1dcd01dc28b4ede0", "knife-954a8f4896f9a220", "piece of lettuce-f4dd1727d9984294", "fork-7a3bbf9f9b31a389", "larger white bowl-5b2d36d76fa615ff", "white bowl-56b27e4eb887bbe1", "kitchen rag-1d9fd887b4e77523", "black marker-3bf0e009f0b82cb0" ], "reasoning": "1. I first watched the video to determine when the user plugs in the portable burner : this occurs from 00:26-00:28.\n2. Next, I paid attention to the chronological sequence of items that the user touches with their right hand, counting them as they occur. The first of these is a bowl of raw chicken (bowl-9b730e42be90b7a9), which the user touches at 00:29.\n3. The second item that the user touches is an electrical outlet switch that the burner is plugged into, at 00:32.\n4. The third thing that the user touches is their own forehead, bringing their right hand up and out of the top of the camera at 00:34.\n5. The fourth thing that the user touches is their own left hand, as they rub their hands together from 00:35-00:38.\n6. The fifth thing that the user touches with their right hand is a button on the portable burner at 00:38-00:39, making D the correct answer.", "referenced_object_masks": [ "bowl-9b730e42be90b7a9" ] }, { "key": "P04-20240413-145245:35aa70be5a6d0bd0c55a70be2568d87dc66b5056", "video_id": "P04-20240413-145245", "question": "After the lettuce is washed, how many times do I shake the lettuce in the sink, before I cut it?", "answer_choice_0": "More than 25 times", "answer_choice_1": "Between 20 and 25 times", "answer_choice_2": "25 times", "answer_choice_3": "Between 17 and 20 times", "answer_choice_4": "15 times", "answer_id": 0, "answer": "More than 25 times", "question_type": [ "Action Recognition", "Counting", "Object Recognition" ], "objects_of_interest": [ "frying pan-921d7b4d7ac74b8f", "plate-015ea96430f9db8a", "plug-5519ea4633309ddf", "bowl-9b730e42be90b7a9", "wooden spoon-12da575b18f1206d", "black slotted spatula-f48cc487bb8b3431", "tongs-ed72550f8b816bf6", "kitchen tongs-3452eded0581664d", "head of romaine lettuce-3cd25bfad2750162", "sieve-0f067a401ba67ef6", "conical shaped white bowl-1dcd01dc28b4ede0", "knife-954a8f4896f9a220", "piece of lettuce-f4dd1727d9984294", "fork-7a3bbf9f9b31a389", "larger white bowl-5b2d36d76fa615ff", "white bowl-56b27e4eb887bbe1", "kitchen rag-1d9fd887b4e77523", "black marker-3bf0e009f0b82cb0" ], "reasoning": "1. I saw the user begin to wash the lettuce (head of romaine lettuce-3cd25bfad2750162) at 3:45.\n2. I saw the user finish washing the lettuce (head of romaine lettuce-3cd25bfad2750162) at 3:50. \n3. I saw the user shake the lettuce (head of romaine lettuce-3cd25bfad2750162) for the 1st time at 3:51. \n4. The user continues to shake the lettuce (head of romaine lettuce-3cd25bfad2750162) more than 25 times.", "referenced_object_masks": [ "head of romaine lettuce-3cd25bfad2750162" ] }, { "key": "P04-20240413-145245:70e3ae81420a1fb2c315d4293883f63bac687187", "video_id": "P04-20240413-145245", "question": "After I pull the tongs out of the drawer, where is the bottle with the orange top in relation to the bowl when it is filled with cutlets?", "answer_choice_0": "Above and to the right", "answer_choice_1": "Centered behind", "answer_choice_2": "Above and to the left", "answer_choice_3": "Below and to the right", "answer_choice_4": "Below and to the left", "answer_id": 0, "answer": "Above and to the right", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Object Recognition" ], "objects_of_interest": [ "frying pan-921d7b4d7ac74b8f", "plate-015ea96430f9db8a", "plug-5519ea4633309ddf", "bowl-9b730e42be90b7a9", "wooden spoon-12da575b18f1206d", "black slotted spatula-f48cc487bb8b3431", "tongs-ed72550f8b816bf6", "kitchen tongs-3452eded0581664d", "head of romaine lettuce-3cd25bfad2750162", "sieve-0f067a401ba67ef6", "conical shaped white bowl-1dcd01dc28b4ede0", "knife-954a8f4896f9a220", "piece of lettuce-f4dd1727d9984294", "fork-7a3bbf9f9b31a389", "larger white bowl-5b2d36d76fa615ff", "white bowl-56b27e4eb887bbe1", "kitchen rag-1d9fd887b4e77523", "black marker-3bf0e009f0b82cb0" ], "reasoning": "1. At 00:04, I first noticed the bottle with the orange top positioned with several other bottles and seasoning jars, lined against the backsplash to the right of the stovetop. The bottle with the orange top remains in this position. \n2. At 01:07, the user places the bowl (bowl-9b730e42be90b7a9) with chicken cutlets on the counter. The bowl (bowl-9b730e42be90b7a9) does not move until 01:26 when the user picks it up to add the cutlets to a pan (frying pan-921d7b4d7ac74b8f). \n3. I noticed the user pull out the tongs (kitchen tongs-3452eded0581664d) at 1:22 from the drawer. \n4. At 01:23, I noticed the bottle with the orange top is above and slightly to the right of the chicken cutlet bowl (bowl-9b730e42be90b7a9).", "referenced_object_masks": [ "kitchen tongs-3452eded0581664d", "frying pan-921d7b4d7ac74b8f", "bowl-9b730e42be90b7a9" ] }, { "key": "P04-20240413-145245:7c75e9256c8a2d10414b34d50ecfc5a0a0cf6f90", "video_id": "P04-20240413-145245", "question": "After I clean my hands with the black dish rag, in what direction do I make cuts in the saut\u00e9ed chicken cutlets after removing them from the pan?", "answer_choice_0": "Single horizontal cuts", "answer_choice_1": "1 horizontal, 1 vertical cut", "answer_choice_2": "Single diagonal cuts", "answer_choice_3": "Single vertical cuts", "answer_choice_4": "1 diagonal, 1 horizontal cut", "answer_id": 0, "answer": "Single horizontal cuts", "question_type": [ "Action Recognition", "Counting", "Object Recognition", "Egocentric & Spatial Perception", "State Changes" ], "objects_of_interest": [ "frying pan-921d7b4d7ac74b8f", "plate-015ea96430f9db8a", "plug-5519ea4633309ddf", "bowl-9b730e42be90b7a9", "wooden spoon-12da575b18f1206d", "black slotted spatula-f48cc487bb8b3431", "tongs-ed72550f8b816bf6", "kitchen tongs-3452eded0581664d", "head of romaine lettuce-3cd25bfad2750162", "sieve-0f067a401ba67ef6", "conical shaped white bowl-1dcd01dc28b4ede0", "knife-954a8f4896f9a220", "piece of lettuce-f4dd1727d9984294", "fork-7a3bbf9f9b31a389", "larger white bowl-5b2d36d76fa615ff", "white bowl-56b27e4eb887bbe1", "kitchen rag-1d9fd887b4e77523", "black marker-3bf0e009f0b82cb0" ], "reasoning": "1. I saw the user clean their hands with the black dish rag (kitchen rag-1d9fd887b4e77523) at 14:07 and place it to the right of the stove top.\n2. I saw 4 chicken cutlets in the pan (frying pan-921d7b4d7ac74b8f) at 14:08.\n3. I saw the user pull out 1 chicken cutlet at 14:17 and at 14:22, the user makes a single horizontal cut across the cutlet . \n4. I saw the user pull out a second chicken cutlet at 14:32, making another single horizontal cut across the cutlet at 14:34.\n5. The user pulls a 3rd cutlet from the pan (frying pan-921d7b4d7ac74b8f) at 14:38 and makes a third single horizontal cut across the cutlet at 14:41.\n6. Finally, the user pulls the 4th piece of cutlet from the pan (frying pan-921d7b4d7ac74b8f) at 14:51, making a fourth single horizontal cut across the cutlet at 14:55.\n7. Therefore, the user makes single horizontal cuts across each cutlet removed from the pan (frying pan-921d7b4d7ac74b8f).", "referenced_object_masks": [ "frying pan-921d7b4d7ac74b8f", "kitchen rag-1d9fd887b4e77523" ] }, { "key": "P04-20240413-145245:7ecf205d7cd99202d231dad1d39e7498ae83f4c5", "video_id": "P04-20240413-145245", "question": "Where do I first place the spatula with a blue head, in relation to the bowl of onions?", "answer_choice_0": "Behind the bowl of onions", "answer_choice_1": "On top of the bowl of onions", "answer_choice_2": "To the right of the bowl of onions", "answer_choice_3": "To the left of the bowl of onions", "answer_choice_4": "In front of the bowl of onions", "answer_id": 4, "answer": "In front of the bowl of onions", "question_type": [ "Object Recognition", "Egocentric & Spatial Perception", "Event Occurence" ], "objects_of_interest": [ "frying pan-921d7b4d7ac74b8f", "plate-015ea96430f9db8a", "plug-5519ea4633309ddf", "bowl-9b730e42be90b7a9", "wooden spoon-12da575b18f1206d", "black slotted spatula-f48cc487bb8b3431", "tongs-ed72550f8b816bf6", "kitchen tongs-3452eded0581664d", "head of romaine lettuce-3cd25bfad2750162", "sieve-0f067a401ba67ef6", "conical shaped white bowl-1dcd01dc28b4ede0", "knife-954a8f4896f9a220", "piece of lettuce-f4dd1727d9984294", "fork-7a3bbf9f9b31a389", "larger white bowl-5b2d36d76fa615ff", "white bowl-56b27e4eb887bbe1", "kitchen rag-1d9fd887b4e77523", "black marker-3bf0e009f0b82cb0" ], "reasoning": "1. I watch the video to identify when the user removes the spatula with a blue head from the cabinet. This happens at 00:56.\n2. I then note the location of the bowl of onions at 00:05 - I identify it as a small white bowl of chopped white onions, which can be seen situated behind the top right corner of the cutting board on the countertop.\n3. I observe the user set the spatula with a blue head down along the right side of the cutting board at 01:05.\n4. I observe the position of the spatula with the blue head in relation to the bowl of onions : the spatula is directly in front of the bowl of onions on the right edge of the cutting board.", "referenced_object_masks": [] }, { "key": "P04-20240413-145245:afe83bc4d0780690c041b2efa5414d4aca1a9193", "video_id": "P04-20240413-145245", "question": "After taking the leaf off the lettuce and putting it back together, where do I place the item I pull out of the drawer in relation to the lettuce?", "answer_choice_0": "Above and to the left", "answer_choice_1": "On top", "answer_choice_2": "Below and to the right", "answer_choice_3": "Above and to the right", "answer_choice_4": "Below and to the left", "answer_id": 0, "answer": "Above and to the left", "question_type": [ "Action Recognition", "Object Recognition", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "frying pan-921d7b4d7ac74b8f", "plate-015ea96430f9db8a", "plug-5519ea4633309ddf", "bowl-9b730e42be90b7a9", "wooden spoon-12da575b18f1206d", "black slotted spatula-f48cc487bb8b3431", "tongs-ed72550f8b816bf6", "kitchen tongs-3452eded0581664d", "head of romaine lettuce-3cd25bfad2750162", "sieve-0f067a401ba67ef6", "conical shaped white bowl-1dcd01dc28b4ede0", "knife-954a8f4896f9a220", "piece of lettuce-f4dd1727d9984294", "fork-7a3bbf9f9b31a389", "larger white bowl-5b2d36d76fa615ff", "white bowl-56b27e4eb887bbe1", "kitchen rag-1d9fd887b4e77523", "black marker-3bf0e009f0b82cb0" ], "reasoning": "1. I noticed the user pull the leaf of lettuce (piece of lettuce-f4dd1727d9984294) at 2:26.\n2. I noticed the user put the (piece of lettuce-f4dd1727d9984294) back on the lettuce stock (head of romaine lettuce-3cd25bfad2750162) at 2:28.\n3. I noticed the user put the lettuce stock (head of romaine lettuce-3cd25bfad2750162) on the counter at 2:30.\n4. I noticed the user open the drawer at 2:33.\n5. I noticed the user pull out the strainer (sieve-0f067a401ba67ef6) at 2:35.\n6. I noticed the user put the strainer (sieve-0f067a401ba67ef6) on the counter above and to the left of the lettuce (head of romaine lettuce-3cd25bfad2750162) at 2:38.", "referenced_object_masks": [ "head of romaine lettuce-3cd25bfad2750162", "piece of lettuce-f4dd1727d9984294", "sieve-0f067a401ba67ef6" ] }, { "key": "P04-20240413-145245:b8b7e878c7feb258f44f6dda6569056d02235880", "video_id": "P04-20240413-145245", "question": "After scooping the chicken into the bowl 7 times with my hands and rearranging the fork and knife, what do I do?", "answer_choice_0": "Wash hands", "answer_choice_1": "Put bowl back in cabinet", "answer_choice_2": "Put tongs away", "answer_choice_3": "Move the pan onto right stove top", "answer_choice_4": "Wash plates", "answer_id": 0, "answer": "Wash hands", "question_type": [ "Action Recognition", "Object Recognition", "Counting", "Event Occurence", "Goal Reasoning", "Situational Awareness" ], "objects_of_interest": [ "frying pan-921d7b4d7ac74b8f", "plate-015ea96430f9db8a", "plug-5519ea4633309ddf", "bowl-9b730e42be90b7a9", "wooden spoon-12da575b18f1206d", "black slotted spatula-f48cc487bb8b3431", "tongs-ed72550f8b816bf6", "kitchen tongs-3452eded0581664d", "head of romaine lettuce-3cd25bfad2750162", "sieve-0f067a401ba67ef6", "conical shaped white bowl-1dcd01dc28b4ede0", "knife-954a8f4896f9a220", "piece of lettuce-f4dd1727d9984294", "fork-7a3bbf9f9b31a389", "larger white bowl-5b2d36d76fa615ff", "white bowl-56b27e4eb887bbe1", "kitchen rag-1d9fd887b4e77523", "black marker-3bf0e009f0b82cb0" ], "reasoning": "1. I saw the user begin to scoop the chicken into the bowl (bowl-9b730e42be90b7a9) with their hands at 20:00. \n2. I saw the user scoop for the 7th time at 20:09. \n3. I saw the user rearrange the knife (knife-954a8f4896f9a220) and fork to the left of the stove top at 20:12. \n4. I saw the user wash their hands at 20:17.", "referenced_object_masks": [ "bowl-9b730e42be90b7a9", "knife-954a8f4896f9a220" ] }, { "key": "P04-20240413-145245:c5e54755f4db7c9c721d7b99da4a71dc182614fc", "video_id": "P04-20240413-145245", "question": "When I remove the first batch of chicken cutlets from the skillet and place them onto the cutting board and cut them, one-by-one, how many from that first batch do I cut only once before placing them back in the pan?", "answer_choice_0": "4", "answer_choice_1": "0", "answer_choice_2": "1", "answer_choice_3": "2", "answer_choice_4": "3", "answer_id": 2, "answer": "1", "question_type": [ "Counting", "Event Occurence", "State Changes", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "frying pan-921d7b4d7ac74b8f", "plate-015ea96430f9db8a", "plug-5519ea4633309ddf", "bowl-9b730e42be90b7a9", "wooden spoon-12da575b18f1206d", "black slotted spatula-f48cc487bb8b3431", "tongs-ed72550f8b816bf6", "kitchen tongs-3452eded0581664d", "head of romaine lettuce-3cd25bfad2750162", "sieve-0f067a401ba67ef6", "conical shaped white bowl-1dcd01dc28b4ede0", "knife-954a8f4896f9a220", "piece of lettuce-f4dd1727d9984294", "fork-7a3bbf9f9b31a389", "larger white bowl-5b2d36d76fa615ff", "white bowl-56b27e4eb887bbe1", "kitchen rag-1d9fd887b4e77523", "black marker-3bf0e009f0b82cb0" ], "reasoning": "1. I watch the video for when the user begins removing the first batch of chicken cutlets from the pan (frying pan-921d7b4d7ac74b8f) and placing them on the cutting board . This starts at 07:15.\n2. I watch the user remove the first cutlet from the skillet (frying pan-921d7b4d7ac74b8f), and from 07:15-07:42, place it on the cutting board and cut it twice with the knife (knife-954a8f4896f9a220) before putting its pieces back in the pan (frying pan-921d7b4d7ac74b8f) - thus, I do not count this one toward the one-cut total.\n3. I watch the user remove the second cutlet from the skillet (frying pan-921d7b4d7ac74b8f), and from 07:43-07:56, place it on the cutting board and cut it once with the knife (knife-954a8f4896f9a220) before putting its pieces back in the pan (frying pan-921d7b4d7ac74b8f) - thus, I count this as the first piece in the one-cut total.\n4. I watch the user remove the third cutlet from the skillet (frying pan-921d7b4d7ac74b8f), and from 08:02-08:18, place it on the cutting board and cut it twice with the knife (knife-954a8f4896f9a220) before putting its pieces back in the pan (frying pan-921d7b4d7ac74b8f) - thus, I do not count this one toward the one-cut total.\n5. I watch the user remove the fourth cutlet from the skillet (frying pan-921d7b4d7ac74b8f), and from 08:19-08:30, place it on the cutting board and cut it twice with the knife (knife-954a8f4896f9a220) before putting its pieces back in the pan (frying pan-921d7b4d7ac74b8f) - thus, I do not count this one toward the one-cut total.\n6. I watch the user remove the fifth cutlet from the skillet (frying pan-921d7b4d7ac74b8f), and from 08:44-08:58, place it on the cutting board and cut it twice with the knife (knife-954a8f4896f9a220) before putting its pieces back in the pan (frying pan-921d7b4d7ac74b8f) - thus, I do not count this one toward the one-cut total.\n7. I observe at 09:01 that the user sets the knife (knife-954a8f4896f9a220) and the other utensil down, signaling that they are done taking cutlets out of the pan (frying pan-921d7b4d7ac74b8f) and cutting them. Thus, I can observe that only 1 of the 5 was cut once.", "referenced_object_masks": [ "knife-954a8f4896f9a220", "frying pan-921d7b4d7ac74b8f" ] }, { "key": "P04-20240413-145245:fbe549567b3b84e35a6e40ce089118c53e64d67d", "video_id": "P04-20240413-145245", "question": "What is the difference between the number of times that I sliced the head of lettuce lengthwise and the number of times it took me to pick up and transfer the fully-chopped lettuce into the sieve?", "answer_choice_0": "1", "answer_choice_1": "4", "answer_choice_2": "3", "answer_choice_3": "0", "answer_choice_4": "2", "answer_id": 3, "answer": "0", "question_type": [ "Numerical Reasoning", "Counting", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "frying pan-921d7b4d7ac74b8f", "plate-015ea96430f9db8a", "plug-5519ea4633309ddf", "bowl-9b730e42be90b7a9", "wooden spoon-12da575b18f1206d", "black slotted spatula-f48cc487bb8b3431", "tongs-ed72550f8b816bf6", "kitchen tongs-3452eded0581664d", "head of romaine lettuce-3cd25bfad2750162", "sieve-0f067a401ba67ef6", "conical shaped white bowl-1dcd01dc28b4ede0", "knife-954a8f4896f9a220", "piece of lettuce-f4dd1727d9984294", "fork-7a3bbf9f9b31a389", "larger white bowl-5b2d36d76fa615ff", "white bowl-56b27e4eb887bbe1", "kitchen rag-1d9fd887b4e77523", "black marker-3bf0e009f0b82cb0" ], "reasoning": "1. I first observed the user slicing the head of lettuce (head of romaine lettuce-3cd25bfad2750162) lengthwise with the knife (knife-954a8f4896f9a220) from 04:48-05:32, counting each slice: 1 slice at 4:48-4:51, 1 at 4:54-4:58, 1 at 5:03-5:04, 1 at 5:08-5:09, 1 at 5:13-5:17, and 1 final slice at 5:24-5:25, or 6 lengthwise slices in all.\n2. Next, from 10:44-11:03, I watched and counted the number of times the user picks up pieces of the fully-chopped lettuce (head of romaine lettuce-3cd25bfad2750162) to transfer them into the sieve (sieve-0f067a401ba67ef6): 1 at 10:44, 1 at 10:48, 1 at 10:54, 1 at 10:58, 1 at 11:02 at 1 last time at 11: 03, or 6 times in all.\n3. I then subtraced the number of times it took the user to pick up the lettuce (head of romaine lettuce-3cd25bfad2750162) from the number of lengthwise slices with the equation 6 - 6 = 0.", "referenced_object_masks": [ "knife-954a8f4896f9a220", "head of romaine lettuce-3cd25bfad2750162", "sieve-0f067a401ba67ef6" ] }, { "key": "P04-20240413-151722:187f8eb123f7b403a02b7e960a41d49991d9c070", "video_id": "P04-20240413-151722", "question": "Which was greater, and by how much: The number of items I took out of cabinets or the number of items I took out of the refrigerator multiplied by 5?", "answer_choice_0": "Refrigerator, by 7", "answer_choice_1": "Neither--the numbers were equal", "answer_choice_2": "Cabinets--by 3", "answer_choice_3": "Refrigerator--by 3", "answer_choice_4": "Cabinets--by 7", "answer_id": 4, "answer": "Cabinets--by 7", "question_type": [ "Action Recognition", "Counting", "Numerical Reasoning", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "phone-ecc69da84e7148da", "pan-eab9c855e0ae0589", "bottle of liquid hand wash-8d89bdb34efd9f67", "spatula-1a056d3a50c064d8", "bowl of shredded lettuce-4da0fbbb88a59879", "conical shaped bowl-5c555508821a8c2f", "plate-4770066f89e13f46", "bowl with red things-4749bcd20d982d88", "bowl with white things-78b04db4fb1eb53e", "scale-39c0115d09bbe157", "lemon-5f3fa2657a347328", "knife-013d81f2be23e243", "one half of cut lemon-b3d97568202d986b", "marker-57ede0bdc38a6fae", "cap of pen-ae68b3022269afa5", "notebook-decc5af70076b9a0", "white serving bowl-60143ee5bdf403d5", "orange wiping cloth-c6e13d4a0c1214e8", "green capped condiment jar-dc0e8e11e1b9e817", "bottle of sriracha sauce-4d32d48f17b34718", "yogurt container-7b741ea2e8a6c56c", "jars of salt-693c973d627b71b0", "another jar-844a32fe7a1d41e1", "spoon-d98f0d94dbe38499", "kitchen rag-7375df10a21895b1", "marker cap-a53daf8135e813ac", "spoon3-13867cc128c5fffa", "cap of jar-3d337eea190bcfab", "cap of another jar-6af43e6480d12e0b", "cap of parsley jar-53d8918d3698e85c", "weighing scale-df94520d94ec9c1d", "spoon5-1f557948e67132ae", "spoon4-5db552d4404d6414", "fork-2c620840335d7411", "cap of salt jar-750d8ebecf2dc446", "jar of peppers-329b2f6766cbb049", "spoon6-b12571fd8ecbde5c", "plate2-28f1a5a953a71827", "packet-0f14c5ba8559b59e", "green pot-44adf4f26ae421e3", "pair of scissors-834f26061844bbb8", "one tortilla-0f4ea3fbede8dade", "tablespoon-146be3ceb5e109fb", "tortilla2-08a761ef36382b8b", "tortilla3-e64b604689a4a5f7", "tortilla4-0b2eea8e1e46f55a", "bottle of hand wash-ef36fb781d0a0800", "to be deleted-f4429bbe75b27172", "chopping board-7f9e37507f3b74d6", "one jar of condiment-87949691c754fc71", "second jar of condiment-33803082586cb2d4", "spice jar-a8484868b015c7c7", "bottile of oil-d581dc7c7b12dad1", "jar of peanut butter-0d6f31058f4f5313", "jar of onion powder-580a22d89982f7d0", "jar3-283bdf4381c97c30", "measuring cup-df406806d0744100", "stack of plastic containers-2621dbf5f5bee38d", "one empty container-dcd8061fd10d0cc0", "lid of container-0789ea2242217180" ], "reasoning": "1. I watched the video and counted the items that the user took out of cabinets. These items were: a white bowl (white serving bowl-60143ee5bdf403d5) at 01:16, a bottle of sriracha (bottle of sriracha sauce-4d32d48f17b34718) at 01:58, a spoon (spoon-d98f0d94dbe38499) at 02:28, a second spoon at 03:35, a third spoon (spoon3-13867cc128c5fffa) at 04:07, a plate (plate2-28f1a5a953a71827) at 07:48, a bag of tortillas (packet-0f14c5ba8559b59e) at 07:56, a fourth spoon (tablespoon-146be3ceb5e109fb) at 09:43, and a stack of containers (stack of plastic containers-2621dbf5f5bee38d) at 20:12. I saw the user also take from the stack of containers (stack of plastic containers-2621dbf5f5bee38d) 1 lid (lid of container-0789ea2242217180) and 1 container base (one empty container-dcd8061fd10d0cc0) from 20:16-20:19. Counting the stack of containers (stack of plastic containers-2621dbf5f5bee38d) as its own object, this makes a total of 12 items taken out of the cabinets. \n2. I counted the items that the user took out of the refrigerator. The only item taken out of the refrigerator was a container of yogurt (yogurt container-7b741ea2e8a6c56c) at 02:05. This makes 1 item. \n3. I multiplied the number of items taken out of the refrigerator (1) by 5: 1*5=5. \n4. I found the difference between the number of items taken out of the cabinets (12) and the previous product (5): 12-5=7.", "referenced_object_masks": [ "plate2-28f1a5a953a71827", "yogurt container-7b741ea2e8a6c56c", "packet-0f14c5ba8559b59e", "white serving bowl-60143ee5bdf403d5", "bottle of sriracha sauce-4d32d48f17b34718", "spoon3-13867cc128c5fffa", "spoon-d98f0d94dbe38499", "one empty container-dcd8061fd10d0cc0", "stack of plastic containers-2621dbf5f5bee38d", "tablespoon-146be3ceb5e109fb", "lid of container-0789ea2242217180" ] }, { "key": "P04-20240413-151722:2aaec9df60d1ec10987a8256412d54aaccd0e295", "video_id": "P04-20240413-151722", "question": "When I grab the third spoon I handled in the cutlery drawer for the second time, approximately how many degrees in a clockwise direction would I need to turn if I wanted to face the objects with smiley faces?", "answer_choice_0": "90 degrees", "answer_choice_1": "180 degrees", "answer_choice_2": "45 degrees", "answer_choice_3": "275 degrees", "answer_choice_4": "220 degrees", "answer_id": 0, "answer": "90 degrees", "question_type": [ "Action Recognition", "Counterfactual", "Counting", "Egocentric & Spatial Perception", "Object Recognition" ], "objects_of_interest": [ "phone-ecc69da84e7148da", "pan-eab9c855e0ae0589", "bottle of liquid hand wash-8d89bdb34efd9f67", "spatula-1a056d3a50c064d8", "bowl of shredded lettuce-4da0fbbb88a59879", "conical shaped bowl-5c555508821a8c2f", "plate-4770066f89e13f46", "bowl with red things-4749bcd20d982d88", "bowl with white things-78b04db4fb1eb53e", "scale-39c0115d09bbe157", "lemon-5f3fa2657a347328", "knife-013d81f2be23e243", "one half of cut lemon-b3d97568202d986b", "marker-57ede0bdc38a6fae", "cap of pen-ae68b3022269afa5", "notebook-decc5af70076b9a0", "white serving bowl-60143ee5bdf403d5", "orange wiping cloth-c6e13d4a0c1214e8", "green capped condiment jar-dc0e8e11e1b9e817", "bottle of sriracha sauce-4d32d48f17b34718", "yogurt container-7b741ea2e8a6c56c", "jars of salt-693c973d627b71b0", "another jar-844a32fe7a1d41e1", "spoon-d98f0d94dbe38499", "kitchen rag-7375df10a21895b1", "marker cap-a53daf8135e813ac", "spoon3-13867cc128c5fffa", "cap of jar-3d337eea190bcfab", "cap of another jar-6af43e6480d12e0b", "cap of parsley jar-53d8918d3698e85c", "weighing scale-df94520d94ec9c1d", "spoon5-1f557948e67132ae", "spoon4-5db552d4404d6414", "fork-2c620840335d7411", "cap of salt jar-750d8ebecf2dc446", "jar of peppers-329b2f6766cbb049", "spoon6-b12571fd8ecbde5c", "plate2-28f1a5a953a71827", "packet-0f14c5ba8559b59e", "green pot-44adf4f26ae421e3", "pair of scissors-834f26061844bbb8", "one tortilla-0f4ea3fbede8dade", "tablespoon-146be3ceb5e109fb", "tortilla2-08a761ef36382b8b", "tortilla3-e64b604689a4a5f7", "tortilla4-0b2eea8e1e46f55a", "bottle of hand wash-ef36fb781d0a0800", "to be deleted-f4429bbe75b27172", "chopping board-7f9e37507f3b74d6", "one jar of condiment-87949691c754fc71", "second jar of condiment-33803082586cb2d4", "spice jar-a8484868b015c7c7", "bottile of oil-d581dc7c7b12dad1", "jar of peanut butter-0d6f31058f4f5313", "jar of onion powder-580a22d89982f7d0", "jar3-283bdf4381c97c30", "measuring cup-df406806d0744100", "stack of plastic containers-2621dbf5f5bee38d", "one empty container-dcd8061fd10d0cc0", "lid of container-0789ea2242217180" ], "reasoning": "1. First, I looked for the user to handle 3 spoons in the cutlery drawer. I saw them touch the first spoon at 02:24, the second spoon at 02:26, and the third spoon (spoon-d98f0d94dbe38499) at 02:27. \n2. Since the user grabs the third spoon (spoon-d98f0d94dbe38499) the first time they touch it, I counted that as the first grab. I continued watching until the user placed the spoon (spoon-d98f0d94dbe38499) down on a cutting board above the cutlery drawer at 02:23. The user grabs the same spoon (spoon-d98f0d94dbe38499) for the second time at 02:38. \n3. I took note of where the user is in the space at 02:38 and realized they are facing the counter directly above the cutlery drawer, which runs perpendicularly along the wall to the left of the sink. \n4. Next, I went back and looked for any objects with smiley faces. I saw two yellow sponges with smiley faces on them at the sink at 00:10. I took note that the sink is directly next to the counter that runs perpendicularly along the wall to the left of the sink. \n5. Reasoning about the scene, I realized the user would need to turn 90 degrees in a clockwise direction if they wanted to face the sponges when they grab the third spoon (spoon-d98f0d94dbe38499) they handled in the cutlery drawer for the second time.", "referenced_object_masks": [ "spoon-d98f0d94dbe38499" ] }, { "key": "P04-20240413-151722:6ac7ea35d40b14a5d3ca2ea0dd4e347e38566296", "video_id": "P04-20240413-151722", "question": "What do the following 3 items all have in common: the final ingredient I add to the sauce, the 3rd object directly behind the last object I place in the front leftmost spot beneath the utensil drawer, and the last object I place in the front leftmost spot beneath the utensil drawer?", "answer_choice_0": "They all contain the color yellow", "answer_choice_1": "They all contain the color red", "answer_choice_2": ", They all contain the color blue", "answer_choice_3": "They all contain the color purple", "answer_choice_4": "They all contain the color green", "answer_id": 0, "answer": "They all contain the color yellow", "question_type": [ "Action Recognition", "Counting", "Egocentric & Spatial Perception", "Object Recognition" ], "objects_of_interest": [ "phone-ecc69da84e7148da", "pan-eab9c855e0ae0589", "bottle of liquid hand wash-8d89bdb34efd9f67", "spatula-1a056d3a50c064d8", "bowl of shredded lettuce-4da0fbbb88a59879", "conical shaped bowl-5c555508821a8c2f", "plate-4770066f89e13f46", "bowl with red things-4749bcd20d982d88", "bowl with white things-78b04db4fb1eb53e", "scale-39c0115d09bbe157", "lemon-5f3fa2657a347328", "knife-013d81f2be23e243", "one half of cut lemon-b3d97568202d986b", "marker-57ede0bdc38a6fae", "cap of pen-ae68b3022269afa5", "notebook-decc5af70076b9a0", "white serving bowl-60143ee5bdf403d5", "orange wiping cloth-c6e13d4a0c1214e8", "green capped condiment jar-dc0e8e11e1b9e817", "bottle of sriracha sauce-4d32d48f17b34718", "yogurt container-7b741ea2e8a6c56c", "jars of salt-693c973d627b71b0", "another jar-844a32fe7a1d41e1", "spoon-d98f0d94dbe38499", "kitchen rag-7375df10a21895b1", "marker cap-a53daf8135e813ac", "spoon3-13867cc128c5fffa", "cap of jar-3d337eea190bcfab", "cap of another jar-6af43e6480d12e0b", "cap of parsley jar-53d8918d3698e85c", "weighing scale-df94520d94ec9c1d", "spoon5-1f557948e67132ae", "spoon4-5db552d4404d6414", "fork-2c620840335d7411", "cap of salt jar-750d8ebecf2dc446", "jar of peppers-329b2f6766cbb049", "spoon6-b12571fd8ecbde5c", "plate2-28f1a5a953a71827", "packet-0f14c5ba8559b59e", "green pot-44adf4f26ae421e3", "pair of scissors-834f26061844bbb8", "one tortilla-0f4ea3fbede8dade", "tablespoon-146be3ceb5e109fb", "tortilla2-08a761ef36382b8b", "tortilla3-e64b604689a4a5f7", "tortilla4-0b2eea8e1e46f55a", "bottle of hand wash-ef36fb781d0a0800", "to be deleted-f4429bbe75b27172", "chopping board-7f9e37507f3b74d6", "one jar of condiment-87949691c754fc71", "second jar of condiment-33803082586cb2d4", "spice jar-a8484868b015c7c7", "bottile of oil-d581dc7c7b12dad1", "jar of peanut butter-0d6f31058f4f5313", "jar of onion powder-580a22d89982f7d0", "jar3-283bdf4381c97c30", "measuring cup-df406806d0744100", "stack of plastic containers-2621dbf5f5bee38d", "one empty container-dcd8061fd10d0cc0", "lid of container-0789ea2242217180" ], "reasoning": "1. First, I looked for the user to add a final ingredient to the sauce and saw them squeeze lemon juice on it beginning at 05:20. \n2. Next, I looked for the user to place a final object in the front leftmost spot beneath the utensil drawer. I identified the cabinet space housing various spice jars beneath the utensil drawer when the user opens the utensil drawer for the first time at 02:21.\n3. Then, I looked for the last item the user places in the designated spot in that cabinet. After rearranging a few spice jars , the user places a spice jar with a yellow lid (jar of onion powder-580a22d89982f7d0) in the front leftmost spot of the cabinet at 19:31. \n4. I then went back to locate the 3rd object behind that spice jar (jar of onion powder-580a22d89982f7d0) and saw the user move another spice jar with a yellow lid back from the front leftmost spot in the cabinet at 19:27. The user places a spice jar with an orange lid (spice jar-a8484868b015c7c7) directly in front of it, then places a spice jar with a green lid directly in front of the spice jar with the orange lid (spice jar-a8484868b015c7c7) at 19:28, followed by the final spice jar (jar of onion powder-580a22d89982f7d0) in the front leftmost spot at 19:31. The lids of the spice jars, beginning with the one in the front leftmost spot, are now as follows: yellow, green, orange, yellow. This means that the 3rd spice jar behind the jar in the front leftmost spot in the cabinet has a yellow lid. \n5. I realized that both spice jars (jar of onion powder-580a22d89982f7d0) in question and the lemon juice contain the color yellow.", "referenced_object_masks": [ "jar of onion powder-580a22d89982f7d0", "spice jar-a8484868b015c7c7" ] }, { "key": "P04-20240413-151722:77b58b63d2d8b81dfb3c7fd389a5a279f4cf594f", "video_id": "P04-20240413-151722", "question": "When I first touched the bag of tortillas, approximately how many degrees clockwise would I turn if I wanted to face the trash can?", "answer_choice_0": "180", "answer_choice_1": "45", "answer_choice_2": "90", "answer_choice_3": "135", "answer_choice_4": "225", "answer_id": 0, "answer": "180", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Numerical Reasoning", "Object Recognition", "Situational Awareness", "Temporal Reasoning" ], "objects_of_interest": [ "phone-ecc69da84e7148da", "pan-eab9c855e0ae0589", "bottle of liquid hand wash-8d89bdb34efd9f67", "spatula-1a056d3a50c064d8", "bowl of shredded lettuce-4da0fbbb88a59879", "conical shaped bowl-5c555508821a8c2f", "plate-4770066f89e13f46", "bowl with red things-4749bcd20d982d88", "bowl with white things-78b04db4fb1eb53e", "scale-39c0115d09bbe157", "lemon-5f3fa2657a347328", "knife-013d81f2be23e243", "one half of cut lemon-b3d97568202d986b", "marker-57ede0bdc38a6fae", "cap of pen-ae68b3022269afa5", "notebook-decc5af70076b9a0", "white serving bowl-60143ee5bdf403d5", "orange wiping cloth-c6e13d4a0c1214e8", "green capped condiment jar-dc0e8e11e1b9e817", "bottle of sriracha sauce-4d32d48f17b34718", "yogurt container-7b741ea2e8a6c56c", "jars of salt-693c973d627b71b0", "another jar-844a32fe7a1d41e1", "spoon-d98f0d94dbe38499", "kitchen rag-7375df10a21895b1", "marker cap-a53daf8135e813ac", "spoon3-13867cc128c5fffa", "cap of jar-3d337eea190bcfab", "cap of another jar-6af43e6480d12e0b", "cap of parsley jar-53d8918d3698e85c", "weighing scale-df94520d94ec9c1d", "spoon5-1f557948e67132ae", "spoon4-5db552d4404d6414", "fork-2c620840335d7411", "cap of salt jar-750d8ebecf2dc446", "jar of peppers-329b2f6766cbb049", "spoon6-b12571fd8ecbde5c", "plate2-28f1a5a953a71827", "packet-0f14c5ba8559b59e", "green pot-44adf4f26ae421e3", "pair of scissors-834f26061844bbb8", "one tortilla-0f4ea3fbede8dade", "tablespoon-146be3ceb5e109fb", "tortilla2-08a761ef36382b8b", "tortilla3-e64b604689a4a5f7", "tortilla4-0b2eea8e1e46f55a", "bottle of hand wash-ef36fb781d0a0800", "to be deleted-f4429bbe75b27172", "chopping board-7f9e37507f3b74d6", "one jar of condiment-87949691c754fc71", "second jar of condiment-33803082586cb2d4", "spice jar-a8484868b015c7c7", "bottile of oil-d581dc7c7b12dad1", "jar of peanut butter-0d6f31058f4f5313", "jar of onion powder-580a22d89982f7d0", "jar3-283bdf4381c97c30", "measuring cup-df406806d0744100", "stack of plastic containers-2621dbf5f5bee38d", "one empty container-dcd8061fd10d0cc0", "lid of container-0789ea2242217180" ], "reasoning": "1. I watched the video and looked for when the user first touched the bag of tortillas (packet-0f14c5ba8559b59e). This occurred at 07:56. \n2. I observed the environment at 07:56. I noticed that the user was facing the intersection where the cabinet door and counter meet such that the counter was about 45 degrees to their right. I also noted that the counter ended after the cabinet on the left side. At 08:01 I got a better view and could see the stove to the right of the cabinet where the user had taken the bag of tortillas (packet-0f14c5ba8559b59e). \n3. I looked for a trash can . At 08:28, I saw a trash can to the right of the user, across from the cutting board to the right of the stove. I confirmed it was a trash can when I saw the user throw away a piece of garbage at 08:30. \n4. If the user wanted to turn to see the trash can from where they were at 07:56, they would first turn 45 degrees clockwise to face the counter, then 90 degrees clockwise to face between the two counters, then approximately 45 degrees clockwise again to face the trash can across from the stove. I added these together: 45+90+45=180 degrees.", "referenced_object_masks": [ "packet-0f14c5ba8559b59e" ] }, { "key": "P04-20240413-151722:9a2878a0be5631fffc78807538cbc2092ae0537f", "video_id": "P04-20240413-151722", "question": "Which is larger, and by how much: the mean amount of time (with the final calculation rounded to the nearest 10 seconds) it took me to fill and wrap each burrito, or the mode of all times (with each actual duration rounded to the nearest 10 seconds) it took me to fill and wrap each burrito?", "answer_choice_0": "Neither--the numbers were the same", "answer_choice_1": "Mode--by 10 seconds", "answer_choice_2": "Mode--by 20 seconds", "answer_choice_3": "Mean--by 20 seconds", "answer_choice_4": "Mean--by 10 seconds", "answer_id": 4, "answer": "Mean--by 10 seconds", "question_type": [ "Action Recognition", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "phone-ecc69da84e7148da", "pan-eab9c855e0ae0589", "bottle of liquid hand wash-8d89bdb34efd9f67", "spatula-1a056d3a50c064d8", "bowl of shredded lettuce-4da0fbbb88a59879", "conical shaped bowl-5c555508821a8c2f", "plate-4770066f89e13f46", "bowl with red things-4749bcd20d982d88", "bowl with white things-78b04db4fb1eb53e", "scale-39c0115d09bbe157", "lemon-5f3fa2657a347328", "knife-013d81f2be23e243", "one half of cut lemon-b3d97568202d986b", "marker-57ede0bdc38a6fae", "cap of pen-ae68b3022269afa5", "notebook-decc5af70076b9a0", "white serving bowl-60143ee5bdf403d5", "orange wiping cloth-c6e13d4a0c1214e8", "green capped condiment jar-dc0e8e11e1b9e817", "bottle of sriracha sauce-4d32d48f17b34718", "yogurt container-7b741ea2e8a6c56c", "jars of salt-693c973d627b71b0", "another jar-844a32fe7a1d41e1", "spoon-d98f0d94dbe38499", "kitchen rag-7375df10a21895b1", "marker cap-a53daf8135e813ac", "spoon3-13867cc128c5fffa", "cap of jar-3d337eea190bcfab", "cap of another jar-6af43e6480d12e0b", "cap of parsley jar-53d8918d3698e85c", "weighing scale-df94520d94ec9c1d", "spoon5-1f557948e67132ae", "spoon4-5db552d4404d6414", "fork-2c620840335d7411", "cap of salt jar-750d8ebecf2dc446", "jar of peppers-329b2f6766cbb049", "spoon6-b12571fd8ecbde5c", "plate2-28f1a5a953a71827", "packet-0f14c5ba8559b59e", "green pot-44adf4f26ae421e3", "pair of scissors-834f26061844bbb8", "one tortilla-0f4ea3fbede8dade", "tablespoon-146be3ceb5e109fb", "tortilla2-08a761ef36382b8b", "tortilla3-e64b604689a4a5f7", "tortilla4-0b2eea8e1e46f55a", "bottle of hand wash-ef36fb781d0a0800", "to be deleted-f4429bbe75b27172", "chopping board-7f9e37507f3b74d6", "one jar of condiment-87949691c754fc71", "second jar of condiment-33803082586cb2d4", "spice jar-a8484868b015c7c7", "bottile of oil-d581dc7c7b12dad1", "jar of peanut butter-0d6f31058f4f5313", "jar of onion powder-580a22d89982f7d0", "jar3-283bdf4381c97c30", "measuring cup-df406806d0744100", "stack of plastic containers-2621dbf5f5bee38d", "one empty container-dcd8061fd10d0cc0", "lid of container-0789ea2242217180" ], "reasoning": "1. I watched the video and noted the segments in which a burrito was filled and wrapped. The user loaded ingredients onto the first tortilla (one tortilla-0f4ea3fbede8dade) and then wrapped it from 09:21-11:45. The user loaded ingredients onto the second tortilla (tortilla2-08a761ef36382b8b) and then wrapped it from 12:14-13:45. The user loaded ingredients onto a third tortilla (tortilla3-e64b604689a4a5f7) and then wrapped it from 14:00-15:33. The user loaded ingredients onto a fourth tortilla (tortilla4-0b2eea8e1e46f55a) and then wrapped it from 15:47-17:07. \n2. I found the duration for each burrito by subtracting the start time from the end time: 11:45-09:21=02:24 for the first; 13:45-12:14=01:31 for the second; 15:33-14:00=01:33 for the third; and 17:07-15:47=01:20 for the fourth. \n3. I converted the durations from minutes and seconds into seconds. The first burrito: (2*60)+24=144 seconds. The second burrito: (1*60)+31=91 seconds. The third burrito: (1*60)+33=93 seconds. The fourth burrito: (1*60)+20=80 seconds. \n4. I found the mean duration by adding all durations and dividing by number of occurrences: (144+91+93+80)/4=102 seconds. I rounded this to the nearest 10 seconds: 100 seconds. \n5. I rounded the durations to the nearest 10 seconds each to find the mode: 140 seconds; 90 seconds; 90 seconds; 80 seconds. Since 90 seconds occurs twice in the set, this is the mode. \n6. I found the difference between the mean and mode: 100-90=10 seconds.", "referenced_object_masks": [ "one tortilla-0f4ea3fbede8dade", "tortilla2-08a761ef36382b8b", "tortilla3-e64b604689a4a5f7", "tortilla4-0b2eea8e1e46f55a" ] }, { "key": "P04-20240413-151722:af0a04dd0fe168356a908022d049d4d191031821", "video_id": "P04-20240413-151722", "question": "If I rounded the approximate number of individual movements I made with the spoon when stirring the sauce to a multiple of 25, what would be the average number of movements I made with the spoon per second when stirring the sauce rounded to the nearest whole number?", "answer_choice_0": "1", "answer_choice_1": "4", "answer_choice_2": "2", "answer_choice_3": "5", "answer_choice_4": "3", "answer_id": 4, "answer": "3", "question_type": [ "Temporal Reasoning", "Action Recognition", "Numerical Reasoning", "Object Recognition", "Counting" ], "objects_of_interest": [ "phone-ecc69da84e7148da", "pan-eab9c855e0ae0589", "bottle of liquid hand wash-8d89bdb34efd9f67", "spatula-1a056d3a50c064d8", "bowl of shredded lettuce-4da0fbbb88a59879", "conical shaped bowl-5c555508821a8c2f", "plate-4770066f89e13f46", "bowl with red things-4749bcd20d982d88", "bowl with white things-78b04db4fb1eb53e", "scale-39c0115d09bbe157", "lemon-5f3fa2657a347328", "knife-013d81f2be23e243", "one half of cut lemon-b3d97568202d986b", "marker-57ede0bdc38a6fae", "cap of pen-ae68b3022269afa5", "notebook-decc5af70076b9a0", "white serving bowl-60143ee5bdf403d5", "orange wiping cloth-c6e13d4a0c1214e8", "green capped condiment jar-dc0e8e11e1b9e817", "bottle of sriracha sauce-4d32d48f17b34718", "yogurt container-7b741ea2e8a6c56c", "jars of salt-693c973d627b71b0", "another jar-844a32fe7a1d41e1", "spoon-d98f0d94dbe38499", "kitchen rag-7375df10a21895b1", "marker cap-a53daf8135e813ac", "spoon3-13867cc128c5fffa", "cap of jar-3d337eea190bcfab", "cap of another jar-6af43e6480d12e0b", "cap of parsley jar-53d8918d3698e85c", "weighing scale-df94520d94ec9c1d", "spoon5-1f557948e67132ae", "spoon4-5db552d4404d6414", "fork-2c620840335d7411", "cap of salt jar-750d8ebecf2dc446", "jar of peppers-329b2f6766cbb049", "spoon6-b12571fd8ecbde5c", "plate2-28f1a5a953a71827", "packet-0f14c5ba8559b59e", "green pot-44adf4f26ae421e3", "pair of scissors-834f26061844bbb8", "one tortilla-0f4ea3fbede8dade", "tablespoon-146be3ceb5e109fb", "tortilla2-08a761ef36382b8b", "tortilla3-e64b604689a4a5f7", "tortilla4-0b2eea8e1e46f55a", "bottle of hand wash-ef36fb781d0a0800", "to be deleted-f4429bbe75b27172", "chopping board-7f9e37507f3b74d6", "one jar of condiment-87949691c754fc71", "second jar of condiment-33803082586cb2d4", "spice jar-a8484868b015c7c7", "bottile of oil-d581dc7c7b12dad1", "jar of peanut butter-0d6f31058f4f5313", "jar of onion powder-580a22d89982f7d0", "jar3-283bdf4381c97c30", "measuring cup-df406806d0744100", "stack of plastic containers-2621dbf5f5bee38d", "one empty container-dcd8061fd10d0cc0", "lid of container-0789ea2242217180" ], "reasoning": "1. First, I identified the number of seconds the user spent stirring the sauce . I saw them begin stirring at 05:50 and end stirring at 06:35, for a total of 45 seconds. \n2. Next, I counted the number of times the user moved the spoon (spoon3-13867cc128c5fffa) in an individual direction between 05:50-06:35, and got a total of 123. \n3. I rounded 123 to the nearest multiple of 25 and got an approximate total of 125 individual movements with the spoon (spoon3-13867cc128c5fffa). \n4. Finally, I divided 125 by 45 and got 2.78, which I then rounded to the nearest whole number of 3.", "referenced_object_masks": [ "spoon3-13867cc128c5fffa" ] }, { "key": "P04-20240413-151722:b63063c8babeb6bd01076a3dfbf9618a49347e64", "video_id": "P04-20240413-151722", "question": "If I walked in through the door I opened and wanted to face the next to last place I put the largest plate, what would be my approximate trajectory?", "answer_choice_0": "Walk straight, turn left, walk straight, turn left", "answer_choice_1": "Walk straight, turn right, walk straight, turn right", "answer_choice_2": "Walk straight, turn right, walk straight, turn left", "answer_choice_3": "Turn left, walk straight, turn right, walk straight", "answer_choice_4": "Turn right, walk straight, turn left, walk straight", "answer_id": 0, "answer": "Walk straight, turn left, walk straight, turn left", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Object Recognition" ], "objects_of_interest": [ "phone-ecc69da84e7148da", "pan-eab9c855e0ae0589", "bottle of liquid hand wash-8d89bdb34efd9f67", "spatula-1a056d3a50c064d8", "bowl of shredded lettuce-4da0fbbb88a59879", "conical shaped bowl-5c555508821a8c2f", "plate-4770066f89e13f46", "bowl with red things-4749bcd20d982d88", "bowl with white things-78b04db4fb1eb53e", "scale-39c0115d09bbe157", "lemon-5f3fa2657a347328", "knife-013d81f2be23e243", "one half of cut lemon-b3d97568202d986b", "marker-57ede0bdc38a6fae", "cap of pen-ae68b3022269afa5", "notebook-decc5af70076b9a0", "white serving bowl-60143ee5bdf403d5", "orange wiping cloth-c6e13d4a0c1214e8", "green capped condiment jar-dc0e8e11e1b9e817", "bottle of sriracha sauce-4d32d48f17b34718", "yogurt container-7b741ea2e8a6c56c", "jars of salt-693c973d627b71b0", "another jar-844a32fe7a1d41e1", "spoon-d98f0d94dbe38499", "kitchen rag-7375df10a21895b1", "marker cap-a53daf8135e813ac", "spoon3-13867cc128c5fffa", "cap of jar-3d337eea190bcfab", "cap of another jar-6af43e6480d12e0b", "cap of parsley jar-53d8918d3698e85c", "weighing scale-df94520d94ec9c1d", "spoon5-1f557948e67132ae", "spoon4-5db552d4404d6414", "fork-2c620840335d7411", "cap of salt jar-750d8ebecf2dc446", "jar of peppers-329b2f6766cbb049", "spoon6-b12571fd8ecbde5c", "plate2-28f1a5a953a71827", "packet-0f14c5ba8559b59e", "green pot-44adf4f26ae421e3", "pair of scissors-834f26061844bbb8", "one tortilla-0f4ea3fbede8dade", "tablespoon-146be3ceb5e109fb", "tortilla2-08a761ef36382b8b", "tortilla3-e64b604689a4a5f7", "tortilla4-0b2eea8e1e46f55a", "bottle of hand wash-ef36fb781d0a0800", "to be deleted-f4429bbe75b27172", "chopping board-7f9e37507f3b74d6", "one jar of condiment-87949691c754fc71", "second jar of condiment-33803082586cb2d4", "spice jar-a8484868b015c7c7", "bottile of oil-d581dc7c7b12dad1", "jar of peanut butter-0d6f31058f4f5313", "jar of onion powder-580a22d89982f7d0", "jar3-283bdf4381c97c30", "measuring cup-df406806d0744100", "stack of plastic containers-2621dbf5f5bee38d", "one empty container-dcd8061fd10d0cc0", "lid of container-0789ea2242217180" ], "reasoning": "1. First, I looked for the user to open a door and saw them do this at 03:14. I took note of where this door is in the space and realized it is to the left of the long counter when the user is facing the counter. \n2. Next, I looked for the largest plate visible in the video. I identified this plate (plate2-28f1a5a953a71827) at 07:50 when the user takes it out of a cabinet and sets it down next to the other dishes (plate-4770066f89e13f46, bowl of shredded lettuce-4da0fbbb88a59879, conical shaped bowl-5c555508821a8c2f, bowl with red things-4749bcd20d982d88, bowl with white things-78b04db4fb1eb53e, white serving bowl-60143ee5bdf403d5). It (plate2-28f1a5a953a71827) is noticeably bigger than the rest. \n3. Then, I looked for the last place the user put this plate (plate2-28f1a5a953a71827). They place it near the sink at 17:28. \n4. I went back and located the plate (plate2-28f1a5a953a71827) in its location directly before 17:28. I saw that it was clearly on the counter to the right of the door the user opened at 17:24. I counted this as its next to last location. \n5. Reasoning about the scene, I imagined the user walking in through the door they opened at 03:14 and realized they would need to walk straight, turn left, walk straight, and turn left again in order to face the next to last location of the largest plate (plate2-28f1a5a953a71827).", "referenced_object_masks": [ "plate2-28f1a5a953a71827", "plate-4770066f89e13f46", "bowl of shredded lettuce-4da0fbbb88a59879", "conical shaped bowl-5c555508821a8c2f", "bowl with red things-4749bcd20d982d88", "bowl with white things-78b04db4fb1eb53e", "white serving bowl-60143ee5bdf403d5" ] }, { "key": "P04-20240413-151722:bf05ec4b5680770a85bbf401fb81da8a64f818f6", "video_id": "P04-20240413-151722", "question": "Not counting either half of the lemon, what was the 3rd unique food I touched after cutting the lemon?", "answer_choice_0": "Cheese", "answer_choice_1": "Yogurt", "answer_choice_2": "Sriracha", "answer_choice_3": "Lettuce", "answer_choice_4": "Salt", "answer_id": 1, "answer": "Yogurt", "question_type": [ "Object Recognition", "Counting", "Temporal Reasoning", "Action Recognition" ], "objects_of_interest": [ "phone-ecc69da84e7148da", "pan-eab9c855e0ae0589", "bottle of liquid hand wash-8d89bdb34efd9f67", "spatula-1a056d3a50c064d8", "bowl of shredded lettuce-4da0fbbb88a59879", "conical shaped bowl-5c555508821a8c2f", "plate-4770066f89e13f46", "bowl with red things-4749bcd20d982d88", "bowl with white things-78b04db4fb1eb53e", "scale-39c0115d09bbe157", "lemon-5f3fa2657a347328", "knife-013d81f2be23e243", "one half of cut lemon-b3d97568202d986b", "marker-57ede0bdc38a6fae", "cap of pen-ae68b3022269afa5", "notebook-decc5af70076b9a0", "white serving bowl-60143ee5bdf403d5", "orange wiping cloth-c6e13d4a0c1214e8", "green capped condiment jar-dc0e8e11e1b9e817", "bottle of sriracha sauce-4d32d48f17b34718", "yogurt container-7b741ea2e8a6c56c", "jars of salt-693c973d627b71b0", "another jar-844a32fe7a1d41e1", "spoon-d98f0d94dbe38499", "kitchen rag-7375df10a21895b1", "marker cap-a53daf8135e813ac", "spoon3-13867cc128c5fffa", "cap of jar-3d337eea190bcfab", "cap of another jar-6af43e6480d12e0b", "cap of parsley jar-53d8918d3698e85c", "weighing scale-df94520d94ec9c1d", "spoon5-1f557948e67132ae", "spoon4-5db552d4404d6414", "fork-2c620840335d7411", "cap of salt jar-750d8ebecf2dc446", "jar of peppers-329b2f6766cbb049", "spoon6-b12571fd8ecbde5c", "plate2-28f1a5a953a71827", "packet-0f14c5ba8559b59e", "green pot-44adf4f26ae421e3", "pair of scissors-834f26061844bbb8", "one tortilla-0f4ea3fbede8dade", "tablespoon-146be3ceb5e109fb", "tortilla2-08a761ef36382b8b", "tortilla3-e64b604689a4a5f7", "tortilla4-0b2eea8e1e46f55a", "bottle of hand wash-ef36fb781d0a0800", "to be deleted-f4429bbe75b27172", "chopping board-7f9e37507f3b74d6", "one jar of condiment-87949691c754fc71", "second jar of condiment-33803082586cb2d4", "spice jar-a8484868b015c7c7", "bottile of oil-d581dc7c7b12dad1", "jar of peanut butter-0d6f31058f4f5313", "jar of onion powder-580a22d89982f7d0", "jar3-283bdf4381c97c30", "measuring cup-df406806d0744100", "stack of plastic containers-2621dbf5f5bee38d", "one empty container-dcd8061fd10d0cc0", "lid of container-0789ea2242217180" ], "reasoning": "1. I watched the video and saw the user cut the lemon (lemon-5f3fa2657a347328) from 00:51-00:52. \n2. I counted the first 3 unique food objects that the user handled after cutting the lemon (lemon-5f3fa2657a347328) at 00:52. These items were: a green condiment bottle (green capped condiment jar-dc0e8e11e1b9e817) at 01:32, a bottle of sriracha (bottle of sriracha sauce-4d32d48f17b34718) at 01:58, and a container of yogurt (yogurt container-7b741ea2e8a6c56c) at 02:05.\n3. Therefore, the third unique food item touched by the user after cutting the lemon (lemon-5f3fa2657a347328) is the container of yogurt (yogurt container-7b741ea2e8a6c56c).", "referenced_object_masks": [ "bottle of sriracha sauce-4d32d48f17b34718", "green capped condiment jar-dc0e8e11e1b9e817", "lemon-5f3fa2657a347328", "yogurt container-7b741ea2e8a6c56c" ] }, { "key": "P04-20240413-151722:cfb297c2f5a4131bbac77d10e91e80cfe66b3734", "video_id": "P04-20240413-151722", "question": "Which was greater, and by how much: The number of burritos rolled or the number of scoops of yogurt I used in the sauce?", "answer_choice_0": "Burritos--by 1", "answer_choice_1": "Neither--the numbers were the same", "answer_choice_2": "Scoops of yogurt--by 1", "answer_choice_3": "Scoops of yogurt--by 2", "answer_choice_4": "Burritos--by 2", "answer_id": 2, "answer": "Scoops of yogurt--by 1", "question_type": [ "Action Recognition", "Object Recognition", "Counting", "Numerical Reasoning", "Temporal Reasoning" ], "objects_of_interest": [ "phone-ecc69da84e7148da", "pan-eab9c855e0ae0589", "bottle of liquid hand wash-8d89bdb34efd9f67", "spatula-1a056d3a50c064d8", "bowl of shredded lettuce-4da0fbbb88a59879", "conical shaped bowl-5c555508821a8c2f", "plate-4770066f89e13f46", "bowl with red things-4749bcd20d982d88", "bowl with white things-78b04db4fb1eb53e", "scale-39c0115d09bbe157", "lemon-5f3fa2657a347328", "knife-013d81f2be23e243", "one half of cut lemon-b3d97568202d986b", "marker-57ede0bdc38a6fae", "cap of pen-ae68b3022269afa5", "notebook-decc5af70076b9a0", "white serving bowl-60143ee5bdf403d5", "orange wiping cloth-c6e13d4a0c1214e8", "green capped condiment jar-dc0e8e11e1b9e817", "bottle of sriracha sauce-4d32d48f17b34718", "yogurt container-7b741ea2e8a6c56c", "jars of salt-693c973d627b71b0", "another jar-844a32fe7a1d41e1", "spoon-d98f0d94dbe38499", "kitchen rag-7375df10a21895b1", "marker cap-a53daf8135e813ac", "spoon3-13867cc128c5fffa", "cap of jar-3d337eea190bcfab", "cap of another jar-6af43e6480d12e0b", "cap of parsley jar-53d8918d3698e85c", "weighing scale-df94520d94ec9c1d", "spoon5-1f557948e67132ae", "spoon4-5db552d4404d6414", "fork-2c620840335d7411", "cap of salt jar-750d8ebecf2dc446", "jar of peppers-329b2f6766cbb049", "spoon6-b12571fd8ecbde5c", "plate2-28f1a5a953a71827", "packet-0f14c5ba8559b59e", "green pot-44adf4f26ae421e3", "pair of scissors-834f26061844bbb8", "one tortilla-0f4ea3fbede8dade", "tablespoon-146be3ceb5e109fb", "tortilla2-08a761ef36382b8b", "tortilla3-e64b604689a4a5f7", "tortilla4-0b2eea8e1e46f55a", "bottle of hand wash-ef36fb781d0a0800", "to be deleted-f4429bbe75b27172", "chopping board-7f9e37507f3b74d6", "one jar of condiment-87949691c754fc71", "second jar of condiment-33803082586cb2d4", "spice jar-a8484868b015c7c7", "bottile of oil-d581dc7c7b12dad1", "jar of peanut butter-0d6f31058f4f5313", "jar of onion powder-580a22d89982f7d0", "jar3-283bdf4381c97c30", "measuring cup-df406806d0744100", "stack of plastic containers-2621dbf5f5bee38d", "one empty container-dcd8061fd10d0cc0", "lid of container-0789ea2242217180" ], "reasoning": "1. I watched the video and counted each tortilla that was filled and rolled into a burrito. The first tortilla (one tortilla-0f4ea3fbede8dade) was filled and wrapped from 09:21-11:45. The second tortilla (tortilla2-08a761ef36382b8b) was filled and wrapped from 12:14-13:45. The third tortilla (tortilla3-e64b604689a4a5f7) was filled and wrapped from 14:00-15:33. The fourth tortilla (tortilla4-0b2eea8e1e46f55a) was filled and wrapped from 15:47-17:07. This makes a total of 4 burritos. \n2. I looked for when the user scooped spoonfuls from the container of yogurt (yogurt container-7b741ea2e8a6c56c) into a sauce and found that the ingredients for the sauce were added from 02:39-05:38 and it was mixed from 05:51-06:33. I noted the time of each scoop from the container of yogurt (yogurt container-7b741ea2e8a6c56c): 02:39-02:42, 02:43-02:44, 02:45-02:46, 02:47-02:48, and 02:50-02:52. This makes a total of 5 occurrences. \n3. I found the difference: 5-4=1.", "referenced_object_masks": [ "one tortilla-0f4ea3fbede8dade", "tortilla2-08a761ef36382b8b", "tortilla3-e64b604689a4a5f7", "yogurt container-7b741ea2e8a6c56c", "tortilla4-0b2eea8e1e46f55a" ] }, { "key": "P04-20240413-151722:d5a666cb7e8c5d24b472da8b6509efbcf8baf029", "video_id": "P04-20240413-151722", "question": "If I multiplied the number of lines of text on the notepad when I open the utensil drawer for a second time by the weight of the bowl on the scale immediately after I add the spoon, what number would I get?", "answer_choice_0": "925", "answer_choice_1": "759", "answer_choice_2": "1136", "answer_choice_3": "1342", "answer_choice_4": "364", "answer_id": 2, "answer": "1136", "question_type": [ "Action Recognition", "Counting", "Numerical Reasoning", "Object Recognition", "Reading" ], "objects_of_interest": [ "phone-ecc69da84e7148da", "pan-eab9c855e0ae0589", "bottle of liquid hand wash-8d89bdb34efd9f67", "spatula-1a056d3a50c064d8", "bowl of shredded lettuce-4da0fbbb88a59879", "conical shaped bowl-5c555508821a8c2f", "plate-4770066f89e13f46", "bowl with red things-4749bcd20d982d88", "bowl with white things-78b04db4fb1eb53e", "scale-39c0115d09bbe157", "lemon-5f3fa2657a347328", "knife-013d81f2be23e243", "one half of cut lemon-b3d97568202d986b", "marker-57ede0bdc38a6fae", "cap of pen-ae68b3022269afa5", "notebook-decc5af70076b9a0", "white serving bowl-60143ee5bdf403d5", "orange wiping cloth-c6e13d4a0c1214e8", "green capped condiment jar-dc0e8e11e1b9e817", "bottle of sriracha sauce-4d32d48f17b34718", "yogurt container-7b741ea2e8a6c56c", "jars of salt-693c973d627b71b0", "another jar-844a32fe7a1d41e1", "spoon-d98f0d94dbe38499", "kitchen rag-7375df10a21895b1", "marker cap-a53daf8135e813ac", "spoon3-13867cc128c5fffa", "cap of jar-3d337eea190bcfab", "cap of another jar-6af43e6480d12e0b", "cap of parsley jar-53d8918d3698e85c", "weighing scale-df94520d94ec9c1d", "spoon5-1f557948e67132ae", "spoon4-5db552d4404d6414", "fork-2c620840335d7411", "cap of salt jar-750d8ebecf2dc446", "jar of peppers-329b2f6766cbb049", "spoon6-b12571fd8ecbde5c", "plate2-28f1a5a953a71827", "packet-0f14c5ba8559b59e", "green pot-44adf4f26ae421e3", "pair of scissors-834f26061844bbb8", "one tortilla-0f4ea3fbede8dade", "tablespoon-146be3ceb5e109fb", "tortilla2-08a761ef36382b8b", "tortilla3-e64b604689a4a5f7", "tortilla4-0b2eea8e1e46f55a", "bottle of hand wash-ef36fb781d0a0800", "to be deleted-f4429bbe75b27172", "chopping board-7f9e37507f3b74d6", "one jar of condiment-87949691c754fc71", "second jar of condiment-33803082586cb2d4", "spice jar-a8484868b015c7c7", "bottile of oil-d581dc7c7b12dad1", "jar of peanut butter-0d6f31058f4f5313", "jar of onion powder-580a22d89982f7d0", "jar3-283bdf4381c97c30", "measuring cup-df406806d0744100", "stack of plastic containers-2621dbf5f5bee38d", "one empty container-dcd8061fd10d0cc0", "lid of container-0789ea2242217180" ], "reasoning": "1. First, I located the moment where the user opens the utensil drawer for the 2nd time. I saw them open the utensil drawer for the 1st time at 02:22, and for the 2nd time at 03:34. \n2. Next, I looked for a clear view of the notepad (notebook-decc5af70076b9a0), and found one as the user writes on it at 03:21, adding 1 additional line of text to the page. I counted up the number of lines of text on the page after this occurred and got 8. Since the user doesn't write anything else on the notepad (notebook-decc5af70076b9a0) between 03:21 and 03:34, I knew there were 8 lines of text on the notepad when the user opens the utensil drawer for the 2nd time. \n3. Then, I looked for the user to add a spoon (spoon3-13867cc128c5fffa) to a bowl (white serving bowl-60143ee5bdf403d5) being weighed. I saw this occur at 04:00. \n4. I looked at the numbers on the scale (scale-39c0115d09bbe157) at 04:00 and read 142. \n5. Finally, I computed the equation 142*8 and got a total of 1136.", "referenced_object_masks": [ "white serving bowl-60143ee5bdf403d5", "spoon3-13867cc128c5fffa", "scale-39c0115d09bbe157", "notebook-decc5af70076b9a0" ] }, { "key": "P04-20240414-064514:22f70c34e33f3d1e5b38366f1dd93a161945cc9b", "video_id": "P04-20240414-064514", "question": "If the letter 'i' on the egg carton was replaced by the first letter of the word on the blue container next to the trash can, what would the new word be?", "answer_choice_0": "Ogg", "answer_choice_1": "Bug", "answer_choice_2": "Beg", "answer_choice_3": "Bag", "answer_choice_4": "End", "answer_id": 3, "answer": "Bag", "question_type": [ "Counterfactual", "Reading", "Situational Awareness", "Object Recognition" ], "objects_of_interest": [ "mug-9be005686c417a8d", "plastic container-eedcac33cde5c12a", "plastic bag-30588be093589da9", "kitchen rag-d78f03ed85b6c128", "carton of eggs-d5644ddbb42c974f", "bottle of olive oil-1872c07ecc827c83", "jar-c3b396c2b383a39a", "big cover of chips-e064cbea9bbf1a39", "condiment jar-21d7bf6d3a80ab8f", "pan-e1f8bb3c85d6f824", "large knife-05f967f427632bdf", "tablespoon-7e14a60dbf1261cf", "stack of plastic container boxes-af96ed19230c0fcc", "green teabag-459fc4c4fa4fe031", "one plastic container-22ee3c65c00da1e3", "one medium sized egg-658e41f49024845e", "rubber spatula-8805b8313e0d63ce", "weighing scale-0f85d2facd58d3a8" ], "reasoning": "1. I identified the egg carton (carton of eggs-d5644ddbb42c974f) at 00:40. The single letter 'i' is present in the word, \"Big.\"\n2. I observed the blue container next to the trash can at 00:19 when the user throws away a plastic bag (plastic bag-30588be093589da9). At this time, the first letter, \"A,\" of the word on the container is visible.\n3. If the 'i' in 'Big' were an 'a' instead, the new word would be 'bag.'", "referenced_object_masks": [ "plastic bag-30588be093589da9", "carton of eggs-d5644ddbb42c974f" ] }, { "key": "P04-20240414-064514:32839685152e80a1f0420b85facae633fbdd2540", "video_id": "P04-20240414-064514", "question": "How much longer would I need to run water over my hands to equal the amount of time I spent retrieving the whisk and the spatula from the moment I opened the drawer they were in?", "answer_choice_0": "10 seconds", "answer_choice_1": "15 seconds", "answer_choice_2": "8 seconds", "answer_choice_3": "12 seconds", "answer_choice_4": "17 seconds", "answer_id": 1, "answer": "15 seconds", "question_type": [ "Action Recognition", "Numerical Reasoning", "Temporal Reasoning", "Object Recognition" ], "objects_of_interest": [ "mug-9be005686c417a8d", "plastic container-eedcac33cde5c12a", "plastic bag-30588be093589da9", "kitchen rag-d78f03ed85b6c128", "carton of eggs-d5644ddbb42c974f", "bottle of olive oil-1872c07ecc827c83", "jar-c3b396c2b383a39a", "big cover of chips-e064cbea9bbf1a39", "condiment jar-21d7bf6d3a80ab8f", "pan-e1f8bb3c85d6f824", "large knife-05f967f427632bdf", "tablespoon-7e14a60dbf1261cf", "stack of plastic container boxes-af96ed19230c0fcc", "green teabag-459fc4c4fa4fe031", "one plastic container-22ee3c65c00da1e3", "one medium sized egg-658e41f49024845e", "rubber spatula-8805b8313e0d63ce", "weighing scale-0f85d2facd58d3a8" ], "reasoning": "1. The user began rinsing their hands at 03:00. The user then removed their hands from the stream of water at 03:05. They spent 5 seconds rinsing their hands.\n2. I found the moment the user pulls out a spatula (rubber spatula-8805b8313e0d63ce) at 02:38 and the moment the user pulls out a whisk at 02:45.\n3. To mark the beginning of the user's search for these 2 items, I went to the moment when the user opened up the drawer at 02:25. From here to 02:45 totals 20 seconds.\n4. The user would have to rinse their hands for 15 additional seconds to reach the time they spent, 20 seconds, looking for both the whisk and the spatula (rubber spatula-8805b8313e0d63ce).", "referenced_object_masks": [ "rubber spatula-8805b8313e0d63ce" ] }, { "key": "P04-20240414-064514:8d817418e68b00efcc5551469b4a600f8b37be01", "video_id": "P04-20240414-064514", "question": "What is the product of the amount of times I touched an egg and the number of items I retrieved from the silverware cabinet and then placed on the cutting board?", "answer_choice_0": "21", "answer_choice_1": "27", "answer_choice_2": "18", "answer_choice_3": "30", "answer_choice_4": "16", "answer_id": 1, "answer": "27", "question_type": [ "Numerical Reasoning", "Counting", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "mug-9be005686c417a8d", "plastic container-eedcac33cde5c12a", "plastic bag-30588be093589da9", "kitchen rag-d78f03ed85b6c128", "carton of eggs-d5644ddbb42c974f", "bottle of olive oil-1872c07ecc827c83", "jar-c3b396c2b383a39a", "big cover of chips-e064cbea9bbf1a39", "condiment jar-21d7bf6d3a80ab8f", "pan-e1f8bb3c85d6f824", "large knife-05f967f427632bdf", "tablespoon-7e14a60dbf1261cf", "stack of plastic container boxes-af96ed19230c0fcc", "green teabag-459fc4c4fa4fe031", "one plastic container-22ee3c65c00da1e3", "one medium sized egg-658e41f49024845e", "rubber spatula-8805b8313e0d63ce", "weighing scale-0f85d2facd58d3a8" ], "reasoning": "1. I observed that the user touched 2 eggs at 03:31, the right egg (one medium sized egg-658e41f49024845e) and the left egg . The user touched the left egg again at 03:39. This totals 3 times that the user touched an egg.\n2. I first identified the silverware within a cabinet at 01:32. This is the cabinet at the end of the kitchen counter. I went to the beginning of the video to count every item that the user takes out of this cabinet.\n3. Between 00:36 and the end of the video, the user retrieves 9 items from this cabinet and places them on the cutting board: a carton of eggs (carton of eggs-d5644ddbb42c974f) from 00:36-00:40, a bottle of oil (bottle of olive oil-1872c07ecc827c83) and jar (jar-c3b396c2b383a39a) from 00:44-00:50, another jar (condiment jar-21d7bf6d3a80ab8f) from 00:57-01:07, a pan (pan-e1f8bb3c85d6f824) from 01:20-01:26, a knife (large knife-05f967f427632bdf) and spoon (tablespoon-7e14a60dbf1261cf) from 01:33-01:55, a plastic container (one plastic container-22ee3c65c00da1e3) from 02:02-02:09, and a scale (weighing scale-0f85d2facd58d3a8) from 03:59-04:10.\n4. The product of 3 multiplied by 9 is 27.", "referenced_object_masks": [ "jar-c3b396c2b383a39a", "weighing scale-0f85d2facd58d3a8", "condiment jar-21d7bf6d3a80ab8f", "pan-e1f8bb3c85d6f824", "tablespoon-7e14a60dbf1261cf", "bottle of olive oil-1872c07ecc827c83", "one medium sized egg-658e41f49024845e", "large knife-05f967f427632bdf", "one plastic container-22ee3c65c00da1e3", "carton of eggs-d5644ddbb42c974f" ] }, { "key": "P04-20240414-064514:b9f06725a807f8fd3bf8fd51133bd35a8264b274", "video_id": "P04-20240414-064514", "question": "If I were sitting on one of the stools with my knees under the counter, which direction would I need to go to dispose of garbage?", "answer_choice_0": "Behind, then right", "answer_choice_1": "Left, then left", "answer_choice_2": "Left", "answer_choice_3": "behind", "answer_choice_4": "Right, then right", "answer_id": 2, "answer": "Left", "question_type": [ "Counterfactual", "Egocentric & Spatial Perception", "Situational Awareness" ], "objects_of_interest": [ "mug-9be005686c417a8d", "plastic container-eedcac33cde5c12a", "plastic bag-30588be093589da9", "kitchen rag-d78f03ed85b6c128", "carton of eggs-d5644ddbb42c974f", "bottle of olive oil-1872c07ecc827c83", "jar-c3b396c2b383a39a", "big cover of chips-e064cbea9bbf1a39", "condiment jar-21d7bf6d3a80ab8f", "pan-e1f8bb3c85d6f824", "large knife-05f967f427632bdf", "tablespoon-7e14a60dbf1261cf", "stack of plastic container boxes-af96ed19230c0fcc", "green teabag-459fc4c4fa4fe031", "one plastic container-22ee3c65c00da1e3", "one medium sized egg-658e41f49024845e", "rubber spatula-8805b8313e0d63ce", "weighing scale-0f85d2facd58d3a8" ], "reasoning": "1. I identified the stools at 00:03 along the walls. I continued watching to find the trash can.\n2. At 00:19, the user places the plastic bag (plastic bag-30588be093589da9) into the trash can .\n3. This view shows that the trash can is positioned to the left of the stools. If the user were sitting on the stools with their knees under the counter, they would have to turn to the left to dispose of garbage.", "referenced_object_masks": [ "plastic bag-30588be093589da9" ] }, { "key": "P04-20240414-064514:d1a4472ade509f0842f81a994f867001c108a33d", "video_id": "P04-20240414-064514", "question": "Which item from the cabinets did I take out but didn't place on the kitchen counter?", "answer_choice_0": "Frying pan", "answer_choice_1": "Tablespoon", "answer_choice_2": "Empty plastic container", "answer_choice_3": "Fork", "answer_choice_4": "Spatula", "answer_id": 2, "answer": "Empty plastic container", "question_type": [ "Counterfactual", "Action Recognition", "Object Recognition" ], "objects_of_interest": [ "mug-9be005686c417a8d", "plastic container-eedcac33cde5c12a", "plastic bag-30588be093589da9", "kitchen rag-d78f03ed85b6c128", "carton of eggs-d5644ddbb42c974f", "bottle of olive oil-1872c07ecc827c83", "jar-c3b396c2b383a39a", "big cover of chips-e064cbea9bbf1a39", "condiment jar-21d7bf6d3a80ab8f", "pan-e1f8bb3c85d6f824", "large knife-05f967f427632bdf", "tablespoon-7e14a60dbf1261cf", "stack of plastic container boxes-af96ed19230c0fcc", "green teabag-459fc4c4fa4fe031", "one plastic container-22ee3c65c00da1e3", "one medium sized egg-658e41f49024845e", "rubber spatula-8805b8313e0d63ce", "weighing scale-0f85d2facd58d3a8" ], "reasoning": "1. I observed the user taking out items from the kitchen cabinets from the first item, the egg carton (carton of eggs-d5644ddbb42c974f) at 00:36 to the final item before the end of the video, the weighing scale (weighing scale-0f85d2facd58d3a8) at 04:01, for a total of 11 items.\n2. Of all the items that the user took out from the cabinets, the user did not place the empty plastic container from the stack of containers (stack of plastic container boxes-af96ed19230c0fcc) they retrieved at 02:05 onto the counter. Instead, at 02:12, the user returned the empty plastic container to the cabinet.", "referenced_object_masks": [ "stack of plastic container boxes-af96ed19230c0fcc", "carton of eggs-d5644ddbb42c974f", "weighing scale-0f85d2facd58d3a8" ] }, { "key": "P04-20240414-065311:1d9e924830fbcfd3e079398c0ce081e7297a562d", "video_id": "P04-20240414-065311", "question": "At what point in the video do I touch the largest, bowl-shaped kitchen tool positioned facedown in the first drawer?", "answer_choice_0": "20:24", "answer_choice_1": "04:48", "answer_choice_2": "06:11", "answer_choice_3": "06:17", "answer_choice_4": "19:29", "answer_id": 0, "answer": "20:24", "question_type": [ "Temporal Reasoning", "Action Recognition", "Object Recognition", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "notebook-02bdb15c5e0fe283", "marker-682b9206618c53be", "empty plastic container box-1c6baa284943ffa8", "plastic container-f5a4e153cc240565", "egg2-da3705d64b0cffe9", "egg1-cb27293ba8f99a3e", "first egg-1f32c0240febded2", "egg3-cccb39acd90034b6", "egg4-f961f78612d57f68", "second egg-e049fd4ac12da049", "correct weighing scale-d5068055230c4da4", "eggshells-cbdae18f19ff20f4", "large knife-76546fac7fc5bad1", "third egg-ad5ab695a740c077", "egg carton-cd4102d37d9e2cac", "spoon-80c4076af3818b30", "fourth egg-c0d55cc89ef9b1d6", "yellow pepper-59e24eb29d22dfee", "box of mushrooms-303c6b9b622c66f2", "plastic bag-3967805d5a39a77d", "mushroom-dab7070b32dd5cbd", "green pepper-6e668c0ff6a22bf8", "chopping board-39f0463050297140", "yellow pepper 8-5dc836af44798409", "mushroom container-87e64f60098d47c8", "empty plastic cover-28154996946f5f20", "plastic box2-91b76314a1ff9d22", "plastic box3-851b6f056afd363e", "whisk-324e2e3f6084dbb4", "electric hob-5d5feb3826ca8501", "rubber spatula-767954c6cde5bc3a", "bowl2-12586cd531b79e58", "lid-a2e8a464602ab8fc", "bowl-2cea3a2c933a4ba5", "salt jar-0e6e4e8b4291f578", "pepper jar-182250f0df199a39", "plastic container in fridge-7cbe7ad891857724", "plastic container in fridge 2-b0cffd3396cb45ea", "plastic container in fridge 3-7c1d91d305a6ad2a", "tablespoon-87e1e2ec75c7d8df", "teaspoon2-d392be23ab3210c1", "colander-f5d61e09a407b36b", "cap of milk carton-c3b52e4fc0eeb284", "plastic cup-4085e245d92fe5a3", "correct whisk-0104fa3069f8860d", "plug of electric hob-174bbeb2af19ffa9", "jar of cinnamon powder-a76839f873a09fa7", "cap of olive oil bottle-31debbd52b845f6c", "spoon3-54d498d956a68225", "plate-0100d730b65fb951", "orange kitchen towel-703e761eeafcb0bc", "plate2-97dcc7e125e351e0", "another kitchen towel-b8aca97c478f2ff7", "pan-f987efd8203dbd58", "weighing scale-80a15ce1d41e5389", "coffee mug-7b3820fb398bf3da", "sponge-8d81f29a1a164899", "spoon holder-23c52c09d84ce83c", "white cap of plastic container-6398e298025d38c1" ], "reasoning": "1. The user pulls open the first drawer at 06:10, where an array of objects are visible. \n2. I recognize the largest, bowl-shaped item is a silver strainer (colander-f5d61e09a407b36b), positioned facedown.\n3. I continue watching the video to determine when the strainer (colander-f5d61e09a407b36b) is touched. This occurs at 20:24 when the user lifts the strainer (colander-f5d61e09a407b36b) while searching for the measuring cup .", "referenced_object_masks": [ "colander-f5d61e09a407b36b" ] }, { "key": "P04-20240414-065311:33aab9a8c01bbea445b4e45c0925ffa5c7d452bd", "video_id": "P04-20240414-065311", "question": "Where is the refrigerator in relation to where I am standing at 15:37?", "answer_choice_0": "To my left", "answer_choice_1": "Directly in front of me", "answer_choice_2": "To my right", "answer_choice_3": "Behind me to my left", "answer_choice_4": "Behind me to my right", "answer_id": 0, "answer": "To my left", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness", "Temporal Reasoning" ], "objects_of_interest": [ "notebook-02bdb15c5e0fe283", "marker-682b9206618c53be", "empty plastic container box-1c6baa284943ffa8", "plastic container-f5a4e153cc240565", "egg2-da3705d64b0cffe9", "egg1-cb27293ba8f99a3e", "first egg-1f32c0240febded2", "egg3-cccb39acd90034b6", "egg4-f961f78612d57f68", "second egg-e049fd4ac12da049", "correct weighing scale-d5068055230c4da4", "eggshells-cbdae18f19ff20f4", "large knife-76546fac7fc5bad1", "third egg-ad5ab695a740c077", "egg carton-cd4102d37d9e2cac", "spoon-80c4076af3818b30", "fourth egg-c0d55cc89ef9b1d6", "yellow pepper-59e24eb29d22dfee", "box of mushrooms-303c6b9b622c66f2", "plastic bag-3967805d5a39a77d", "mushroom-dab7070b32dd5cbd", "green pepper-6e668c0ff6a22bf8", "chopping board-39f0463050297140", "yellow pepper 8-5dc836af44798409", "mushroom container-87e64f60098d47c8", "empty plastic cover-28154996946f5f20", "plastic box2-91b76314a1ff9d22", "plastic box3-851b6f056afd363e", "whisk-324e2e3f6084dbb4", "electric hob-5d5feb3826ca8501", "rubber spatula-767954c6cde5bc3a", "bowl2-12586cd531b79e58", "lid-a2e8a464602ab8fc", "bowl-2cea3a2c933a4ba5", "salt jar-0e6e4e8b4291f578", "pepper jar-182250f0df199a39", "plastic container in fridge-7cbe7ad891857724", "plastic container in fridge 2-b0cffd3396cb45ea", "plastic container in fridge 3-7c1d91d305a6ad2a", "tablespoon-87e1e2ec75c7d8df", "teaspoon2-d392be23ab3210c1", "colander-f5d61e09a407b36b", "cap of milk carton-c3b52e4fc0eeb284", "plastic cup-4085e245d92fe5a3", "correct whisk-0104fa3069f8860d", "plug of electric hob-174bbeb2af19ffa9", "jar of cinnamon powder-a76839f873a09fa7", "cap of olive oil bottle-31debbd52b845f6c", "spoon3-54d498d956a68225", "plate-0100d730b65fb951", "orange kitchen towel-703e761eeafcb0bc", "plate2-97dcc7e125e351e0", "another kitchen towel-b8aca97c478f2ff7", "pan-f987efd8203dbd58", "weighing scale-80a15ce1d41e5389", "coffee mug-7b3820fb398bf3da", "sponge-8d81f29a1a164899", "spoon holder-23c52c09d84ce83c", "white cap of plastic container-6398e298025d38c1" ], "reasoning": "1. I watched the whole video. At 04:15, I see the user turns left from the cutting board (chopping board-39f0463050297140) and walks to the refrigerator . \n2. At 15:37, the user is cooking the eggs and vegetables (coffee mug-7b3820fb398bf3da) in a frying pan (pan-f987efd8203dbd58) on the electric hob (electric hob-5d5feb3826ca8501) . \n3. At 15:37, I can see the refrigerator to their left.", "referenced_object_masks": [ "chopping board-39f0463050297140", "pan-f987efd8203dbd58", "coffee mug-7b3820fb398bf3da", "electric hob-5d5feb3826ca8501" ] }, { "key": "P04-20240414-065311:4bcd5223267bc021339c7064758947e12ef227a3", "video_id": "P04-20240414-065311", "question": "How many more times did I write in my notebook than getting food items out of the refrigerator or putting them back?", "answer_choice_0": "1", "answer_choice_1": "2", "answer_choice_2": "3", "answer_choice_3": "4", "answer_choice_4": "0", "answer_id": 0, "answer": "1", "question_type": [ "Action Recognition", "Counting", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "notebook-02bdb15c5e0fe283", "marker-682b9206618c53be", "empty plastic container box-1c6baa284943ffa8", "plastic container-f5a4e153cc240565", "egg2-da3705d64b0cffe9", "egg1-cb27293ba8f99a3e", "first egg-1f32c0240febded2", "egg3-cccb39acd90034b6", "egg4-f961f78612d57f68", "second egg-e049fd4ac12da049", "correct weighing scale-d5068055230c4da4", "eggshells-cbdae18f19ff20f4", "large knife-76546fac7fc5bad1", "third egg-ad5ab695a740c077", "egg carton-cd4102d37d9e2cac", "spoon-80c4076af3818b30", "fourth egg-c0d55cc89ef9b1d6", "yellow pepper-59e24eb29d22dfee", "box of mushrooms-303c6b9b622c66f2", "plastic bag-3967805d5a39a77d", "mushroom-dab7070b32dd5cbd", "green pepper-6e668c0ff6a22bf8", "chopping board-39f0463050297140", "yellow pepper 8-5dc836af44798409", "mushroom container-87e64f60098d47c8", "empty plastic cover-28154996946f5f20", "plastic box2-91b76314a1ff9d22", "plastic box3-851b6f056afd363e", "whisk-324e2e3f6084dbb4", "electric hob-5d5feb3826ca8501", "rubber spatula-767954c6cde5bc3a", "bowl2-12586cd531b79e58", "lid-a2e8a464602ab8fc", "bowl-2cea3a2c933a4ba5", "salt jar-0e6e4e8b4291f578", "pepper jar-182250f0df199a39", "plastic container in fridge-7cbe7ad891857724", "plastic container in fridge 2-b0cffd3396cb45ea", "plastic container in fridge 3-7c1d91d305a6ad2a", "tablespoon-87e1e2ec75c7d8df", "teaspoon2-d392be23ab3210c1", "colander-f5d61e09a407b36b", "cap of milk carton-c3b52e4fc0eeb284", "plastic cup-4085e245d92fe5a3", "correct whisk-0104fa3069f8860d", "plug of electric hob-174bbeb2af19ffa9", "jar of cinnamon powder-a76839f873a09fa7", "cap of olive oil bottle-31debbd52b845f6c", "spoon3-54d498d956a68225", "plate-0100d730b65fb951", "orange kitchen towel-703e761eeafcb0bc", "plate2-97dcc7e125e351e0", "another kitchen towel-b8aca97c478f2ff7", "pan-f987efd8203dbd58", "weighing scale-80a15ce1d41e5389", "coffee mug-7b3820fb398bf3da", "sponge-8d81f29a1a164899", "spoon holder-23c52c09d84ce83c", "white cap of plastic container-6398e298025d38c1" ], "reasoning": "1. The user wrote in their notebook (notebook-02bdb15c5e0fe283) at 0:13, 04:18, 09:25, 09:56 and 19:52. \n2. So, the user wrote in their notebook (notebook-02bdb15c5e0fe283) 5 times. \n3. The user pulled out a yellow pepper (yellow pepper 8-5dc836af44798409) from the refrigerator at 04:53 and then put it back at 04:56. This is the first item the user took out of the fridge and put back. \n4. The user went to the refrigerator again at 5:05 and pulled a tray of mushrooms (box of mushrooms-303c6b9b622c66f2). The user grabbed one and put the tray (box of mushrooms-303c6b9b622c66f2) back in the fridge at 05:14, being this the second item the user took out of the fridge and put back.\n5. At 10:13, the user opens the fridge again and takes out two plastic containers (plastic container in fridge-7cbe7ad891857724, plastic container in fridge 3-7c1d91d305a6ad2a) at 10:17. At 10:20, the user takes out a third plastic container (plastic container in fridge 2-b0cffd3396cb45ea). At 10:22, the user puts the first two plastic containers (plastic container in fridge-7cbe7ad891857724, plastic container in fridge 3-7c1d91d305a6ad2a) back inside the fridge and one falls to the ground (plastic container in fridge 3-7c1d91d305a6ad2a). The user picks it up and puts the two containers (plastic container in fridge-7cbe7ad891857724, plastic container in fridge 3-7c1d91d305a6ad2a) back in the fridge at 10:26. These two containers are the third and fourth items the user takes out of the fridge and puts them back in. \n6. At 19:40, the user got a carton of milk out of the fridge , and put it back in the fridge at 23:24. This is the fourth and last item the user takes out of the fridge and puts it back. \n7. In conclusion, the user wrote on the notepad (notebook-02bdb15c5e0fe283) 5 times, and took and returned to the fridge 4 items. Therefore, 5 minus 4 equals 1.", "referenced_object_masks": [ "notebook-02bdb15c5e0fe283", "plastic container in fridge 2-b0cffd3396cb45ea", "plastic container in fridge-7cbe7ad891857724", "plastic container in fridge 3-7c1d91d305a6ad2a", "box of mushrooms-303c6b9b622c66f2", "yellow pepper 8-5dc836af44798409" ] }, { "key": "P04-20240414-065311:5ddc722c94ba5757954df51fcbbe6118f891df09", "video_id": "P04-20240414-065311", "question": "What item did I put away at the beginning of the video that I eventually needed later?", "answer_choice_0": "Pepper", "answer_choice_1": "Onion", "answer_choice_2": "Yellow Pepper", "answer_choice_3": "Oil", "answer_choice_4": "Salt", "answer_id": 3, "answer": "Oil", "question_type": [ "Egocentric & Spatial Perception", "Event Occurence", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "notebook-02bdb15c5e0fe283", "marker-682b9206618c53be", "empty plastic container box-1c6baa284943ffa8", "plastic container-f5a4e153cc240565", "egg2-da3705d64b0cffe9", "egg1-cb27293ba8f99a3e", "first egg-1f32c0240febded2", "egg3-cccb39acd90034b6", "egg4-f961f78612d57f68", "second egg-e049fd4ac12da049", "correct weighing scale-d5068055230c4da4", "eggshells-cbdae18f19ff20f4", "large knife-76546fac7fc5bad1", "third egg-ad5ab695a740c077", "egg carton-cd4102d37d9e2cac", "spoon-80c4076af3818b30", "fourth egg-c0d55cc89ef9b1d6", "yellow pepper-59e24eb29d22dfee", "box of mushrooms-303c6b9b622c66f2", "plastic bag-3967805d5a39a77d", "mushroom-dab7070b32dd5cbd", "green pepper-6e668c0ff6a22bf8", "chopping board-39f0463050297140", "yellow pepper 8-5dc836af44798409", "mushroom container-87e64f60098d47c8", "empty plastic cover-28154996946f5f20", "plastic box2-91b76314a1ff9d22", "plastic box3-851b6f056afd363e", "whisk-324e2e3f6084dbb4", "electric hob-5d5feb3826ca8501", "rubber spatula-767954c6cde5bc3a", "bowl2-12586cd531b79e58", "lid-a2e8a464602ab8fc", "bowl-2cea3a2c933a4ba5", "salt jar-0e6e4e8b4291f578", "pepper jar-182250f0df199a39", "plastic container in fridge-7cbe7ad891857724", "plastic container in fridge 2-b0cffd3396cb45ea", "plastic container in fridge 3-7c1d91d305a6ad2a", "tablespoon-87e1e2ec75c7d8df", "teaspoon2-d392be23ab3210c1", "colander-f5d61e09a407b36b", "cap of milk carton-c3b52e4fc0eeb284", "plastic cup-4085e245d92fe5a3", "correct whisk-0104fa3069f8860d", "plug of electric hob-174bbeb2af19ffa9", "jar of cinnamon powder-a76839f873a09fa7", "cap of olive oil bottle-31debbd52b845f6c", "spoon3-54d498d956a68225", "plate-0100d730b65fb951", "orange kitchen towel-703e761eeafcb0bc", "plate2-97dcc7e125e351e0", "another kitchen towel-b8aca97c478f2ff7", "pan-f987efd8203dbd58", "weighing scale-80a15ce1d41e5389", "coffee mug-7b3820fb398bf3da", "sponge-8d81f29a1a164899", "spoon holder-23c52c09d84ce83c", "white cap of plastic container-6398e298025d38c1" ], "reasoning": "1. I watched the whole video and looked for things the user put away and needed to use again. \n2. At 04:36, the user puts the oil in the bottom shelf of the cabinet to the left of the cutting board. \n2. At 12:09, the user realizes they need the oil to cook the eggs .\n3. At 12:10, they turn to the left and walk down to the cabinet to retrieve the oil.", "referenced_object_masks": [] }, { "key": "P04-20240414-065311:990267ced01c54c2e0becda9d6264586cf83a8e7", "video_id": "P04-20240414-065311", "question": "What item did I put in a cabinet that should have gone in the refrigerator?", "answer_choice_0": "Mushrooms", "answer_choice_1": "Green Pepper", "answer_choice_2": "Milk", "answer_choice_3": "Eggs", "answer_choice_4": "Butter", "answer_id": 3, "answer": "Eggs", "question_type": [ "Egocentric & Spatial Perception", "Event Occurence", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "notebook-02bdb15c5e0fe283", "marker-682b9206618c53be", "empty plastic container box-1c6baa284943ffa8", "plastic container-f5a4e153cc240565", "egg2-da3705d64b0cffe9", "egg1-cb27293ba8f99a3e", "first egg-1f32c0240febded2", "egg3-cccb39acd90034b6", "egg4-f961f78612d57f68", "second egg-e049fd4ac12da049", "correct weighing scale-d5068055230c4da4", "eggshells-cbdae18f19ff20f4", "large knife-76546fac7fc5bad1", "third egg-ad5ab695a740c077", "egg carton-cd4102d37d9e2cac", "spoon-80c4076af3818b30", "fourth egg-c0d55cc89ef9b1d6", "yellow pepper-59e24eb29d22dfee", "box of mushrooms-303c6b9b622c66f2", "plastic bag-3967805d5a39a77d", "mushroom-dab7070b32dd5cbd", "green pepper-6e668c0ff6a22bf8", "chopping board-39f0463050297140", "yellow pepper 8-5dc836af44798409", "mushroom container-87e64f60098d47c8", "empty plastic cover-28154996946f5f20", "plastic box2-91b76314a1ff9d22", "plastic box3-851b6f056afd363e", "whisk-324e2e3f6084dbb4", "electric hob-5d5feb3826ca8501", "rubber spatula-767954c6cde5bc3a", "bowl2-12586cd531b79e58", "lid-a2e8a464602ab8fc", "bowl-2cea3a2c933a4ba5", "salt jar-0e6e4e8b4291f578", "pepper jar-182250f0df199a39", "plastic container in fridge-7cbe7ad891857724", "plastic container in fridge 2-b0cffd3396cb45ea", "plastic container in fridge 3-7c1d91d305a6ad2a", "tablespoon-87e1e2ec75c7d8df", "teaspoon2-d392be23ab3210c1", "colander-f5d61e09a407b36b", "cap of milk carton-c3b52e4fc0eeb284", "plastic cup-4085e245d92fe5a3", "correct whisk-0104fa3069f8860d", "plug of electric hob-174bbeb2af19ffa9", "jar of cinnamon powder-a76839f873a09fa7", "cap of olive oil bottle-31debbd52b845f6c", "spoon3-54d498d956a68225", "plate-0100d730b65fb951", "orange kitchen towel-703e761eeafcb0bc", "plate2-97dcc7e125e351e0", "another kitchen towel-b8aca97c478f2ff7", "pan-f987efd8203dbd58", "weighing scale-80a15ce1d41e5389", "coffee mug-7b3820fb398bf3da", "sponge-8d81f29a1a164899", "spoon holder-23c52c09d84ce83c", "white cap of plastic container-6398e298025d38c1" ], "reasoning": "1. At 03:40, the user picks up the eggs (egg carton-cd4102d37d9e2cac) and moves them down the counter to their left on top of the refridgerator . \n2. At 19:06, the user picks up the eggs (egg carton-cd4102d37d9e2cac) off the counter and places them in a cabinet to the left of the refrigerator .", "referenced_object_masks": [ "egg carton-cd4102d37d9e2cac" ] }, { "key": "P04-20240414-065311:a74cfaa0f9c933f5ebb7792f982bcd62a51bc708", "video_id": "P04-20240414-065311", "question": "How many transfers of the yolk to the shell does it take me to completely separate the yolk from the whites of the third egg from which I separate the yolk and the whites?", "answer_choice_0": "5", "answer_choice_1": "2", "answer_choice_2": "4", "answer_choice_3": "1", "answer_choice_4": "3", "answer_id": 4, "answer": "3", "question_type": [ "Action Recognition", "Counting", "Object Recognition" ], "objects_of_interest": [ "notebook-02bdb15c5e0fe283", "marker-682b9206618c53be", "empty plastic container box-1c6baa284943ffa8", "plastic container-f5a4e153cc240565", "egg2-da3705d64b0cffe9", "egg1-cb27293ba8f99a3e", "first egg-1f32c0240febded2", "egg3-cccb39acd90034b6", "egg4-f961f78612d57f68", "second egg-e049fd4ac12da049", "correct weighing scale-d5068055230c4da4", "eggshells-cbdae18f19ff20f4", "large knife-76546fac7fc5bad1", "third egg-ad5ab695a740c077", "egg carton-cd4102d37d9e2cac", "spoon-80c4076af3818b30", "fourth egg-c0d55cc89ef9b1d6", "yellow pepper-59e24eb29d22dfee", "box of mushrooms-303c6b9b622c66f2", "plastic bag-3967805d5a39a77d", "mushroom-dab7070b32dd5cbd", "green pepper-6e668c0ff6a22bf8", "chopping board-39f0463050297140", "yellow pepper 8-5dc836af44798409", "mushroom container-87e64f60098d47c8", "empty plastic cover-28154996946f5f20", "plastic box2-91b76314a1ff9d22", "plastic box3-851b6f056afd363e", "whisk-324e2e3f6084dbb4", "electric hob-5d5feb3826ca8501", "rubber spatula-767954c6cde5bc3a", "bowl2-12586cd531b79e58", "lid-a2e8a464602ab8fc", "bowl-2cea3a2c933a4ba5", "salt jar-0e6e4e8b4291f578", "pepper jar-182250f0df199a39", "plastic container in fridge-7cbe7ad891857724", "plastic container in fridge 2-b0cffd3396cb45ea", "plastic container in fridge 3-7c1d91d305a6ad2a", "tablespoon-87e1e2ec75c7d8df", "teaspoon2-d392be23ab3210c1", "colander-f5d61e09a407b36b", "cap of milk carton-c3b52e4fc0eeb284", "plastic cup-4085e245d92fe5a3", "correct whisk-0104fa3069f8860d", "plug of electric hob-174bbeb2af19ffa9", "jar of cinnamon powder-a76839f873a09fa7", "cap of olive oil bottle-31debbd52b845f6c", "spoon3-54d498d956a68225", "plate-0100d730b65fb951", "orange kitchen towel-703e761eeafcb0bc", "plate2-97dcc7e125e351e0", "another kitchen towel-b8aca97c478f2ff7", "pan-f987efd8203dbd58", "weighing scale-80a15ce1d41e5389", "coffee mug-7b3820fb398bf3da", "sponge-8d81f29a1a164899", "spoon holder-23c52c09d84ce83c", "white cap of plastic container-6398e298025d38c1" ], "reasoning": "1. First, I determine the instances when the user is separating yolks from whites.This separation starts with the first egg (egg3-cccb39acd90034b6) at 01:33, the second egg (third egg-ad5ab695a740c077) at 02:09, and the third egg (egg1-cb27293ba8f99a3e) at 02:37. \n2. Next, I shift my focus to the third egg (egg1-cb27293ba8f99a3e), counting the number of yolk transfers to the shell. \n3. The first transfer occurs at 02:41, followed by another at 02:44, and one more at 02:46 for a total of 3.", "referenced_object_masks": [ "egg3-cccb39acd90034b6", "egg1-cb27293ba8f99a3e", "third egg-ad5ab695a740c077" ] }, { "key": "P04-20240414-065311:be3cec64c880a331542ab80bcf491fc014fd2ca4", "video_id": "P04-20240414-065311", "question": "What are the 2 consecutive directions I gaze while I'm doing the following 2 actions: retrieving the second bowl from the cabinet and turning away after straightening the skillet's handle?", "answer_choice_0": "Looking up; facing left", "answer_choice_1": "Facing up; looking behind me", "answer_choice_2": "Facing down; looking left", "answer_choice_3": "Looking in front of me; facing down", "answer_choice_4": "Looking down; facing right", "answer_id": 0, "answer": "Looking up; facing left", "question_type": [ "Egocentric & Spatial Perception", "Action Recognition", "State Changes" ], "objects_of_interest": [ "notebook-02bdb15c5e0fe283", "marker-682b9206618c53be", "empty plastic container box-1c6baa284943ffa8", "plastic container-f5a4e153cc240565", "egg2-da3705d64b0cffe9", "egg1-cb27293ba8f99a3e", "first egg-1f32c0240febded2", "egg3-cccb39acd90034b6", "egg4-f961f78612d57f68", "second egg-e049fd4ac12da049", "correct weighing scale-d5068055230c4da4", "eggshells-cbdae18f19ff20f4", "large knife-76546fac7fc5bad1", "third egg-ad5ab695a740c077", "egg carton-cd4102d37d9e2cac", "spoon-80c4076af3818b30", "fourth egg-c0d55cc89ef9b1d6", "yellow pepper-59e24eb29d22dfee", "box of mushrooms-303c6b9b622c66f2", "plastic bag-3967805d5a39a77d", "mushroom-dab7070b32dd5cbd", "green pepper-6e668c0ff6a22bf8", "chopping board-39f0463050297140", "yellow pepper 8-5dc836af44798409", "mushroom container-87e64f60098d47c8", "empty plastic cover-28154996946f5f20", "plastic box2-91b76314a1ff9d22", "plastic box3-851b6f056afd363e", "whisk-324e2e3f6084dbb4", "electric hob-5d5feb3826ca8501", "rubber spatula-767954c6cde5bc3a", "bowl2-12586cd531b79e58", "lid-a2e8a464602ab8fc", "bowl-2cea3a2c933a4ba5", "salt jar-0e6e4e8b4291f578", "pepper jar-182250f0df199a39", "plastic container in fridge-7cbe7ad891857724", "plastic container in fridge 2-b0cffd3396cb45ea", "plastic container in fridge 3-7c1d91d305a6ad2a", "tablespoon-87e1e2ec75c7d8df", "teaspoon2-d392be23ab3210c1", "colander-f5d61e09a407b36b", "cap of milk carton-c3b52e4fc0eeb284", "plastic cup-4085e245d92fe5a3", "correct whisk-0104fa3069f8860d", "plug of electric hob-174bbeb2af19ffa9", "jar of cinnamon powder-a76839f873a09fa7", "cap of olive oil bottle-31debbd52b845f6c", "spoon3-54d498d956a68225", "plate-0100d730b65fb951", "orange kitchen towel-703e761eeafcb0bc", "plate2-97dcc7e125e351e0", "another kitchen towel-b8aca97c478f2ff7", "pan-f987efd8203dbd58", "weighing scale-80a15ce1d41e5389", "coffee mug-7b3820fb398bf3da", "sponge-8d81f29a1a164899", "spoon holder-23c52c09d84ce83c", "white cap of plastic container-6398e298025d38c1" ], "reasoning": "1. First, I determine when the user retrieves the first bowl (bowl-2cea3a2c933a4ba5) from the cabinet. This occurs at 08:50. The second bowl (bowl2-12586cd531b79e58) is retrieved at 15:01, where the user is looking up.\n2. Next, I determine the user turns away after straightening the skillet's (plate-0100d730b65fb951) handle at 12:08. The user's gaze at that moment is facing left. \n3. Therefore, the answer is looking up and facing left.", "referenced_object_masks": [ "plate-0100d730b65fb951", "bowl-2cea3a2c933a4ba5", "bowl2-12586cd531b79e58" ] }, { "key": "P04-20240414-065311:c1db3fb096e25c50940478a2401ab802bfe3c28e", "video_id": "P04-20240414-065311", "question": "How does my gaze shift during the next to the last time I handle the oil?", "answer_choice_0": "Sweeps up and left", "answer_choice_1": "Sweeps behind and straight ahead", "answer_choice_2": "Sweeps right and down", "answer_choice_3": "Sweeps down and right", "answer_choice_4": "Sweeps left and down", "answer_id": 4, "answer": "Sweeps left and down", "question_type": [ "Counting", "Egocentric & Spatial Perception", "Action Recognition", "Object Recognition", "State Changes" ], "objects_of_interest": [ "notebook-02bdb15c5e0fe283", "marker-682b9206618c53be", "empty plastic container box-1c6baa284943ffa8", "plastic container-f5a4e153cc240565", "egg2-da3705d64b0cffe9", "egg1-cb27293ba8f99a3e", "first egg-1f32c0240febded2", "egg3-cccb39acd90034b6", "egg4-f961f78612d57f68", "second egg-e049fd4ac12da049", "correct weighing scale-d5068055230c4da4", "eggshells-cbdae18f19ff20f4", "large knife-76546fac7fc5bad1", "third egg-ad5ab695a740c077", "egg carton-cd4102d37d9e2cac", "spoon-80c4076af3818b30", "fourth egg-c0d55cc89ef9b1d6", "yellow pepper-59e24eb29d22dfee", "box of mushrooms-303c6b9b622c66f2", "plastic bag-3967805d5a39a77d", "mushroom-dab7070b32dd5cbd", "green pepper-6e668c0ff6a22bf8", "chopping board-39f0463050297140", "yellow pepper 8-5dc836af44798409", "mushroom container-87e64f60098d47c8", "empty plastic cover-28154996946f5f20", "plastic box2-91b76314a1ff9d22", "plastic box3-851b6f056afd363e", "whisk-324e2e3f6084dbb4", "electric hob-5d5feb3826ca8501", "rubber spatula-767954c6cde5bc3a", "bowl2-12586cd531b79e58", "lid-a2e8a464602ab8fc", "bowl-2cea3a2c933a4ba5", "salt jar-0e6e4e8b4291f578", "pepper jar-182250f0df199a39", "plastic container in fridge-7cbe7ad891857724", "plastic container in fridge 2-b0cffd3396cb45ea", "plastic container in fridge 3-7c1d91d305a6ad2a", "tablespoon-87e1e2ec75c7d8df", "teaspoon2-d392be23ab3210c1", "colander-f5d61e09a407b36b", "cap of milk carton-c3b52e4fc0eeb284", "plastic cup-4085e245d92fe5a3", "correct whisk-0104fa3069f8860d", "plug of electric hob-174bbeb2af19ffa9", "jar of cinnamon powder-a76839f873a09fa7", "cap of olive oil bottle-31debbd52b845f6c", "spoon3-54d498d956a68225", "plate-0100d730b65fb951", "orange kitchen towel-703e761eeafcb0bc", "plate2-97dcc7e125e351e0", "another kitchen towel-b8aca97c478f2ff7", "pan-f987efd8203dbd58", "weighing scale-80a15ce1d41e5389", "coffee mug-7b3820fb398bf3da", "sponge-8d81f29a1a164899", "spoon holder-23c52c09d84ce83c", "white cap of plastic container-6398e298025d38c1" ], "reasoning": "1. First, I determine all the instances where the user handles the bottle of oil (orange kitchen towel-703e761eeafcb0bc). This occurs for the first time at 12:13, second, at 12:36, third, at 13:00, fourth, at 18:28, and the fifth, and last time at 18:40. \n2. I determine that the fourth instance, at 18:28, is the next to last time the user handles the oil (orange kitchen towel-703e761eeafcb0bc), so I focus on that instance only. \n3. I conclude then that the user's gaze sweeps to the left after they pick up the bottle of oil (orange kitchen towel-703e761eeafcb0bc) and then downward after placing the oil (orange kitchen towel-703e761eeafcb0bc) back on the counter at 18:29.", "referenced_object_masks": [ "orange kitchen towel-703e761eeafcb0bc" ] }, { "key": "P04-20240414-065311:d093b4a6afdcfd12bebc70385c89fedc8809d8bd", "video_id": "P04-20240414-065311", "question": "Where is the apron in relation to me when I first start scrambling the egg?", "answer_choice_0": "On my right", "answer_choice_1": "I'm wearing it", "answer_choice_2": "Behind me", "answer_choice_3": "In front of me", "answer_choice_4": "On my left", "answer_id": 4, "answer": "On my left", "question_type": [ "Egocentric & Spatial Perception", "Action Recognition", "Object Recognition" ], "objects_of_interest": [ "notebook-02bdb15c5e0fe283", "marker-682b9206618c53be", "empty plastic container box-1c6baa284943ffa8", "plastic container-f5a4e153cc240565", "egg2-da3705d64b0cffe9", "egg1-cb27293ba8f99a3e", "first egg-1f32c0240febded2", "egg3-cccb39acd90034b6", "egg4-f961f78612d57f68", "second egg-e049fd4ac12da049", "correct weighing scale-d5068055230c4da4", "eggshells-cbdae18f19ff20f4", "large knife-76546fac7fc5bad1", "third egg-ad5ab695a740c077", "egg carton-cd4102d37d9e2cac", "spoon-80c4076af3818b30", "fourth egg-c0d55cc89ef9b1d6", "yellow pepper-59e24eb29d22dfee", "box of mushrooms-303c6b9b622c66f2", "plastic bag-3967805d5a39a77d", "mushroom-dab7070b32dd5cbd", "green pepper-6e668c0ff6a22bf8", "chopping board-39f0463050297140", "yellow pepper 8-5dc836af44798409", "mushroom container-87e64f60098d47c8", "empty plastic cover-28154996946f5f20", "plastic box2-91b76314a1ff9d22", "plastic box3-851b6f056afd363e", "whisk-324e2e3f6084dbb4", "electric hob-5d5feb3826ca8501", "rubber spatula-767954c6cde5bc3a", "bowl2-12586cd531b79e58", "lid-a2e8a464602ab8fc", "bowl-2cea3a2c933a4ba5", "salt jar-0e6e4e8b4291f578", "pepper jar-182250f0df199a39", "plastic container in fridge-7cbe7ad891857724", "plastic container in fridge 2-b0cffd3396cb45ea", "plastic container in fridge 3-7c1d91d305a6ad2a", "tablespoon-87e1e2ec75c7d8df", "teaspoon2-d392be23ab3210c1", "colander-f5d61e09a407b36b", "cap of milk carton-c3b52e4fc0eeb284", "plastic cup-4085e245d92fe5a3", "correct whisk-0104fa3069f8860d", "plug of electric hob-174bbeb2af19ffa9", "jar of cinnamon powder-a76839f873a09fa7", "cap of olive oil bottle-31debbd52b845f6c", "spoon3-54d498d956a68225", "plate-0100d730b65fb951", "orange kitchen towel-703e761eeafcb0bc", "plate2-97dcc7e125e351e0", "another kitchen towel-b8aca97c478f2ff7", "pan-f987efd8203dbd58", "weighing scale-80a15ce1d41e5389", "coffee mug-7b3820fb398bf3da", "sponge-8d81f29a1a164899", "spoon holder-23c52c09d84ce83c", "white cap of plastic container-6398e298025d38c1" ], "reasoning": "1. First, I determine the location of the apron . This is visible at 00:02 when the user walks in the kitchen. The apron is hanging over the door in front of the user on the kitchen's left. \n2. I determine the moment when the user is at the stove, first scrambling the egg (egg1-cb27293ba8f99a3e). This occurs at 14:19, after the user adds in the veggies (green pepper-6e668c0ff6a22bf8), mushrooms . \n3. Since the apron is hanging over the door on the kitchen's left, and the stove is positioned on the same side of the kitchen, just a few feet over, the apron is positioned on the user's left while they are at the stove scrambling the egg (egg1-cb27293ba8f99a3e).", "referenced_object_masks": [ "egg1-cb27293ba8f99a3e", "green pepper-6e668c0ff6a22bf8" ] }, { "key": "P04-20240414-065311:d9a1de7bb3e187366427e68ba53a78f19a045575", "video_id": "P04-20240414-065311", "question": "What is the ratio of the weight of the third egg I weighed to the weight of the white bowl I used later and weighed? Simplify the ratio if possible.", "answer_choice_0": "69:384", "answer_choice_1": "13:78", "answer_choice_2": "65:384", "answer_choice_3": "23:74", "answer_choice_4": "11:64", "answer_id": 4, "answer": "11:64", "question_type": [ "Event Occurence", "Numerical Reasoning", "Object Recognition", "Situational Awareness", "Temporal Reasoning" ], "objects_of_interest": [ "notebook-02bdb15c5e0fe283", "marker-682b9206618c53be", "empty plastic container box-1c6baa284943ffa8", "plastic container-f5a4e153cc240565", "egg2-da3705d64b0cffe9", "egg1-cb27293ba8f99a3e", "first egg-1f32c0240febded2", "egg3-cccb39acd90034b6", "egg4-f961f78612d57f68", "second egg-e049fd4ac12da049", "correct weighing scale-d5068055230c4da4", "eggshells-cbdae18f19ff20f4", "large knife-76546fac7fc5bad1", "third egg-ad5ab695a740c077", "egg carton-cd4102d37d9e2cac", "spoon-80c4076af3818b30", "fourth egg-c0d55cc89ef9b1d6", "yellow pepper-59e24eb29d22dfee", "box of mushrooms-303c6b9b622c66f2", "plastic bag-3967805d5a39a77d", "mushroom-dab7070b32dd5cbd", "green pepper-6e668c0ff6a22bf8", "chopping board-39f0463050297140", "yellow pepper 8-5dc836af44798409", "mushroom container-87e64f60098d47c8", "empty plastic cover-28154996946f5f20", "plastic box2-91b76314a1ff9d22", "plastic box3-851b6f056afd363e", "whisk-324e2e3f6084dbb4", "electric hob-5d5feb3826ca8501", "rubber spatula-767954c6cde5bc3a", "bowl2-12586cd531b79e58", "lid-a2e8a464602ab8fc", "bowl-2cea3a2c933a4ba5", "salt jar-0e6e4e8b4291f578", "pepper jar-182250f0df199a39", "plastic container in fridge-7cbe7ad891857724", "plastic container in fridge 2-b0cffd3396cb45ea", "plastic container in fridge 3-7c1d91d305a6ad2a", "tablespoon-87e1e2ec75c7d8df", "teaspoon2-d392be23ab3210c1", "colander-f5d61e09a407b36b", "cap of milk carton-c3b52e4fc0eeb284", "plastic cup-4085e245d92fe5a3", "correct whisk-0104fa3069f8860d", "plug of electric hob-174bbeb2af19ffa9", "jar of cinnamon powder-a76839f873a09fa7", "cap of olive oil bottle-31debbd52b845f6c", "spoon3-54d498d956a68225", "plate-0100d730b65fb951", "orange kitchen towel-703e761eeafcb0bc", "plate2-97dcc7e125e351e0", "another kitchen towel-b8aca97c478f2ff7", "pan-f987efd8203dbd58", "weighing scale-80a15ce1d41e5389", "coffee mug-7b3820fb398bf3da", "sponge-8d81f29a1a164899", "spoon holder-23c52c09d84ce83c", "white cap of plastic container-6398e298025d38c1" ], "reasoning": "1. I watched the whole video to see the user weighing specific eggs.\n2. The third egg (third egg-ad5ab695a740c077) was weighed at 0:44. \n3. It weighed 66 grams. \n4: The white bowl was weighed at 09:00. \n5. It weighed 384 grams. \n6. The ratio of how much the third egg weighed to how much the white bowl weighed is 66:384.\n7. This ratio can be simplified to 11:64.", "referenced_object_masks": [ "third egg-ad5ab695a740c077" ] }, { "key": "P04-20240414-072251:00257a83e56372fe75a13a8c9cc46f2e1bfa8256", "video_id": "P04-20240414-072251", "question": "Where did I put the white glass bowl in relation to where I put the white glass plate?", "answer_choice_0": "In the same cabinet to the right", "answer_choice_1": "In the same cabinet above the plate", "answer_choice_2": "In the diagonal cabinet to the left", "answer_choice_3": "In the diagonal cabinet to the right", "answer_choice_4": "In the same cabinet to the left", "answer_id": 4, "answer": "In the same cabinet to the left", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "frying pan-a597ca21304f3cc2", "sponge-b398c8dd92cbdd13", "soap dispenser-6fd8734e696816a6", "spoon-75e2910407a73328", "tupperware container1-3851fd61a2b6b508", "tupperware container2-e48346135f204a2c", "whisk-5d77d2aed09bb1f8", "knife-2753a32bae2cd912", "cutlery holder-5ea87e3a428e2c29", "spoon2-8286fa3658081066", "measuring cup-57ae3945e83598d4", "chopping board-699c09d1a4397b26", "lid of plastic container-5bbd6d234a50b302", "mug-06ef18483b937dfa", "bowl-2920ee2f584c003d", "rubber spatula-d8d6ffda1c12ee1e", "plate-3b3ca5d805b1aebc", "mug2-40eac9d84cb1204b", "mug3-e9fc61163aac02c7", "orange cloth-5746fcd9cf7fa0f1", "metal mixing bowl-b84f557ab9339a6d", "plastic bowl-122d1fe6cd83b081", "other stack of plastic containers-2d1240bdca1e4623", "blue cloth-ca7ab436f9779dbd", "spray bottle-53569dfaf5f18839", "cooling rack-568c05aa9895bdba", "plate2-7e8f8234a232897b", "bowl2-31258aa986ce8f5f", "pen-1ee13d6ceb69284c", "orange cloth2-a18546c84607a308", "notebook-b523829ed1d7c1af" ], "reasoning": "1. At 17:57, the user finished drying the bowl (bowl2-31258aa986ce8f5f) and put it on the left side of the cabinet above the counter.\n2. At 18:06, the user dried the plate (plate2-7e8f8234a232897b) and put it on the right side of the cabinet above the counter at 18:20.\n3. The bowl (bowl2-31258aa986ce8f5f) was placed in the same cabinet to the left of the plate (plate2-7e8f8234a232897b), and I could tell this was so because the silver mixing bowl (metal mixing bowl-b84f557ab9339a6d) was directly below the cabinet in both shots.", "referenced_object_masks": [ "bowl2-31258aa986ce8f5f", "metal mixing bowl-b84f557ab9339a6d", "plate2-7e8f8234a232897b" ] }, { "key": "P04-20240414-072251:1a9720f623e25dea4ff4a2aa795a0ac5b892ab52", "video_id": "P04-20240414-072251", "question": "How many more unique items did I wash the second time I pumped soap on the sponge compared to the first time?", "answer_choice_0": "8", "answer_choice_1": "10", "answer_choice_2": "2", "answer_choice_3": "9", "answer_choice_4": "12", "answer_id": 0, "answer": "8", "question_type": [ "Counting", "Numerical Reasoning", "Event Occurence", "Action Recognition" ], "objects_of_interest": [ "frying pan-a597ca21304f3cc2", "sponge-b398c8dd92cbdd13", "soap dispenser-6fd8734e696816a6", "spoon-75e2910407a73328", "tupperware container1-3851fd61a2b6b508", "tupperware container2-e48346135f204a2c", "whisk-5d77d2aed09bb1f8", "knife-2753a32bae2cd912", "cutlery holder-5ea87e3a428e2c29", "spoon2-8286fa3658081066", "measuring cup-57ae3945e83598d4", "chopping board-699c09d1a4397b26", "lid of plastic container-5bbd6d234a50b302", "mug-06ef18483b937dfa", "bowl-2920ee2f584c003d", "rubber spatula-d8d6ffda1c12ee1e", "plate-3b3ca5d805b1aebc", "mug2-40eac9d84cb1204b", "mug3-e9fc61163aac02c7", "orange cloth-5746fcd9cf7fa0f1", "metal mixing bowl-b84f557ab9339a6d", "plastic bowl-122d1fe6cd83b081", "other stack of plastic containers-2d1240bdca1e4623", "blue cloth-ca7ab436f9779dbd", "spray bottle-53569dfaf5f18839", "cooling rack-568c05aa9895bdba", "plate2-7e8f8234a232897b", "bowl2-31258aa986ce8f5f", "pen-1ee13d6ceb69284c", "orange cloth2-a18546c84607a308", "notebook-b523829ed1d7c1af" ], "reasoning": "1. I located the first instance where the user pumps the soap dispenser (soap dispenser-6fd8734e696816a6) to wash the dishes. I found this event at 00:35. After the user pumps soap on the sponge (sponge-b398c8dd92cbdd13), the user washes 2 plastic containers (tupperware container1-3851fd61a2b6b508, tupperware container2-e48346135f204a2c), the first at 00:49 and the second at 01:59. \n2. The user pumps soap on the sponge (sponge-b398c8dd92cbdd13) again at 02:12, and continues to scrub the second plastic container (tupperware container2-e48346135f204a2c) before moving on to a new item. Since this item was already being washed before the second pump of soap, it is not a unique item and I do not count it.\n3. I kept watching to count how many more unique items the subject washed after the second container (tupperware container2-e48346135f204a2c). From 02:12 to 08:51, the user washed 10 additional unique items before pumping soap on the sponge (sponge-b398c8dd92cbdd13) for the third time. Once finished scrubbing the second plastic container (tupperware container2-e48346135f204a2c), the user moves on to the whisk (whisk-5d77d2aed09bb1f8) at 02:45, and then to the knife (knife-2753a32bae2cd912) at 03:10. The user then proceeds to wash the first spoon (spoon-75e2910407a73328) at 03:28, the container lid (lid of plastic container-5bbd6d234a50b302) at 05:43, another spoon (spoon2-8286fa3658081066) at 06:05, a measuring cup (measuring cup-57ae3945e83598d4) at 06:14, a cutting board (chopping board-699c09d1a4397b26) at 06:29, a blue spatula (rubber spatula-d8d6ffda1c12ee1e) at 08:06, a white ceramic bowl (bowl-2920ee2f584c003d) at 08:18, and a white mug (mug-06ef18483b937dfa) at 08:47. After the white mug, the user squeezed more soap onto the sponge (sponge-b398c8dd92cbdd13). \n4. I subtracted the 2 items the user washed with the first squeeze of soap from the 10 unique items the user washed with the second squeeze, and it totaled 8 items. Therefore, the user washed 8 more items with the second pump of soap than with the first one.", "referenced_object_masks": [ "sponge-b398c8dd92cbdd13", "tupperware container2-e48346135f204a2c", "lid of plastic container-5bbd6d234a50b302", "measuring cup-57ae3945e83598d4", "whisk-5d77d2aed09bb1f8", "soap dispenser-6fd8734e696816a6", "spoon2-8286fa3658081066", "knife-2753a32bae2cd912", "bowl-2920ee2f584c003d", "rubber spatula-d8d6ffda1c12ee1e", "chopping board-699c09d1a4397b26", "spoon-75e2910407a73328", "tupperware container1-3851fd61a2b6b508", "mug-06ef18483b937dfa" ] }, { "key": "P04-20240414-072251:6c2bd7ce1b0c07ea9fdd839e4df599168c2da11b", "video_id": "P04-20240414-072251", "question": "Which item did I wash after the knife but right before the cutting board?", "answer_choice_0": "Lid", "answer_choice_1": "Spatula", "answer_choice_2": "Whisker", "answer_choice_3": "Spoon", "answer_choice_4": "Measuring cup", "answer_id": 4, "answer": "Measuring cup", "question_type": [ "Event Occurence", "Object Recognition", "Counting", "Action Recognition" ], "objects_of_interest": [ "frying pan-a597ca21304f3cc2", "sponge-b398c8dd92cbdd13", "soap dispenser-6fd8734e696816a6", "spoon-75e2910407a73328", "tupperware container1-3851fd61a2b6b508", "tupperware container2-e48346135f204a2c", "whisk-5d77d2aed09bb1f8", "knife-2753a32bae2cd912", "cutlery holder-5ea87e3a428e2c29", "spoon2-8286fa3658081066", "measuring cup-57ae3945e83598d4", "chopping board-699c09d1a4397b26", "lid of plastic container-5bbd6d234a50b302", "mug-06ef18483b937dfa", "bowl-2920ee2f584c003d", "rubber spatula-d8d6ffda1c12ee1e", "plate-3b3ca5d805b1aebc", "mug2-40eac9d84cb1204b", "mug3-e9fc61163aac02c7", "orange cloth-5746fcd9cf7fa0f1", "metal mixing bowl-b84f557ab9339a6d", "plastic bowl-122d1fe6cd83b081", "other stack of plastic containers-2d1240bdca1e4623", "blue cloth-ca7ab436f9779dbd", "spray bottle-53569dfaf5f18839", "cooling rack-568c05aa9895bdba", "plate2-7e8f8234a232897b", "bowl2-31258aa986ce8f5f", "pen-1ee13d6ceb69284c", "orange cloth2-a18546c84607a308", "notebook-b523829ed1d7c1af" ], "reasoning": "1. I located the moment the user washes the knife (knife-2753a32bae2cd912) at 03:10. \n2. I watched the next items the user scrubs after the knife (knife-2753a32bae2cd912), but before the cutting board (chopping board-699c09d1a4397b26), which happens between 03:10 and 06:29. \n3. At 06:29, the user scrubs a cutting board (chopping board-699c09d1a4397b26). Right before it, the user scrubs a small, plastic measuring cup (measuring cup-57ae3945e83598d4).", "referenced_object_masks": [ "measuring cup-57ae3945e83598d4", "chopping board-699c09d1a4397b26", "knife-2753a32bae2cd912" ] }, { "key": "P04-20240414-072251:6ffb7ce2913dad85f9643a144b86fe37ae23df70", "video_id": "P04-20240414-072251", "question": "What was the third item that I rinsed after washing the white chopping board?", "answer_choice_0": "A pan", "answer_choice_1": "A bowl", "answer_choice_2": "A lid", "answer_choice_3": "A fork", "answer_choice_4": "A spoon", "answer_id": 4, "answer": "A spoon", "question_type": [ "Object Recognition", "Action Recognition", "Situational Awareness", "Counting" ], "objects_of_interest": [ "frying pan-a597ca21304f3cc2", "sponge-b398c8dd92cbdd13", "soap dispenser-6fd8734e696816a6", "spoon-75e2910407a73328", "tupperware container1-3851fd61a2b6b508", "tupperware container2-e48346135f204a2c", "whisk-5d77d2aed09bb1f8", "knife-2753a32bae2cd912", "cutlery holder-5ea87e3a428e2c29", "spoon2-8286fa3658081066", "measuring cup-57ae3945e83598d4", "chopping board-699c09d1a4397b26", "lid of plastic container-5bbd6d234a50b302", "mug-06ef18483b937dfa", "bowl-2920ee2f584c003d", "rubber spatula-d8d6ffda1c12ee1e", "plate-3b3ca5d805b1aebc", "mug2-40eac9d84cb1204b", "mug3-e9fc61163aac02c7", "orange cloth-5746fcd9cf7fa0f1", "metal mixing bowl-b84f557ab9339a6d", "plastic bowl-122d1fe6cd83b081", "other stack of plastic containers-2d1240bdca1e4623", "blue cloth-ca7ab436f9779dbd", "spray bottle-53569dfaf5f18839", "cooling rack-568c05aa9895bdba", "plate2-7e8f8234a232897b", "bowl2-31258aa986ce8f5f", "pen-1ee13d6ceb69284c", "orange cloth2-a18546c84607a308", "notebook-b523829ed1d7c1af" ], "reasoning": "1. At 06:29, the user picked up the white chopping (chopping board-699c09d1a4397b26) board to wash it.\n2. At 06:42, the user rinsed off the white chopping board (chopping board-699c09d1a4397b26).\n3. At 06:56, the user picked up the white lid container (lid of plastic container-5bbd6d234a50b302) and rinses it off.\n4. At 07:19, the user picked up the small bowl and rinses it.\n5. At 07:27, the user picked the spoon (spoon-75e2910407a73328) and rinses it.\n6. The third item that was rinsed was the spoon (spoon-75e2910407a73328).", "referenced_object_masks": [ "lid of plastic container-5bbd6d234a50b302", "chopping board-699c09d1a4397b26", "spoon-75e2910407a73328" ] }, { "key": "P04-20240414-072251:75fa3fe4f883b0f35ecb7fee83481d9dd5ca56b0", "video_id": "P04-20240414-072251", "question": "What is the difference between how long I washed the tupperware container versus the whisk?", "answer_choice_0": "5 seconds", "answer_choice_1": "7 seconds", "answer_choice_2": "10 seconds", "answer_choice_3": "4 seconds", "answer_choice_4": "6 seconds", "answer_id": 0, "answer": "5 seconds", "question_type": [ "Counting", "Object Recognition", "Situational Awareness", "Temporal Reasoning", "Numerical Reasoning" ], "objects_of_interest": [ "frying pan-a597ca21304f3cc2", "sponge-b398c8dd92cbdd13", "soap dispenser-6fd8734e696816a6", "spoon-75e2910407a73328", "tupperware container1-3851fd61a2b6b508", "tupperware container2-e48346135f204a2c", "whisk-5d77d2aed09bb1f8", "knife-2753a32bae2cd912", "cutlery holder-5ea87e3a428e2c29", "spoon2-8286fa3658081066", "measuring cup-57ae3945e83598d4", "chopping board-699c09d1a4397b26", "lid of plastic container-5bbd6d234a50b302", "mug-06ef18483b937dfa", "bowl-2920ee2f584c003d", "rubber spatula-d8d6ffda1c12ee1e", "plate-3b3ca5d805b1aebc", "mug2-40eac9d84cb1204b", "mug3-e9fc61163aac02c7", "orange cloth-5746fcd9cf7fa0f1", "metal mixing bowl-b84f557ab9339a6d", "plastic bowl-122d1fe6cd83b081", "other stack of plastic containers-2d1240bdca1e4623", "blue cloth-ca7ab436f9779dbd", "spray bottle-53569dfaf5f18839", "cooling rack-568c05aa9895bdba", "plate2-7e8f8234a232897b", "bowl2-31258aa986ce8f5f", "pen-1ee13d6ceb69284c", "orange cloth2-a18546c84607a308", "notebook-b523829ed1d7c1af" ], "reasoning": "1. At 02:15, the user started cleaning the tupperware container (tupperware container1-3851fd61a2b6b508).\n2. At 02:42, the user finished scrubbing the container (tupperware container1-3851fd61a2b6b508).\n3. At 02:45, the user began scrubbing the whisk (whisk-5d77d2aed09bb1f8).\n4. At 03:07, the user finished scrubbing the whisk (whisk-5d77d2aed09bb1f8).\n5. The user scrubbed the tupperware container (tupperware container1-3851fd61a2b6b508) for 27 seconds.\n6. The user scrubbed the whisk (whisk-5d77d2aed09bb1f8) for 22 seconds.\n7. If I subtract 22 seconds from 27 seconds, there is a 5 second difference.", "referenced_object_masks": [ "whisk-5d77d2aed09bb1f8", "tupperware container1-3851fd61a2b6b508" ] }, { "key": "P04-20240414-072251:982a7e7880dabeecd6a9f784c0135c26004f86fe", "video_id": "P04-20240414-072251", "question": "When I spray the spray bottle for the 6th time, what is directly in front of me?", "answer_choice_0": "The cook top", "answer_choice_1": "The sink", "answer_choice_2": "The coffee maker", "answer_choice_3": "A hand towel", "answer_choice_4": "The dining table", "answer_id": 0, "answer": "The cook top", "question_type": [ "Object Recognition", "Egocentric & Spatial Perception", "Action Recognition", "Counting" ], "objects_of_interest": [ "frying pan-a597ca21304f3cc2", "sponge-b398c8dd92cbdd13", "soap dispenser-6fd8734e696816a6", "spoon-75e2910407a73328", "tupperware container1-3851fd61a2b6b508", "tupperware container2-e48346135f204a2c", "whisk-5d77d2aed09bb1f8", "knife-2753a32bae2cd912", "cutlery holder-5ea87e3a428e2c29", "spoon2-8286fa3658081066", "measuring cup-57ae3945e83598d4", "chopping board-699c09d1a4397b26", "lid of plastic container-5bbd6d234a50b302", "mug-06ef18483b937dfa", "bowl-2920ee2f584c003d", "rubber spatula-d8d6ffda1c12ee1e", "plate-3b3ca5d805b1aebc", "mug2-40eac9d84cb1204b", "mug3-e9fc61163aac02c7", "orange cloth-5746fcd9cf7fa0f1", "metal mixing bowl-b84f557ab9339a6d", "plastic bowl-122d1fe6cd83b081", "other stack of plastic containers-2d1240bdca1e4623", "blue cloth-ca7ab436f9779dbd", "spray bottle-53569dfaf5f18839", "cooling rack-568c05aa9895bdba", "plate2-7e8f8234a232897b", "bowl2-31258aa986ce8f5f", "pen-1ee13d6ceb69284c", "orange cloth2-a18546c84607a308", "notebook-b523829ed1d7c1af" ], "reasoning": "1. I watched the video and looked for when the user started using the spray bottle (spray bottle-53569dfaf5f18839). I found that this occurred at 19:58 as they grabbed the spray bottle (spray bottle-53569dfaf5f18839) from a ledge above the sink .\n2. Then, I watched to count the number of times the user sprays the spray bottle (spray bottle-53569dfaf5f18839). The first two sprays occurred from 19:59-20:00 toward the corner of the countertop, to the left of the sink .\n3. Two additional sprays from the spray bottle (spray bottle-53569dfaf5f18839) occurred from 20:01-20:02, directly in front of the sink . I counted these as the 3rd and 4th sprays.\n4. From 20:07-20:08, there are 3 more sprays. These count as the 5th, 6th, and 7th sprays. \n5. Since this was the number from the question, I looked at what was directly in front of the user. I found the cook top .", "referenced_object_masks": [ "spray bottle-53569dfaf5f18839" ] }, { "key": "P04-20240414-072251:b43d796778871d386f4de9a4ea057291abee91ef", "video_id": "P04-20240414-072251", "question": "What would the product be if I multiplied the number of kitchen utensils I dried by the number of bowls I dried, including the tupperware bowl and large silver bowl?", "answer_choice_0": "25", "answer_choice_1": "30", "answer_choice_2": "20", "answer_choice_3": "10", "answer_choice_4": "15", "answer_id": 0, "answer": "25", "question_type": [ "Counting", "Action Recognition", "Numerical Reasoning", "Object Recognition" ], "objects_of_interest": [ "frying pan-a597ca21304f3cc2", "sponge-b398c8dd92cbdd13", "soap dispenser-6fd8734e696816a6", "spoon-75e2910407a73328", "tupperware container1-3851fd61a2b6b508", "tupperware container2-e48346135f204a2c", "whisk-5d77d2aed09bb1f8", "knife-2753a32bae2cd912", "cutlery holder-5ea87e3a428e2c29", "spoon2-8286fa3658081066", "measuring cup-57ae3945e83598d4", "chopping board-699c09d1a4397b26", "lid of plastic container-5bbd6d234a50b302", "mug-06ef18483b937dfa", "bowl-2920ee2f584c003d", "rubber spatula-d8d6ffda1c12ee1e", "plate-3b3ca5d805b1aebc", "mug2-40eac9d84cb1204b", "mug3-e9fc61163aac02c7", "orange cloth-5746fcd9cf7fa0f1", "metal mixing bowl-b84f557ab9339a6d", "plastic bowl-122d1fe6cd83b081", "other stack of plastic containers-2d1240bdca1e4623", "blue cloth-ca7ab436f9779dbd", "spray bottle-53569dfaf5f18839", "cooling rack-568c05aa9895bdba", "plate2-7e8f8234a232897b", "bowl2-31258aa986ce8f5f", "pen-1ee13d6ceb69284c", "orange cloth2-a18546c84607a308", "notebook-b523829ed1d7c1af" ], "reasoning": "1. At 14:40, the user picked up the rubber spatula (rubber spatula-d8d6ffda1c12ee1e) to dry it.\n2. At 14:51, the user picked up a whisk (whisk-5d77d2aed09bb1f8), making this utensil number 2.\n3. At 14:58, the user picked up the spoon (spoon-75e2910407a73328).\n4. At 15:05, the user picked up another spoon , making it utensil number 4.\n5. At 15:12, the user picked up the knife (knife-2753a32bae2cd912), making it number 5. \n6. The spoons are stacked on top of each other.\n7. At 15:37, the user dried the small bowl .\n8. At 15:48, the user grabbed the tupperware container (tupperware container1-3851fd61a2b6b508).\n9. At 16:09, the user picked up the other tupperware container (tupperware container2-e48346135f204a2c) and stacked it into the other one.\n10. The large silver bowl (metal mixing bowl-b84f557ab9339a6d) is already present, making this bowl number 4.\n11. At 17:44, the user picks up the last bowl (bowl2-31258aa986ce8f5f) to dry, making it bowl number 5.\n12. There are 5 utensils and 5 bowls . I multiplied 5 by 5 to get 25.", "referenced_object_masks": [ "tupperware container2-e48346135f204a2c", "whisk-5d77d2aed09bb1f8", "tupperware container1-3851fd61a2b6b508", "knife-2753a32bae2cd912", "metal mixing bowl-b84f557ab9339a6d", "rubber spatula-d8d6ffda1c12ee1e", "bowl2-31258aa986ce8f5f", "spoon-75e2910407a73328" ] }, { "key": "P04-20240414-072251:ba9076380e79f2d71e4caa7b7a544fd13f15f0d6", "video_id": "P04-20240414-072251", "question": "Which item did I dry with the orange rag before the first spoon?", "answer_choice_0": "Knife", "answer_choice_1": "Mug", "answer_choice_2": "Whisk", "answer_choice_3": "Measuring cup", "answer_choice_4": "Spatula", "answer_id": 2, "answer": "Whisk", "question_type": [ "Event Occurence", "Object Recognition", "State Changes", "Action Recognition" ], "objects_of_interest": [ "frying pan-a597ca21304f3cc2", "sponge-b398c8dd92cbdd13", "soap dispenser-6fd8734e696816a6", "spoon-75e2910407a73328", "tupperware container1-3851fd61a2b6b508", "tupperware container2-e48346135f204a2c", "whisk-5d77d2aed09bb1f8", "knife-2753a32bae2cd912", "cutlery holder-5ea87e3a428e2c29", "spoon2-8286fa3658081066", "measuring cup-57ae3945e83598d4", "chopping board-699c09d1a4397b26", "lid of plastic container-5bbd6d234a50b302", "mug-06ef18483b937dfa", "bowl-2920ee2f584c003d", "rubber spatula-d8d6ffda1c12ee1e", "plate-3b3ca5d805b1aebc", "mug2-40eac9d84cb1204b", "mug3-e9fc61163aac02c7", "orange cloth-5746fcd9cf7fa0f1", "metal mixing bowl-b84f557ab9339a6d", "plastic bowl-122d1fe6cd83b081", "other stack of plastic containers-2d1240bdca1e4623", "blue cloth-ca7ab436f9779dbd", "spray bottle-53569dfaf5f18839", "cooling rack-568c05aa9895bdba", "plate2-7e8f8234a232897b", "bowl2-31258aa986ce8f5f", "pen-1ee13d6ceb69284c", "orange cloth2-a18546c84607a308", "notebook-b523829ed1d7c1af" ], "reasoning": "1. I looked for the moment the user was using an orange rag (orange cloth2-a18546c84607a308). At 14:30 the user grabs a folded orange rag (orange cloth2-a18546c84607a308) from the countertop. \n2. The user then proceeds to start the coffee maker before grabbing the first item to dry, which is a blue spatula (rubber spatula-d8d6ffda1c12ee1e) at 14:40. The second item is a whisk (whisk-5d77d2aed09bb1f8) at 14:51, and the third item is a spoon (spoon-75e2910407a73328) at 14:59. Therefore, the item the user dried before the first spoon was the whisk (whisk-5d77d2aed09bb1f8).", "referenced_object_masks": [ "whisk-5d77d2aed09bb1f8", "orange cloth2-a18546c84607a308", "rubber spatula-d8d6ffda1c12ee1e", "spoon-75e2910407a73328" ] }, { "key": "P04-20240414-072251:da2923822848dc6a2f7232a40d58f242280462af", "video_id": "P04-20240414-072251", "question": "If instead of turning and placing the dry items to my left, I decided to turn and place them 90 degrees to my right, where would I be putting the dry items?", "answer_choice_0": "Napkin holder", "answer_choice_1": "Cutting board", "answer_choice_2": "Small counter", "answer_choice_3": "Stools", "answer_choice_4": "Coffee maker", "answer_id": 2, "answer": "Small counter", "question_type": [ "Counting", "Event Occurence", "Object Recognition", "Egocentric & Spatial Perception", "Counterfactual" ], "objects_of_interest": [ "frying pan-a597ca21304f3cc2", "sponge-b398c8dd92cbdd13", "soap dispenser-6fd8734e696816a6", "spoon-75e2910407a73328", "tupperware container1-3851fd61a2b6b508", "tupperware container2-e48346135f204a2c", "whisk-5d77d2aed09bb1f8", "knife-2753a32bae2cd912", "cutlery holder-5ea87e3a428e2c29", "spoon2-8286fa3658081066", "measuring cup-57ae3945e83598d4", "chopping board-699c09d1a4397b26", "lid of plastic container-5bbd6d234a50b302", "mug-06ef18483b937dfa", "bowl-2920ee2f584c003d", "rubber spatula-d8d6ffda1c12ee1e", "plate-3b3ca5d805b1aebc", "mug2-40eac9d84cb1204b", "mug3-e9fc61163aac02c7", "orange cloth-5746fcd9cf7fa0f1", "metal mixing bowl-b84f557ab9339a6d", "plastic bowl-122d1fe6cd83b081", "other stack of plastic containers-2d1240bdca1e4623", "blue cloth-ca7ab436f9779dbd", "spray bottle-53569dfaf5f18839", "cooling rack-568c05aa9895bdba", "plate2-7e8f8234a232897b", "bowl2-31258aa986ce8f5f", "pen-1ee13d6ceb69284c", "orange cloth2-a18546c84607a308", "notebook-b523829ed1d7c1af" ], "reasoning": "1. I looked for the segment where the user is drying the dishes at 14:40. In this segment, the user grabs the wet items from the dish rack, dries them with an orange cloth (orange cloth2-a18546c84607a308) and turns 90 degrees left to place the dry items on the kitchen counter. \n2. I looked for a moment in the video where a wider view of the kitchen is presented to identify what is at the right of the user if the user was standing in front of the sink. I scrolled back to the moment the user grabbed the orange rag (orange cloth2-a18546c84607a308) at 14:30 for a better view of what was on the right side of the kitchen. I noticed that there's a partial wall next to the sink, followed by a small light gray counter with two gray stools underneath. On top, there's a small white container with folded rags. \n3. I concluded that if the user decided to place the dry items 90 degrees to their right, then the user would be placing the items on the small light gray counter.", "referenced_object_masks": [ "orange cloth2-a18546c84607a308" ] }, { "key": "P04-20240414-072251:f893c111f7e9943abc2decf27d32a19e0fad5018", "video_id": "P04-20240414-072251", "question": "If I multiplied the number of halves I folded the orange towel into by the number of towels I used after I folded it, what would the product be?", "answer_choice_0": "3", "answer_choice_1": "2", "answer_choice_2": "5", "answer_choice_3": "4", "answer_choice_4": "1", "answer_id": 3, "answer": "4", "question_type": [ "Counting", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "frying pan-a597ca21304f3cc2", "sponge-b398c8dd92cbdd13", "soap dispenser-6fd8734e696816a6", "spoon-75e2910407a73328", "tupperware container1-3851fd61a2b6b508", "tupperware container2-e48346135f204a2c", "whisk-5d77d2aed09bb1f8", "knife-2753a32bae2cd912", "cutlery holder-5ea87e3a428e2c29", "spoon2-8286fa3658081066", "measuring cup-57ae3945e83598d4", "chopping board-699c09d1a4397b26", "lid of plastic container-5bbd6d234a50b302", "mug-06ef18483b937dfa", "bowl-2920ee2f584c003d", "rubber spatula-d8d6ffda1c12ee1e", "plate-3b3ca5d805b1aebc", "mug2-40eac9d84cb1204b", "mug3-e9fc61163aac02c7", "orange cloth-5746fcd9cf7fa0f1", "metal mixing bowl-b84f557ab9339a6d", "plastic bowl-122d1fe6cd83b081", "other stack of plastic containers-2d1240bdca1e4623", "blue cloth-ca7ab436f9779dbd", "spray bottle-53569dfaf5f18839", "cooling rack-568c05aa9895bdba", "plate2-7e8f8234a232897b", "bowl2-31258aa986ce8f5f", "pen-1ee13d6ceb69284c", "orange cloth2-a18546c84607a308", "notebook-b523829ed1d7c1af" ], "reasoning": "1. At 19:25, the user picked up the towel (orange cloth-5746fcd9cf7fa0f1) off the chopping board (chopping board-699c09d1a4397b26).\n2. At 19:27, the user folded the towel (orange cloth-5746fcd9cf7fa0f1) in half.\n3. At 19:28, the user folded the towel (orange cloth-5746fcd9cf7fa0f1) into another half.\n4. The user folded the towel (orange cloth-5746fcd9cf7fa0f1) into 2 halves.\n5. At 19:33, the user grabbed the blue towel (blue cloth-ca7ab436f9779dbd) and started cleaning the counters.\n6. At 20:27, the user grabbed the black towel to dry their hands.\n7. The user folded the orange towel (orange cloth-5746fcd9cf7fa0f1) into 2 halves and used 2 towels afterwards. 2 mulitplied by 2 gives a product of 4.", "referenced_object_masks": [ "blue cloth-ca7ab436f9779dbd", "orange cloth-5746fcd9cf7fa0f1", "chopping board-699c09d1a4397b26" ] }, { "key": "P04-20240414-162242:552a2512416e2f5dca9e03a919b66f900e28b139", "video_id": "P04-20240414-162242", "question": "Rounding to the nearest 10 seconds, how much time passed between taking out the potato masher and taking out the first potato?", "answer_choice_0": "00:50", "answer_choice_1": "00:30", "answer_choice_2": "01:30", "answer_choice_3": "01:50", "answer_choice_4": "01:10", "answer_id": 0, "answer": "00:50", "question_type": [ "Action Recognition", "Object Recognition", "Temporal Reasoning", "Numerical Reasoning" ], "objects_of_interest": [ "wooden chopping board-0b4c111b5f8311c5", "cooling rack-b7aa617f91554398", "bottle of cleaning solution-9ad03203a3ae4da1", "soap dispenser-98cc607fb79bd70f", "soap dispenser2-1463093e98c20660", "cooling rack2-acd29b44bb88be3c", "lid-40c466381e64904e", "pan-ac3b6c7808b929fe", "pot-773772759648ba6c", "wooden spatula-7146f5bb248dd7f3", "metallic cup-9c50054583bf17ce", "potato masher-f7d9166c69814708", "ginger-5f42d99f2836f895", "plastic bag-9d58364b528502de", "bag of potatoes-b4735ae204714298", "pair of scissors-d98e6f99871f9337", "potato-1aaca89a9f523f15", "potato2-36c510e75ef741c2", "olive oil bottle-6ec164b7a134debc", "jar of salt-eaf61ba6def63f0b" ], "reasoning": "1. I watched the video and saw that the user took out the potato masher (potato masher-f7d9166c69814708) at 01:11. \n2. I saw that the user took out a potato (potato-1aaca89a9f523f15) for the first time at 02:02. \n3. I subtracted 02:02-01:11=00:51.\n4. I rounded to the nearest 10 seconds and determined the answer was 00:50.", "referenced_object_masks": [ "potato-1aaca89a9f523f15", "potato masher-f7d9166c69814708" ] }, { "key": "P04-20240414-162242:6a423dfef07578f783055ecfb83e4f56822accb9", "video_id": "P04-20240414-162242", "question": "Which was greater, and by how much: The number of items I took out of cabinets, or the number of items I took out of drawers?", "answer_choice_0": "Drawers--by 3", "answer_choice_1": "Drawers--by 1", "answer_choice_2": "Neither--the numbers were equal", "answer_choice_3": "Cabinets--by 3", "answer_choice_4": "Cabinets--by 1", "answer_id": 4, "answer": "Cabinets--by 1", "question_type": [ "Object Recognition", "Action Recognition", "Situational Awareness", "Counting", "Numerical Reasoning" ], "objects_of_interest": [ "wooden chopping board-0b4c111b5f8311c5", "cooling rack-b7aa617f91554398", "bottle of cleaning solution-9ad03203a3ae4da1", "soap dispenser-98cc607fb79bd70f", "soap dispenser2-1463093e98c20660", "cooling rack2-acd29b44bb88be3c", "lid-40c466381e64904e", "pan-ac3b6c7808b929fe", "pot-773772759648ba6c", "wooden spatula-7146f5bb248dd7f3", "metallic cup-9c50054583bf17ce", "potato masher-f7d9166c69814708", "ginger-5f42d99f2836f895", "plastic bag-9d58364b528502de", "bag of potatoes-b4735ae204714298", "pair of scissors-d98e6f99871f9337", "potato-1aaca89a9f523f15", "potato2-36c510e75ef741c2", "olive oil bottle-6ec164b7a134debc", "jar of salt-eaf61ba6def63f0b" ], "reasoning": "1. I watched the video and counted each item that was taken out of a cabinet. These items were: A lid (lid-40c466381e64904e) at 00:40, a pan (pan-ac3b6c7808b929fe) at 00:45, a pot (pot-773772759648ba6c) at 00:58, a bottle of oil (olive oil bottle-6ec164b7a134debc) at 02:46, and a jar of salt (jar of salt-eaf61ba6def63f0b) at 02:47. This makes a total of 5 items. \n2. I counted each item that was taken out of a drawer. These items were: A wooden spatula (wooden spatula-7146f5bb248dd7f3) at 01:06, a silver cup (metallic cup-9c50054583bf17ce) at 01:08, a potato masher (potato masher-f7d9166c69814708) at 01:11, and a pair of scissors (pair of scissors-d98e6f99871f9337) at 01:53. This makes a total of 4 items.\n3. I subtracted 5-4 to find a difference of 1.", "referenced_object_masks": [ "metallic cup-9c50054583bf17ce", "pair of scissors-d98e6f99871f9337", "potato masher-f7d9166c69814708", "pot-773772759648ba6c", "wooden spatula-7146f5bb248dd7f3", "lid-40c466381e64904e", "pan-ac3b6c7808b929fe", "jar of salt-eaf61ba6def63f0b", "olive oil bottle-6ec164b7a134debc" ] }, { "key": "P04-20240414-162242:a389a0ad41760616f0800034242df51b7564ebca", "video_id": "P04-20240414-162242", "question": "What would be the product if I multiplied the number of plastic bags I handled by the number of potatoes I decided to cook plus the number of times I opened a cabinet?", "answer_choice_0": "8", "answer_choice_1": "10", "answer_choice_2": "4", "answer_choice_3": "6", "answer_choice_4": "5", "answer_id": 0, "answer": "8", "question_type": [ "Action Recognition", "Object Recognition", "Counting", "Numerical Reasoning" ], "objects_of_interest": [ "wooden chopping board-0b4c111b5f8311c5", "cooling rack-b7aa617f91554398", "bottle of cleaning solution-9ad03203a3ae4da1", "soap dispenser-98cc607fb79bd70f", "soap dispenser2-1463093e98c20660", "cooling rack2-acd29b44bb88be3c", "lid-40c466381e64904e", "pan-ac3b6c7808b929fe", "pot-773772759648ba6c", "wooden spatula-7146f5bb248dd7f3", "metallic cup-9c50054583bf17ce", "potato masher-f7d9166c69814708", "ginger-5f42d99f2836f895", "plastic bag-9d58364b528502de", "bag of potatoes-b4735ae204714298", "pair of scissors-d98e6f99871f9337", "potato-1aaca89a9f523f15", "potato2-36c510e75ef741c2", "olive oil bottle-6ec164b7a134debc", "jar of salt-eaf61ba6def63f0b" ], "reasoning": "1. I watched the video and identified the first time the user handled a unique plastic bag. These items were: the large plastic bag (plastic bag-9d58364b528502de), first handled at 01:42, and the plastic sleeve that held the bag of potatoes (bag of potatoes-b4735ae204714298), first handled at 01:43. This makes a total of 2 times the user handled a plastic bag. \n2. I looked for the number of potatoes that the user decided to cook. I saw the user put a potato (potato-1aaca89a9f523f15) on the cutting board (wooden chopping board-0b4c111b5f8311c5) at 02:03. I saw the user put a second potato (potato2-36c510e75ef741c2) on the cutting board (wooden chopping board-0b4c111b5f8311c5) at 02:09. However, the user then put the second potato (potato2-36c510e75ef741c2) back into the bag of potatoes (bag of potatoes-b4735ae204714298), therefore I did not count this in the total number of potatoes they decided to cook. This made a total of 1. \n3. I counted each time the user opened a cabinet. This occurred at 00:35, 00:54, and 02:43. This makes a total of 3 occurrences.\n4. I added the number of potatoes they decided to cook (1) to the number of times a cabinet was opened (3): 1+3=4. \n5. I multiplied the number of plastic bags handled (2) by the previous sum (4): 2*4=8.", "referenced_object_masks": [ "potato2-36c510e75ef741c2", "potato-1aaca89a9f523f15", "bag of potatoes-b4735ae204714298", "wooden chopping board-0b4c111b5f8311c5", "plastic bag-9d58364b528502de" ] }, { "key": "P04-20240414-162242:b4a024c5e9c89db6dd41a45fdc24a10cb6631539", "video_id": "P04-20240414-162242", "question": "Which was greater, and by how much: The number of food items I put directly on the cutting board, or the number of items (not counting lids) I put to the right of the cutting board?", "answer_choice_0": "Items to the right--by 2", "answer_choice_1": "Items on board--by 2", "answer_choice_2": "Items to the right--by 5", "answer_choice_3": "Neither--the numbers were equal", "answer_choice_4": "Items on board--by 5", "answer_id": 0, "answer": "Items to the right--by 2", "question_type": [ "Object Recognition", "Counting", "Numerical Reasoning", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "wooden chopping board-0b4c111b5f8311c5", "cooling rack-b7aa617f91554398", "bottle of cleaning solution-9ad03203a3ae4da1", "soap dispenser-98cc607fb79bd70f", "soap dispenser2-1463093e98c20660", "cooling rack2-acd29b44bb88be3c", "lid-40c466381e64904e", "pan-ac3b6c7808b929fe", "pot-773772759648ba6c", "wooden spatula-7146f5bb248dd7f3", "metallic cup-9c50054583bf17ce", "potato masher-f7d9166c69814708", "ginger-5f42d99f2836f895", "plastic bag-9d58364b528502de", "bag of potatoes-b4735ae204714298", "pair of scissors-d98e6f99871f9337", "potato-1aaca89a9f523f15", "potato2-36c510e75ef741c2", "olive oil bottle-6ec164b7a134debc", "jar of salt-eaf61ba6def63f0b" ], "reasoning": "1. I watched the video and counted every food item placed directly on the cutting board (wooden chopping board-0b4c111b5f8311c5). These items were: a small chunk of ginger at 01:40, a potato (potato-1aaca89a9f523f15) at 02:03, and a second potato (potato2-36c510e75ef741c2) at 02:09. I also saw the user place a whole bag of potatoes (bag of potatoes-b4735ae204714298) on the cutting board (wooden chopping board-0b4c111b5f8311c5) at 01:49, but did not count this since the food items were separated from the cutting board (wooden chopping board-0b4c111b5f8311c5) by a plastic sleeve . Therefore, I counted a total of 3 food items. \n2. I counted the items placed to the right of the cutting board (wooden chopping board-0b4c111b5f8311c5). These items were: a pan (pan-ac3b6c7808b929fe) at 00:53, a pot (pot-773772759648ba6c) at 01:00, a wooden spatula (wooden spatula-7146f5bb248dd7f3) at 01:07, a silver cup (metallic cup-9c50054583bf17ce) at 01:09, and a potato masher (potato masher-f7d9166c69814708) at 01:13. This makes a total of 5 items (not including any lids). \n3. I subtracted 5-3 to find a difference of 2.", "referenced_object_masks": [ "potato2-36c510e75ef741c2", "metallic cup-9c50054583bf17ce", "potato-1aaca89a9f523f15", "potato masher-f7d9166c69814708", "pot-773772759648ba6c", "bag of potatoes-b4735ae204714298", "wooden spatula-7146f5bb248dd7f3", "wooden chopping board-0b4c111b5f8311c5", "pan-ac3b6c7808b929fe" ] }, { "key": "P04-20240414-162242:f29a4cfa62b8b975ee198a2af1ce551d109afaea", "video_id": "P04-20240414-162242", "question": "When I took out the lid and pan, how many degrees clockwise would I turn if I wanted to face the place where I found the oil?", "answer_choice_0": "225", "answer_choice_1": "270", "answer_choice_2": "90", "answer_choice_3": "135", "answer_choice_4": "0", "answer_id": 4, "answer": "0", "question_type": [ "Egocentric & Spatial Perception", "Temporal Reasoning", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "wooden chopping board-0b4c111b5f8311c5", "cooling rack-b7aa617f91554398", "bottle of cleaning solution-9ad03203a3ae4da1", "soap dispenser-98cc607fb79bd70f", "soap dispenser2-1463093e98c20660", "cooling rack2-acd29b44bb88be3c", "lid-40c466381e64904e", "pan-ac3b6c7808b929fe", "pot-773772759648ba6c", "wooden spatula-7146f5bb248dd7f3", "metallic cup-9c50054583bf17ce", "potato masher-f7d9166c69814708", "ginger-5f42d99f2836f895", "plastic bag-9d58364b528502de", "bag of potatoes-b4735ae204714298", "pair of scissors-d98e6f99871f9337", "potato-1aaca89a9f523f15", "potato2-36c510e75ef741c2", "olive oil bottle-6ec164b7a134debc", "jar of salt-eaf61ba6def63f0b" ], "reasoning": "1. I watched the video and saw that the user took out a lid (lid-40c466381e64904e) at 00:40 and a pan (pan-ac3b6c7808b929fe) at 00:45. \n2. I observed the surroundings from 00:40-00:45 and saw that the user had taken the items out of a cabinet below a countertop left of the stove. I also noted that the counter ended just to the left of where the user was facing. \n3. I looked for the bottle of oil (olive oil bottle-6ec164b7a134debc) and saw the user take it out of a cabinet at 02:46. \n4. I observed the surroundings at 02:46 and noted that the user had taken the item out of a cabinet below a countertop. The countertop ended just to the left. When the user stood up and turned to the right from 02:48-02:49, I saw the stove to the right of the counter they had just left. Therefore, I determined that the user took the bottle of oil (olive oil bottle-6ec164b7a134debc) from the same cabinet as the lid (lid-40c466381e64904e) at 00:40 and pan (pan-ac3b6c7808b929fe).", "referenced_object_masks": [ "pan-ac3b6c7808b929fe", "lid-40c466381e64904e", "olive oil bottle-6ec164b7a134debc" ] }, { "key": "P04-20240414-162750:0e275b1465c236cda75b438945235fdb100995c9", "video_id": "P04-20240414-162750", "question": "If I picked the cilantro leaves from the stems as quickly as I did for the second sprig, how much time would it have taken me to pick the cilantro sprigs I prepared for chopping?", "answer_choice_0": "80 seconds", "answer_choice_1": "60 seconds", "answer_choice_2": "53 seconds", "answer_choice_3": "67 seconds", "answer_choice_4": "72 seconds", "answer_id": 4, "answer": "72 seconds", "question_type": [ "Counterfactual", "Counting", "Temporal Reasoning", "Numerical Reasoning" ], "objects_of_interest": [ "packet of ghee-46e8386403e45d8f", "spoon-e11f874d227b40fe", "spoon2-40ca91a14a0c120a", "spoon3-29ee510753a90ec2", "bag of granola-fa4f260f54bc9960", "turmeric powder pack-4e137df5263ff0c9", "pack of mustard seeds-fd6caf8da234e179", "pan with bag of chips-658f0fed155dead9", "bag of chips-c5a78bf9e01faaf6", "weighing scale-96274e148c532468", "box of dosa mix-aafcde40caba9409", "bag of dal-d063664686e82143", "notebook-d4898fa4e2d94567", "pen-5705112568133bc9", "green chilli 1-f774f141714cc834", "green chilli 2-28ed5a9e602b5bab", "stack of coriander leaves-4680c3e8a961d61d", "plastic container box-8f9267227c505ddb", "few twigs of coriander leaves-49ac68f768efc20a", "large brown onion-5a8d4da0478588a4", "large brown onion2-b58f4229a688338a", "knife-791fde7b1d890af0", "pair of scissors-ecf82534fc547bef", "potato-4cd2a4e4fb71c2d7", "pot-03341a974f69ea33", "plate-afe429b441ecbc1a", "cap of pen-bbc3d2026cc5055e", "blue coloured plastic chopping board-50d346f59d847512", "white coloured plastic chopping board-772b76f7fce60bf3", "potato half-e3e326773e5c2013", "lid of pressure cooker-e663da0dd466b2de", "second half of potato-a7811a2741400b5e", "pressure cooker regulator-397a057151fe2b7e", "metallic cup-60e8c95c9db50482", "electric hob-f53f289a5d1384b0", "pan-0dda3ddd9d1cf9a1", "rubbish bin-3072870f57109f60", "second half of onion-14e8a3d7c61d5416", "piece of ginger-da6d99888c17dc27", "peeled ginger skin-1a58d3f854c1dc40", "lid of pan-c43e4a69de280f65", "bottle of olive oil-4bf1a7995c33f06c", "cap of olive oil bottle-ad4e691e03adadc5" ], "reasoning": "1. I looked for the user to pick cilantro leaves from stems and saw them begin at 14:16.\n2. I saw the user pick up the second cilantro sprig at 14:20 and finish picking its leaves at 14:29.\n3. Subtracting 14:20 from 14:29 gives me a total time of 9 seconds spent picking the second cilantro sprig .\n4. I continued watching to see how many more cilantro sprigs the user prepared for chopping, and saw them pick 6 more from 14:29-15:54, at which point the user began chopping. This gave me a total of 8 sprigs.\n5. I multiplied the 8 sprigs by 9 seconds to get a total of 72 seconds.", "referenced_object_masks": [] }, { "key": "P04-20240414-162750:1e11cebcc4d8f10f40ce6af9bd5f2dc569dcc60e", "video_id": "P04-20240414-162750", "question": "How many towels are stacked on top of the towel that most closely matches the color of the first object I removed from the low cabinet?", "answer_choice_0": "3", "answer_choice_1": "5", "answer_choice_2": "4", "answer_choice_3": "2", "answer_choice_4": "1", "answer_id": 3, "answer": "2", "question_type": [ "Counting", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "packet of ghee-46e8386403e45d8f", "spoon-e11f874d227b40fe", "spoon2-40ca91a14a0c120a", "spoon3-29ee510753a90ec2", "bag of granola-fa4f260f54bc9960", "turmeric powder pack-4e137df5263ff0c9", "pack of mustard seeds-fd6caf8da234e179", "pan with bag of chips-658f0fed155dead9", "bag of chips-c5a78bf9e01faaf6", "weighing scale-96274e148c532468", "box of dosa mix-aafcde40caba9409", "bag of dal-d063664686e82143", "notebook-d4898fa4e2d94567", "pen-5705112568133bc9", "green chilli 1-f774f141714cc834", "green chilli 2-28ed5a9e602b5bab", "stack of coriander leaves-4680c3e8a961d61d", "plastic container box-8f9267227c505ddb", "few twigs of coriander leaves-49ac68f768efc20a", "large brown onion-5a8d4da0478588a4", "large brown onion2-b58f4229a688338a", "knife-791fde7b1d890af0", "pair of scissors-ecf82534fc547bef", "potato-4cd2a4e4fb71c2d7", "pot-03341a974f69ea33", "plate-afe429b441ecbc1a", "cap of pen-bbc3d2026cc5055e", "blue coloured plastic chopping board-50d346f59d847512", "white coloured plastic chopping board-772b76f7fce60bf3", "potato half-e3e326773e5c2013", "lid of pressure cooker-e663da0dd466b2de", "second half of potato-a7811a2741400b5e", "pressure cooker regulator-397a057151fe2b7e", "metallic cup-60e8c95c9db50482", "electric hob-f53f289a5d1384b0", "pan-0dda3ddd9d1cf9a1", "rubbish bin-3072870f57109f60", "second half of onion-14e8a3d7c61d5416", "piece of ginger-da6d99888c17dc27", "peeled ginger skin-1a58d3f854c1dc40", "lid of pan-c43e4a69de280f65", "bottle of olive oil-4bf1a7995c33f06c", "cap of olive oil bottle-ad4e691e03adadc5" ], "reasoning": "1. I watched for the user to remove something from the low cabinet and saw them remove a packet of ghee (packet of ghee-46e8386403e45d8f) at 00:07. I noted its color as blue.\n2. I continued watching for a stack of towels and identified them on top of a table at 02:04.\n3. I saw a single blue towel in the stack of towels. I counted an orange towel and a purple towel on top of it for a total of 2 towels.", "referenced_object_masks": [ "packet of ghee-46e8386403e45d8f" ] }, { "key": "P04-20240414-162750:6aa40023374c815f1a8536ff305e0451a2dbdfb7", "video_id": "P04-20240414-162750", "question": "Of the ingredients I weighed on the scale, what's the difference in weight between the ingredients I added to the pressure cooker and the ones I didn't?", "answer_choice_0": "143 grams", "answer_choice_1": "239 grams", "answer_choice_2": "209 grams", "answer_choice_3": "272 grams", "answer_choice_4": "129 grams", "answer_id": 0, "answer": "143 grams", "question_type": [ "Reading", "Object Recognition", "Numerical Reasoning", "Action Recognition" ], "objects_of_interest": [ "packet of ghee-46e8386403e45d8f", "spoon-e11f874d227b40fe", "spoon2-40ca91a14a0c120a", "spoon3-29ee510753a90ec2", "bag of granola-fa4f260f54bc9960", "turmeric powder pack-4e137df5263ff0c9", "pack of mustard seeds-fd6caf8da234e179", "pan with bag of chips-658f0fed155dead9", "bag of chips-c5a78bf9e01faaf6", "weighing scale-96274e148c532468", "box of dosa mix-aafcde40caba9409", "bag of dal-d063664686e82143", "notebook-d4898fa4e2d94567", "pen-5705112568133bc9", "green chilli 1-f774f141714cc834", "green chilli 2-28ed5a9e602b5bab", "stack of coriander leaves-4680c3e8a961d61d", "plastic container box-8f9267227c505ddb", "few twigs of coriander leaves-49ac68f768efc20a", "large brown onion-5a8d4da0478588a4", "large brown onion2-b58f4229a688338a", "knife-791fde7b1d890af0", "pair of scissors-ecf82534fc547bef", "potato-4cd2a4e4fb71c2d7", "pot-03341a974f69ea33", "plate-afe429b441ecbc1a", "cap of pen-bbc3d2026cc5055e", "blue coloured plastic chopping board-50d346f59d847512", "white coloured plastic chopping board-772b76f7fce60bf3", "potato half-e3e326773e5c2013", "lid of pressure cooker-e663da0dd466b2de", "second half of potato-a7811a2741400b5e", "pressure cooker regulator-397a057151fe2b7e", "metallic cup-60e8c95c9db50482", "electric hob-f53f289a5d1384b0", "pan-0dda3ddd9d1cf9a1", "rubbish bin-3072870f57109f60", "second half of onion-14e8a3d7c61d5416", "piece of ginger-da6d99888c17dc27", "peeled ginger skin-1a58d3f854c1dc40", "lid of pan-c43e4a69de280f65", "bottle of olive oil-4bf1a7995c33f06c", "cap of olive oil bottle-ad4e691e03adadc5" ], "reasoning": "1. I first looked for the user to weigh different ingredients on the scale (weighing scale-96274e148c532468) so that I could identify them and their weights.\n2. I saw the user place a plate (plate-afe429b441ecbc1a) on the scale (weighing scale-96274e148c532468) at 05:21, but this is not an ingredient, so I did not include it in my count of ingredients.\n3. I saw the user weigh a potato (potato-4cd2a4e4fb71c2d7) at 05:36 and noted the weight as 272 grams.\n4. I saw the user weigh 2 chiles (green chilli 1-f774f141714cc834, green chilli 2-28ed5a9e602b5bab) at 06:00 and noted the weight as 33 grams.\n5. I saw the user weigh an onion (large brown onion2-b58f4229a688338a) at 06:18 and noted the weight as 96 grams. The user did not weigh any more ingredients after this.\n6. From 07:31-07:34, I noted the user adding the two halves of the potato (potato half-e3e326773e5c2013, second half of potato-a7811a2741400b5e) to the pressure cooker (pot-03341a974f69ea33). The user did not add any other ingredients they weighed on the scale (weighing scale-96274e148c532468) after that.\n7. I added the weight of the 2 chiles (green chilli 1-f774f141714cc834, green chilli 2-28ed5a9e602b5bab) and the onion (large brown onion2-b58f4229a688338a) for a total of 129 grams.\n8. I then subtracted 129 grams from 272 grams for a final difference of 143 grams.", "referenced_object_masks": [ "plate-afe429b441ecbc1a", "green chilli 1-f774f141714cc834", "green chilli 2-28ed5a9e602b5bab", "pot-03341a974f69ea33", "potato-4cd2a4e4fb71c2d7", "potato half-e3e326773e5c2013", "second half of potato-a7811a2741400b5e", "weighing scale-96274e148c532468", "large brown onion2-b58f4229a688338a" ] }, { "key": "P04-20240414-162750:7a97d79f1ff2091b79cfd1b04266015a7fa1fb84", "video_id": "P04-20240414-162750", "question": "What is directly above the second unique item I removed from the fridge?", "answer_choice_0": ", A tupperware container with a white lid", "answer_choice_1": "A head of lettuce", "answer_choice_2": "A white bowl with sauce", "answer_choice_3": "A plastic bottle with a yellow cap", "answer_choice_4": "Red and yellow bell peppers", "answer_id": 2, "answer": "A white bowl with sauce", "question_type": [ "Egocentric & Spatial Perception", "Temporal Reasoning", "Object Recognition" ], "objects_of_interest": [ "packet of ghee-46e8386403e45d8f", "spoon-e11f874d227b40fe", "spoon2-40ca91a14a0c120a", "spoon3-29ee510753a90ec2", "bag of granola-fa4f260f54bc9960", "turmeric powder pack-4e137df5263ff0c9", "pack of mustard seeds-fd6caf8da234e179", "pan with bag of chips-658f0fed155dead9", "bag of chips-c5a78bf9e01faaf6", "weighing scale-96274e148c532468", "box of dosa mix-aafcde40caba9409", "bag of dal-d063664686e82143", "notebook-d4898fa4e2d94567", "pen-5705112568133bc9", "green chilli 1-f774f141714cc834", "green chilli 2-28ed5a9e602b5bab", "stack of coriander leaves-4680c3e8a961d61d", "plastic container box-8f9267227c505ddb", "few twigs of coriander leaves-49ac68f768efc20a", "large brown onion-5a8d4da0478588a4", "large brown onion2-b58f4229a688338a", "knife-791fde7b1d890af0", "pair of scissors-ecf82534fc547bef", "potato-4cd2a4e4fb71c2d7", "pot-03341a974f69ea33", "plate-afe429b441ecbc1a", "cap of pen-bbc3d2026cc5055e", "blue coloured plastic chopping board-50d346f59d847512", "white coloured plastic chopping board-772b76f7fce60bf3", "potato half-e3e326773e5c2013", "lid of pressure cooker-e663da0dd466b2de", "second half of potato-a7811a2741400b5e", "pressure cooker regulator-397a057151fe2b7e", "metallic cup-60e8c95c9db50482", "electric hob-f53f289a5d1384b0", "pan-0dda3ddd9d1cf9a1", "rubbish bin-3072870f57109f60", "second half of onion-14e8a3d7c61d5416", "piece of ginger-da6d99888c17dc27", "peeled ginger skin-1a58d3f854c1dc40", "lid of pan-c43e4a69de280f65", "bottle of olive oil-4bf1a7995c33f06c", "cap of olive oil bottle-ad4e691e03adadc5" ], "reasoning": "1. I watched for the user to remove things from the fridge and noted them removing 2 chiles (green chilli 1-f774f141714cc834, green chilli 2-28ed5a9e602b5bab) from 02:15-02:18.\n2. Because the first 2 items removed from the fridge were both chiles, I looked for the next unique item to be removed and saw the user pull out a bunch of cilantro (stack of coriander leaves-4680c3e8a961d61d) from the bottom shelf of the fridge at 02:36.\n3. I looked directly above the cilantro (stack of coriander leaves-4680c3e8a961d61d) and saw a white bowl with some kind of sauce in it on the top shelf.", "referenced_object_masks": [ "green chilli 1-f774f141714cc834", "green chilli 2-28ed5a9e602b5bab", "stack of coriander leaves-4680c3e8a961d61d" ] }, { "key": "P04-20240414-162750:7df25ddd8e9d3888d1ed2a7025624d7689e1c378", "video_id": "P04-20240414-162750", "question": "What do the third item I cut on the blue cutting board and the final item I cut on the blue cutting board have in common?", "answer_choice_0": "They are both fruits", "answer_choice_1": "They are both white", "answer_choice_2": "They are both yellow", "answer_choice_3": "They are both green", "answer_choice_4": "They are both peppers", "answer_id": 3, "answer": "They are both green", "question_type": [ "Object Recognition", "Counting", "Action Recognition" ], "objects_of_interest": [ "packet of ghee-46e8386403e45d8f", "spoon-e11f874d227b40fe", "spoon2-40ca91a14a0c120a", "spoon3-29ee510753a90ec2", "bag of granola-fa4f260f54bc9960", "turmeric powder pack-4e137df5263ff0c9", "pack of mustard seeds-fd6caf8da234e179", "pan with bag of chips-658f0fed155dead9", "bag of chips-c5a78bf9e01faaf6", "weighing scale-96274e148c532468", "box of dosa mix-aafcde40caba9409", "bag of dal-d063664686e82143", "notebook-d4898fa4e2d94567", "pen-5705112568133bc9", "green chilli 1-f774f141714cc834", "green chilli 2-28ed5a9e602b5bab", "stack of coriander leaves-4680c3e8a961d61d", "plastic container box-8f9267227c505ddb", "few twigs of coriander leaves-49ac68f768efc20a", "large brown onion-5a8d4da0478588a4", "large brown onion2-b58f4229a688338a", "knife-791fde7b1d890af0", "pair of scissors-ecf82534fc547bef", "potato-4cd2a4e4fb71c2d7", "pot-03341a974f69ea33", "plate-afe429b441ecbc1a", "cap of pen-bbc3d2026cc5055e", "blue coloured plastic chopping board-50d346f59d847512", "white coloured plastic chopping board-772b76f7fce60bf3", "potato half-e3e326773e5c2013", "lid of pressure cooker-e663da0dd466b2de", "second half of potato-a7811a2741400b5e", "pressure cooker regulator-397a057151fe2b7e", "metallic cup-60e8c95c9db50482", "electric hob-f53f289a5d1384b0", "pan-0dda3ddd9d1cf9a1", "rubbish bin-3072870f57109f60", "second half of onion-14e8a3d7c61d5416", "piece of ginger-da6d99888c17dc27", "peeled ginger skin-1a58d3f854c1dc40", "lid of pan-c43e4a69de280f65", "bottle of olive oil-4bf1a7995c33f06c", "cap of olive oil bottle-ad4e691e03adadc5" ], "reasoning": "1. First, I looked for the user to begin cutting items on the blue cutting board (blue coloured plastic chopping board-50d346f59d847512) and found this moment at 08:41 when they cut an onion (large brown onion2-b58f4229a688338a). \n2. Next, I looked for the user to cut a second and third item, and found that they cut the second item, a green pepper, at 10:33 (green chilli 1-f774f141714cc834), and a third green pepper (green chilli 2-28ed5a9e602b5bab) at 10:41. \n3. Then, I looked for the user to cut a final item on the blue cutting board (blue coloured plastic chopping board-50d346f59d847512) , and found that after cutting ginger (piece of ginger-da6d99888c17dc27) and crumbling coriander leaves onto the board (blue coloured plastic chopping board-50d346f59d847512), the user cuts green coriander leaves at 16:01. Since the user never cuts anything else on the cutting board (blue coloured plastic chopping board-50d346f59d847512) , I determined that coriander leaves were the final item that they cut. \n4. Finally, since both the coriander and the pepper (green chilli 2-28ed5a9e602b5bab) are green, I determined that this is what they have in common.", "referenced_object_masks": [ "blue coloured plastic chopping board-50d346f59d847512", "green chilli 1-f774f141714cc834", "piece of ginger-da6d99888c17dc27", "green chilli 2-28ed5a9e602b5bab", "large brown onion2-b58f4229a688338a" ] }, { "key": "P04-20240414-162750:9a885ea0623d0234efa2a8a7ee667b7d3b9a4dd7", "video_id": "P04-20240414-162750", "question": "How would I move through the kitchen if I threw away the onion immediately after picking it up the first time?", "answer_choice_0": "Turn around 180 degrees, walk forward, and turn left 90 degrees", "answer_choice_1": "Turn left 90 degrees, walk forward, and turn left 90 degrees", "answer_choice_2": "Walk forward and turn right 90 degrees", "answer_choice_3": "Turn around 180 degrees, walk forward, and turn right 90 degrees", "answer_choice_4": "Turn right 90 degrees and walk forward", "answer_id": 3, "answer": "Turn around 180 degrees, walk forward, and turn right 90 degrees", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Counterfactual", "Temporal Reasoning" ], "objects_of_interest": [ "packet of ghee-46e8386403e45d8f", "spoon-e11f874d227b40fe", "spoon2-40ca91a14a0c120a", "spoon3-29ee510753a90ec2", "bag of granola-fa4f260f54bc9960", "turmeric powder pack-4e137df5263ff0c9", "pack of mustard seeds-fd6caf8da234e179", "pan with bag of chips-658f0fed155dead9", "bag of chips-c5a78bf9e01faaf6", "weighing scale-96274e148c532468", "box of dosa mix-aafcde40caba9409", "bag of dal-d063664686e82143", "notebook-d4898fa4e2d94567", "pen-5705112568133bc9", "green chilli 1-f774f141714cc834", "green chilli 2-28ed5a9e602b5bab", "stack of coriander leaves-4680c3e8a961d61d", "plastic container box-8f9267227c505ddb", "few twigs of coriander leaves-49ac68f768efc20a", "large brown onion-5a8d4da0478588a4", "large brown onion2-b58f4229a688338a", "knife-791fde7b1d890af0", "pair of scissors-ecf82534fc547bef", "potato-4cd2a4e4fb71c2d7", "pot-03341a974f69ea33", "plate-afe429b441ecbc1a", "cap of pen-bbc3d2026cc5055e", "blue coloured plastic chopping board-50d346f59d847512", "white coloured plastic chopping board-772b76f7fce60bf3", "potato half-e3e326773e5c2013", "lid of pressure cooker-e663da0dd466b2de", "second half of potato-a7811a2741400b5e", "pressure cooker regulator-397a057151fe2b7e", "metallic cup-60e8c95c9db50482", "electric hob-f53f289a5d1384b0", "pan-0dda3ddd9d1cf9a1", "rubbish bin-3072870f57109f60", "second half of onion-14e8a3d7c61d5416", "piece of ginger-da6d99888c17dc27", "peeled ginger skin-1a58d3f854c1dc40", "lid of pan-c43e4a69de280f65", "bottle of olive oil-4bf1a7995c33f06c", "cap of olive oil bottle-ad4e691e03adadc5" ], "reasoning": "1. I saw the user first reach down to grab an onion (large brown onion2-b58f4229a688338a) from a shelf at 04:06. I noted that they grabbed it with their left hand while facing 90 degrees to its right.\n2. I went back in the video to find a trashcan in the kitchen. At 02:03, I could see a clear view of a trashcan to the left of a table that was immediately to the left of where the onion (large brown onion2-b58f4229a688338a) was stored.\n3. Therefore, in order to throw the onion (large brown onion2-b58f4229a688338a) away immediately after picking it up, the user would have to turn around 180 degrees, walk forward, and turn right 90 degrees to face the trashcan .", "referenced_object_masks": [ "large brown onion2-b58f4229a688338a" ] }, { "key": "P04-20240414-162750:9bc8600f72ad93e6febe02e7a813aa149518a945", "video_id": "P04-20240414-162750", "question": "If I added the number of rubber bands I took completely off the corriander leaves to the number of metal spoons I retrieved from the drawer, what would the sum be?", "answer_choice_0": "6", "answer_choice_1": "8", "answer_choice_2": "4", "answer_choice_3": "2", "answer_choice_4": "3", "answer_id": 2, "answer": "4", "question_type": [ "Counting", "Action Recognition", "Numerical Reasoning", "Object Recognition" ], "objects_of_interest": [ "packet of ghee-46e8386403e45d8f", "spoon-e11f874d227b40fe", "spoon2-40ca91a14a0c120a", "spoon3-29ee510753a90ec2", "bag of granola-fa4f260f54bc9960", "turmeric powder pack-4e137df5263ff0c9", "pack of mustard seeds-fd6caf8da234e179", "pan with bag of chips-658f0fed155dead9", "bag of chips-c5a78bf9e01faaf6", "weighing scale-96274e148c532468", "box of dosa mix-aafcde40caba9409", "bag of dal-d063664686e82143", "notebook-d4898fa4e2d94567", "pen-5705112568133bc9", "green chilli 1-f774f141714cc834", "green chilli 2-28ed5a9e602b5bab", "stack of coriander leaves-4680c3e8a961d61d", "plastic container box-8f9267227c505ddb", "few twigs of coriander leaves-49ac68f768efc20a", "large brown onion-5a8d4da0478588a4", "large brown onion2-b58f4229a688338a", "knife-791fde7b1d890af0", "pair of scissors-ecf82534fc547bef", "potato-4cd2a4e4fb71c2d7", "pot-03341a974f69ea33", "plate-afe429b441ecbc1a", "cap of pen-bbc3d2026cc5055e", "blue coloured plastic chopping board-50d346f59d847512", "white coloured plastic chopping board-772b76f7fce60bf3", "potato half-e3e326773e5c2013", "lid of pressure cooker-e663da0dd466b2de", "second half of potato-a7811a2741400b5e", "pressure cooker regulator-397a057151fe2b7e", "metallic cup-60e8c95c9db50482", "electric hob-f53f289a5d1384b0", "pan-0dda3ddd9d1cf9a1", "rubbish bin-3072870f57109f60", "second half of onion-14e8a3d7c61d5416", "piece of ginger-da6d99888c17dc27", "peeled ginger skin-1a58d3f854c1dc40", "lid of pan-c43e4a69de280f65", "bottle of olive oil-4bf1a7995c33f06c", "cap of olive oil bottle-ad4e691e03adadc5" ], "reasoning": "1. First, I looked for the user to retrieve metal spoons from a drawer and saw them grab a spoon (spoon-e11f874d227b40fe) at 00:19, then another at 00:20 (spoon2-40ca91a14a0c120a), and a final one at 00:24 (spoon3-29ee510753a90ec2), making the total number of metal spoons the user retrieves from the drawer 3. \n2. Next, I looked for the user to take a rubberband off the coriander leaves (stack of coriander leaves-4680c3e8a961d61d). I saw the user take a rubberband off the coriander leaves (stack of coriander leaves-4680c3e8a961d61d) at 3:00. The user then removes some twigs (few twigs of coriander leaves-49ac68f768efc20a) from the coriander leaves (stack of coriander leaves-4680c3e8a961d61d) and places them on the counter at 03:08. The user fiddles with a second rubberband at 03:04 and 03:08-03:14 but never takes the rubberband fully off the corriander leaves . I concluded that the user only removes one rubberband from the coriander leaves (stack of coriander leaves-4680c3e8a961d61d). \n3. Finally, I added 3+1 and got a sum of 4.", "referenced_object_masks": [ "stack of coriander leaves-4680c3e8a961d61d", "few twigs of coriander leaves-49ac68f768efc20a", "spoon-e11f874d227b40fe", "spoon2-40ca91a14a0c120a", "spoon3-29ee510753a90ec2" ] }, { "key": "P04-20240414-162750:cd3057067507491cf59440faa043747768b565c6", "video_id": "P04-20240414-162750", "question": "When I open the drawer with the metal spoons, where is the wooden cutting board that I eventually use located in relation to me?", "answer_choice_0": "In front", "answer_choice_1": "To the left", "answer_choice_2": "To the right", "answer_choice_3": "Behind", "answer_choice_4": "Below", "answer_id": 2, "answer": "To the right", "question_type": [ "Object Recognition", "Egocentric & Spatial Perception", "Situational Awareness", "Action Recognition" ], "objects_of_interest": [ "packet of ghee-46e8386403e45d8f", "spoon-e11f874d227b40fe", "spoon2-40ca91a14a0c120a", "spoon3-29ee510753a90ec2", "bag of granola-fa4f260f54bc9960", "turmeric powder pack-4e137df5263ff0c9", "pack of mustard seeds-fd6caf8da234e179", "pan with bag of chips-658f0fed155dead9", "bag of chips-c5a78bf9e01faaf6", "weighing scale-96274e148c532468", "box of dosa mix-aafcde40caba9409", "bag of dal-d063664686e82143", "notebook-d4898fa4e2d94567", "pen-5705112568133bc9", "green chilli 1-f774f141714cc834", "green chilli 2-28ed5a9e602b5bab", "stack of coriander leaves-4680c3e8a961d61d", "plastic container box-8f9267227c505ddb", "few twigs of coriander leaves-49ac68f768efc20a", "large brown onion-5a8d4da0478588a4", "large brown onion2-b58f4229a688338a", "knife-791fde7b1d890af0", "pair of scissors-ecf82534fc547bef", "potato-4cd2a4e4fb71c2d7", "pot-03341a974f69ea33", "plate-afe429b441ecbc1a", "cap of pen-bbc3d2026cc5055e", "blue coloured plastic chopping board-50d346f59d847512", "white coloured plastic chopping board-772b76f7fce60bf3", "potato half-e3e326773e5c2013", "lid of pressure cooker-e663da0dd466b2de", "second half of potato-a7811a2741400b5e", "pressure cooker regulator-397a057151fe2b7e", "metallic cup-60e8c95c9db50482", "electric hob-f53f289a5d1384b0", "pan-0dda3ddd9d1cf9a1", "rubbish bin-3072870f57109f60", "second half of onion-14e8a3d7c61d5416", "piece of ginger-da6d99888c17dc27", "peeled ginger skin-1a58d3f854c1dc40", "lid of pan-c43e4a69de280f65", "bottle of olive oil-4bf1a7995c33f06c", "cap of olive oil bottle-ad4e691e03adadc5" ], "reasoning": "1. First, I looked for a wooden cutting board and saw a clear visual of two of them on the counter when the user entered the kitchen at 00:00. One wooden cutting board was located on the left side of the counter, propped up against the wall, and a second wooden cutting board was located to the right, with a potato (potato-4cd2a4e4fb71c2d7) sitting on top of it.\n2. Next, I looked for the user to open a drawer with metal spoons and saw this occur at 00:18. I noticed that the first wooden cutting board was located directly above the drawer , and the second wooden cutting board was located to the right, using the visual information from 00:00. I continued watching to see which wooden cutting board got used.\n3. At 06:47, the user placed another cutting board (blue coloured plastic chopping board-50d346f59d847512) on top of the second wooden cutting board . Then, at 06:59, the user began to cut the potato (potato-4cd2a4e4fb71c2d7) on top of the blue cutting board (blue coloured plastic chopping board-50d346f59d847512).\n4. Reasoning about the scene, I determined that the appropriate wooden cutting board is to the right of the user when they open the drawer with the metal spoons .", "referenced_object_masks": [ "blue coloured plastic chopping board-50d346f59d847512", "potato-4cd2a4e4fb71c2d7" ] }, { "key": "P04-20240414-162750:d0a29c4951ae23ca91be7cf5c3ce889b0b321cc7", "video_id": "P04-20240414-162750", "question": "How many times do I press the buttons of the stove top in the first 30 seconds after plugging it in?", "answer_choice_0": "6", "answer_choice_1": "2", "answer_choice_2": "4", "answer_choice_3": "10", "answer_choice_4": "8", "answer_id": 0, "answer": "6", "question_type": [ "Counting", "Action Recognition", "Object Recognition", "Egocentric & Spatial Perception", "Temporal Reasoning" ], "objects_of_interest": [ "packet of ghee-46e8386403e45d8f", "spoon-e11f874d227b40fe", "spoon2-40ca91a14a0c120a", "spoon3-29ee510753a90ec2", "bag of granola-fa4f260f54bc9960", "turmeric powder pack-4e137df5263ff0c9", "pack of mustard seeds-fd6caf8da234e179", "pan with bag of chips-658f0fed155dead9", "bag of chips-c5a78bf9e01faaf6", "weighing scale-96274e148c532468", "box of dosa mix-aafcde40caba9409", "bag of dal-d063664686e82143", "notebook-d4898fa4e2d94567", "pen-5705112568133bc9", "green chilli 1-f774f141714cc834", "green chilli 2-28ed5a9e602b5bab", "stack of coriander leaves-4680c3e8a961d61d", "plastic container box-8f9267227c505ddb", "few twigs of coriander leaves-49ac68f768efc20a", "large brown onion-5a8d4da0478588a4", "large brown onion2-b58f4229a688338a", "knife-791fde7b1d890af0", "pair of scissors-ecf82534fc547bef", "potato-4cd2a4e4fb71c2d7", "pot-03341a974f69ea33", "plate-afe429b441ecbc1a", "cap of pen-bbc3d2026cc5055e", "blue coloured plastic chopping board-50d346f59d847512", "white coloured plastic chopping board-772b76f7fce60bf3", "potato half-e3e326773e5c2013", "lid of pressure cooker-e663da0dd466b2de", "second half of potato-a7811a2741400b5e", "pressure cooker regulator-397a057151fe2b7e", "metallic cup-60e8c95c9db50482", "electric hob-f53f289a5d1384b0", "pan-0dda3ddd9d1cf9a1", "rubbish bin-3072870f57109f60", "second half of onion-14e8a3d7c61d5416", "piece of ginger-da6d99888c17dc27", "peeled ginger skin-1a58d3f854c1dc40", "lid of pan-c43e4a69de280f65", "bottle of olive oil-4bf1a7995c33f06c", "cap of olive oil bottle-ad4e691e03adadc5" ], "reasoning": "1. First, I looked for the user to plug in the stovetop (electric hob-f53f289a5d1384b0) and found that they do this at 08:12. \n2. Next, I added 30 seconds to 08:12 and got a final timecode of 08:42. \n3. Then, I watched the video between 08:12 to 08:42, looking for moments when the user touches the buttons on the stovetop (electric hob-f53f289a5d1384b0). I saw the user press the \"on\" button once at 08:29, then saw them press buttons to the left of the \"on\" button four times between 08:30-08:32. The user then presses the \"on' button for a final time at 08:32. I added up these instances and concluded that the user touches buttons on the stovetop (electric hob-f53f289a5d1384b0) a total of 6 times after plugging it in.", "referenced_object_masks": [ "electric hob-f53f289a5d1384b0" ] }, { "key": "P04-20240414-162750:ecf83c2b490c3024a28f7e498b1326ff18bf14a3", "video_id": "P04-20240414-162750", "question": "After placing the pot on the leftmost burner, in which two directions do I move it before turning on the stovetop?", "answer_choice_0": "Right and 45 degrees in a clockwise direction", "answer_choice_1": "Left and 180 degrees in a counter-clockwise direction", "answer_choice_2": "Left and 45 degrees in a clockwise direction", "answer_choice_3": "Right and 180 degrees in a counter-clockwise direction", "answer_choice_4": "Left and 45 degrees in a counter-clockwise direction", "answer_id": 0, "answer": "Right and 45 degrees in a clockwise direction", "question_type": [ "Egocentric & Spatial Perception", "Action Recognition", "Temporal Reasoning", "Object Recognition" ], "objects_of_interest": [ "packet of ghee-46e8386403e45d8f", "spoon-e11f874d227b40fe", "spoon2-40ca91a14a0c120a", "spoon3-29ee510753a90ec2", "bag of granola-fa4f260f54bc9960", "turmeric powder pack-4e137df5263ff0c9", "pack of mustard seeds-fd6caf8da234e179", "pan with bag of chips-658f0fed155dead9", "bag of chips-c5a78bf9e01faaf6", "weighing scale-96274e148c532468", "box of dosa mix-aafcde40caba9409", "bag of dal-d063664686e82143", "notebook-d4898fa4e2d94567", "pen-5705112568133bc9", "green chilli 1-f774f141714cc834", "green chilli 2-28ed5a9e602b5bab", "stack of coriander leaves-4680c3e8a961d61d", "plastic container box-8f9267227c505ddb", "few twigs of coriander leaves-49ac68f768efc20a", "large brown onion-5a8d4da0478588a4", "large brown onion2-b58f4229a688338a", "knife-791fde7b1d890af0", "pair of scissors-ecf82534fc547bef", "potato-4cd2a4e4fb71c2d7", "pot-03341a974f69ea33", "plate-afe429b441ecbc1a", "cap of pen-bbc3d2026cc5055e", "blue coloured plastic chopping board-50d346f59d847512", "white coloured plastic chopping board-772b76f7fce60bf3", "potato half-e3e326773e5c2013", "lid of pressure cooker-e663da0dd466b2de", "second half of potato-a7811a2741400b5e", "pressure cooker regulator-397a057151fe2b7e", "metallic cup-60e8c95c9db50482", "electric hob-f53f289a5d1384b0", "pan-0dda3ddd9d1cf9a1", "rubbish bin-3072870f57109f60", "second half of onion-14e8a3d7c61d5416", "piece of ginger-da6d99888c17dc27", "peeled ginger skin-1a58d3f854c1dc40", "lid of pan-c43e4a69de280f65", "bottle of olive oil-4bf1a7995c33f06c", "cap of olive oil bottle-ad4e691e03adadc5" ], "reasoning": "1. First, I looked for the user to place the pot on the leftmost burner of the stovetop (electric hob-f53f289a5d1384b0) and saw them do this at 08:15. \n2. Next, I looked for the user to move the pot in an initial direction, and saw them move the pot to the right at 08:21.\n3. Then, I looked for the user to move the pot in a second direction and saw them turn the pot 45 degrees in a clockwise direction at 08:24. \n4. Finally, I saw the user turn on the stovetop (electric hob-f53f289a5d1384b0) at 08:30, and concluded that right and 45 degrees in a clockwise direction were the two ways in which the user moved the pot during that timespan.", "referenced_object_masks": [ "electric hob-f53f289a5d1384b0" ] }, { "key": "P04-20240414-164751:20c5630befe3c5e53f84217fe779048bab0a43ff", "video_id": "P04-20240414-164751", "question": "Immediately after placing the pan on the hot plate, in which direction would the pressure cooker's handle point if I rotated the hot plate clockwise 90 degrees?", "answer_choice_0": "Southeast", "answer_choice_1": "East", "answer_choice_2": "North", "answer_choice_3": "Southwest", "answer_choice_4": "Northeast", "answer_id": 3, "answer": "Southwest", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counterfactual" ], "objects_of_interest": [ "pan-a14a845fb725257f", "electric hob-1549b4e938e5528e", "pressure cooker-8b64c534d6c1a78a", "small package of black mustard seeds-125f3b222ea67f54", "seal-acfcb6efce62312f", "pack of mustard-1905ce839e8301b0", "seal2-6c6e4c73a5a235db", "knife-c0edc37312552f79", "box of dosa mix-2e1ac1e6b9438a40", "wooden spatula-5173a77ba61cfd84", "twigs of coriander leaves-596ad9530c9ddb39", "jar-b7871d48d38013c3", "plastic bag-ae052aee268eee89", "spoon-7dc447594f04b25c", "pen-cfcb39f85fc7589f", "green chillies-06aed2b9f63dcaec" ], "reasoning": "1. I locate the hot plate (electric hob-1549b4e938e5528e) at 00:00 to the left of the sink .\n2. I observe the user place the pan (pan-a14a845fb725257f) on the hot plate (electric hob-1549b4e938e5528e) to the left of the pressure cooker (pressure cooker-8b64c534d6c1a78a) at 00:07.\n3. At 00:07, the handle of the pressure cooker (electric hob-1549b4e938e5528e) points southeast.\n4. I determine that rotating the hot plate (electric hob-1549b4e938e5528e) 90 degrees would leave the handle of the pressure cooker (pressure cooker-8b64c534d6c1a78a) pointing southwest.", "referenced_object_masks": [ "electric hob-1549b4e938e5528e", "pressure cooker-8b64c534d6c1a78a", "pan-a14a845fb725257f" ] }, { "key": "P04-20240414-164751:26739033bd2ad2ba2777ca61880692f879a9c390", "video_id": "P04-20240414-164751", "question": "After the pressure cooker stops emitting steam, what is the first object I touch with my left hand after letting go of the wooden spatula with my right?", "answer_choice_0": "Green chillies", "answer_choice_1": "Wooden spatula", "answer_choice_2": "Spoon", "answer_choice_3": "Blue cutting board", "answer_choice_4": "Knife", "answer_id": 0, "answer": "Green chillies", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "pan-a14a845fb725257f", "electric hob-1549b4e938e5528e", "pressure cooker-8b64c534d6c1a78a", "small package of black mustard seeds-125f3b222ea67f54", "seal-acfcb6efce62312f", "pack of mustard-1905ce839e8301b0", "seal2-6c6e4c73a5a235db", "knife-c0edc37312552f79", "box of dosa mix-2e1ac1e6b9438a40", "wooden spatula-5173a77ba61cfd84", "twigs of coriander leaves-596ad9530c9ddb39", "jar-b7871d48d38013c3", "plastic bag-ae052aee268eee89", "spoon-7dc447594f04b25c", "pen-cfcb39f85fc7589f", "green chillies-06aed2b9f63dcaec" ], "reasoning": "1. I locate the pressure cooker (pressure cooker-8b64c534d6c1a78a) atop the hot plate (electric hob-1549b4e938e5528e) in front of the user at 00:13.\n2. I observe the pressure cooker (pressure cooker-8b64c534d6c1a78a) steaming from 04:39 to 04:42.\n3. At 05:00, I observe the user letting go of the wooden spatula (wooden spatula-5173a77ba61cfd84) and resting it on the pan (pan-a14a845fb725257f) to their right. \n4. At 05:01, I identify the user touching the green chillies (green chillies-06aed2b9f63dcaec) on the wooden cutting board with their left hand.", "referenced_object_masks": [ "pan-a14a845fb725257f", "pressure cooker-8b64c534d6c1a78a", "electric hob-1549b4e938e5528e", "wooden spatula-5173a77ba61cfd84", "green chillies-06aed2b9f63dcaec" ] }, { "key": "P04-20240414-164751:3c330727b305507a1ed3da7ece6d3e3cb5753bf0", "video_id": "P04-20240414-164751", "question": "Just before picking up a spoon for the first time, how many more spoons are there on the wooden cutting board than there are electrical panels above the plastic cutting board?", "answer_choice_0": "3", "answer_choice_1": "4", "answer_choice_2": "1", "answer_choice_3": "2", "answer_choice_4": "5", "answer_id": 2, "answer": "1", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning", "Numerical Reasoning", "Counting" ], "objects_of_interest": [ "pan-a14a845fb725257f", "electric hob-1549b4e938e5528e", "pressure cooker-8b64c534d6c1a78a", "small package of black mustard seeds-125f3b222ea67f54", "seal-acfcb6efce62312f", "pack of mustard-1905ce839e8301b0", "seal2-6c6e4c73a5a235db", "knife-c0edc37312552f79", "box of dosa mix-2e1ac1e6b9438a40", "wooden spatula-5173a77ba61cfd84", "twigs of coriander leaves-596ad9530c9ddb39", "jar-b7871d48d38013c3", "plastic bag-ae052aee268eee89", "spoon-7dc447594f04b25c", "pen-cfcb39f85fc7589f", "green chillies-06aed2b9f63dcaec" ], "reasoning": "1. I observe the user pick up a spoon (spoon-7dc447594f04b25c) from the wooden cutting board for the first time at 02:46.\n2. I rewind to 02:45 and count 3 spoons (OBJ, spoon-7dc447594f04b25c) on the wooden cutting board and 2 electrical panels above the plastic cutting board .\n3. I subtract 2 from 3 to get 1.", "referenced_object_masks": [ "spoon-7dc447594f04b25c" ] }, { "key": "P04-20240414-164751:9a8b8df990e45139840576b5c8939468df43af27", "video_id": "P04-20240414-164751", "question": "What is the first cooking utensil I hold in my right hand after writing in my notebook?", "answer_choice_0": "Cutting board", "answer_choice_1": "Knife", "answer_choice_2": "Spoon", "answer_choice_3": "Measuring cup", "answer_choice_4": "Wooden spatula", "answer_id": 4, "answer": "Wooden spatula", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "pan-a14a845fb725257f", "electric hob-1549b4e938e5528e", "pressure cooker-8b64c534d6c1a78a", "small package of black mustard seeds-125f3b222ea67f54", "seal-acfcb6efce62312f", "pack of mustard-1905ce839e8301b0", "seal2-6c6e4c73a5a235db", "knife-c0edc37312552f79", "box of dosa mix-2e1ac1e6b9438a40", "wooden spatula-5173a77ba61cfd84", "twigs of coriander leaves-596ad9530c9ddb39", "jar-b7871d48d38013c3", "plastic bag-ae052aee268eee89", "spoon-7dc447594f04b25c", "pen-cfcb39f85fc7589f", "green chillies-06aed2b9f63dcaec" ], "reasoning": "1. I locate the notebook and pen (pen-cfcb39f85fc7589f) at 00:00 on the counter in front and to the left of the user. \n2. From 03:55 to 04:13, I observe the user writing with the pen (pen-cfcb39f85fc7589f) in the notebook .\n3. At 04:16, I observe the user pick up the wooden spatula (wooden spatula-5173a77ba61cfd84) from the wooden cutting board with their left hand.\n4. At 04:17, I observe the user transfer the wooden spatula (wooden spatula-5173a77ba61cfd84) from their left hand to their right hand.", "referenced_object_masks": [ "pen-cfcb39f85fc7589f", "wooden spatula-5173a77ba61cfd84" ] }, { "key": "P04-20240414-164751:dd988b7bd45d6761f249f19447dbc2cf95e8c968", "video_id": "P04-20240414-164751", "question": "What is the difference between the first and last numbered temperature readings on the hot plate?", "answer_choice_0": "200", "answer_choice_1": "500", "answer_choice_2": "300", "answer_choice_3": "400", "answer_choice_4": "600", "answer_id": 3, "answer": "400", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning", "Numerical Reasoning", "Reading" ], "objects_of_interest": [ "pan-a14a845fb725257f", "electric hob-1549b4e938e5528e", "pressure cooker-8b64c534d6c1a78a", "small package of black mustard seeds-125f3b222ea67f54", "seal-acfcb6efce62312f", "pack of mustard-1905ce839e8301b0", "seal2-6c6e4c73a5a235db", "knife-c0edc37312552f79", "box of dosa mix-2e1ac1e6b9438a40", "wooden spatula-5173a77ba61cfd84", "twigs of coriander leaves-596ad9530c9ddb39", "jar-b7871d48d38013c3", "plastic bag-ae052aee268eee89", "spoon-7dc447594f04b25c", "pen-cfcb39f85fc7589f", "green chillies-06aed2b9f63dcaec" ], "reasoning": "1. I locate the hot plate (electric hob-1549b4e938e5528e) at 00:00 to the left of the sink .\n2. I determine the first numbered temperature reading seen on the hot plate (electric hob-1549b4e938e5528e) is 600, at 00:08. \n3. I observe the user turn the hot plate (electric hob-1549b4e938e5528e) off at 05:15.\n4. I turn to just before the hot plate (electric hob-1549b4e938e5528e) turns off, at 05:14, to determine the last numbered temperature reading of the hot plate (electric hob-1549b4e938e5528e). I discover that it is 200.\n5. I subtract 200 from 600 to get 400.", "referenced_object_masks": [ "electric hob-1549b4e938e5528e" ] }, { "key": "P04-20240414-165333:264a0c34889cb6c8e2146dcde0db44219ab1a7a8", "video_id": "P04-20240414-165333", "question": "In the 10 seconds leading up to the moment I scooped some salt, how many times did I hesitate to grab something?", "answer_choice_0": "Twice", "answer_choice_1": "Thrice", "answer_choice_2": "5 times", "answer_choice_3": "4 times", "answer_choice_4": "Once", "answer_id": 3, "answer": "4 times", "question_type": [ "Egocentric & Spatial Perception", "Situational Awareness", "Counting", "Action Recognition" ], "objects_of_interest": [ "spatula-698fd6c88603d750", "chili-383087b29e839a16", "knife-353c0ce699aa17aa", "pressure cooker-0ef98aa54c776891", "tablespoon-6fe1e0520c4bb8a6", "open notebook-10fa332ca686f6d9", "cooling rack-c767cfd63d13daa5", "onion piece-7cb2e40a420c0c25", "onione piece 2-695fc1f216eeae30", "bottle of oil-994ac467e1a4013e", "cap-ee313d56db8cdd01", "tablespoon2-676223f5e13ff2a2", "jar of salt-1bf798cf7a93d107", "packet of turmeric powder-a9451025ec911959", "large white ceramic bowl-82395476e05be14d", "box of instant dosa mixture-0c697f34d1fb83cc" ], "reasoning": "1. I watched the video to see when the user grabs the jar of salt (jar of salt-1bf798cf7a93d107) and scoops some salt out of it and found that it happens at 04:38.\n2. I went to 10 seconds before this at 04:28 and saw that the user hesitated to grab the knife (knife-353c0ce699aa17aa) at 04:29, the chili (chili-383087b29e839a16) at 04:30, the bottle of oil (bottle of oil-994ac467e1a4013e) directly afterwards (also at 04:30), and finally the turmeric (packet of turmeric powder-a9451025ec911959) at 04:31.\n3. So, before finally grabbing what the user was actually looking for, the jar of salt (jar of salt-1bf798cf7a93d107), they hesitated and almost grabbed other items 4 seperate times.", "referenced_object_masks": [ "jar of salt-1bf798cf7a93d107", "bottle of oil-994ac467e1a4013e", "chili-383087b29e839a16", "knife-353c0ce699aa17aa", "packet of turmeric powder-a9451025ec911959" ] }, { "key": "P04-20240414-165333:83dc1c19d501bdb8a7d0bd69f507f3f494092e72", "video_id": "P04-20240414-165333", "question": "What percentage of the bottom shelf dishes in the cabinet above the portable stove are bowls?", "answer_choice_0": "72.73%", "answer_choice_1": "80.1%", "answer_choice_2": "19.2%", "answer_choice_3": "27.27%", "answer_choice_4": "75.74%", "answer_id": 3, "answer": "27.27%", "question_type": [ "Object Recognition", "Numerical Reasoning", "Counterfactual" ], "objects_of_interest": [ "spatula-698fd6c88603d750", "chili-383087b29e839a16", "knife-353c0ce699aa17aa", "pressure cooker-0ef98aa54c776891", "tablespoon-6fe1e0520c4bb8a6", "open notebook-10fa332ca686f6d9", "cooling rack-c767cfd63d13daa5", "onion piece-7cb2e40a420c0c25", "onione piece 2-695fc1f216eeae30", "bottle of oil-994ac467e1a4013e", "cap-ee313d56db8cdd01", "tablespoon2-676223f5e13ff2a2", "jar of salt-1bf798cf7a93d107", "packet of turmeric powder-a9451025ec911959", "large white ceramic bowl-82395476e05be14d", "box of instant dosa mixture-0c697f34d1fb83cc" ], "reasoning": "1. I watched the video to look for a clear visual of the bottom shelf of the cabinet above the portable stove , and found all of the dishes in plain view at 07:30.\n2. Of the dishes, there are 22 total, with 6 of them being bowls and 16 of them being plates .\n3. I then calculated the percentage using 6/22 and got 27.27%.", "referenced_object_masks": [] }, { "key": "P04-20240414-165333:84c7582ea7dcce2cb3c3639c731d900d56ed194b", "video_id": "P04-20240414-165333", "question": "What was the purpose of putting water in the big white bowl before I used it?", "answer_choice_0": "To clean the bowl before storing it", "answer_choice_1": "To make sure the bowl is wet", "answer_choice_2": "It's a step in the recipe", "answer_choice_3": "To simply rinse it out", "answer_choice_4": "To check the bowl for leaks", "answer_id": 3, "answer": "To simply rinse it out", "question_type": [ "Action Recognition", "Object Recognition" ], "objects_of_interest": [ "spatula-698fd6c88603d750", "chili-383087b29e839a16", "knife-353c0ce699aa17aa", "pressure cooker-0ef98aa54c776891", "tablespoon-6fe1e0520c4bb8a6", "open notebook-10fa332ca686f6d9", "cooling rack-c767cfd63d13daa5", "onion piece-7cb2e40a420c0c25", "onione piece 2-695fc1f216eeae30", "bottle of oil-994ac467e1a4013e", "cap-ee313d56db8cdd01", "tablespoon2-676223f5e13ff2a2", "jar of salt-1bf798cf7a93d107", "packet of turmeric powder-a9451025ec911959", "large white ceramic bowl-82395476e05be14d", "box of instant dosa mixture-0c697f34d1fb83cc" ], "reasoning": "1. I watched the video to find the big white bowl (large white ceramic bowl-82395476e05be14d) and saw the user grab it from the cabinet above the sink at 07:39.\n2. At 07:47, I saw the user grab a small object from the bowl (large white ceramic bowl-82395476e05be14d) and throw it in the garbage can at 07:49.\n3. Then, I saw the user run a small amount of water in it at 07:53 and promptly pour it out at 07:57.\n4. Therefore, it appears the user found a small object in the bowl (large white ceramic bowl-82395476e05be14d), removed it, then rinsed out the bowl (large white ceramic bowl-82395476e05be14d).", "referenced_object_masks": [ "large white ceramic bowl-82395476e05be14d" ] }, { "key": "P04-20240414-165333:9669aa882cd1d529923cca0f2c5cbe84cb57dd87", "video_id": "P04-20240414-165333", "question": "What is the difference between the number of times I press a button on the hot plate and the number of unique ingredients I add to the pan?", "answer_choice_0": "1", "answer_choice_1": "3", "answer_choice_2": "5", "answer_choice_3": "7", "answer_choice_4": "8", "answer_id": 4, "answer": "8", "question_type": [ "Object Recognition", "Numerical Reasoning", "Counterfactual" ], "objects_of_interest": [ "spatula-698fd6c88603d750", "chili-383087b29e839a16", "knife-353c0ce699aa17aa", "pressure cooker-0ef98aa54c776891", "tablespoon-6fe1e0520c4bb8a6", "open notebook-10fa332ca686f6d9", "cooling rack-c767cfd63d13daa5", "onion piece-7cb2e40a420c0c25", "onione piece 2-695fc1f216eeae30", "bottle of oil-994ac467e1a4013e", "cap-ee313d56db8cdd01", "tablespoon2-676223f5e13ff2a2", "jar of salt-1bf798cf7a93d107", "packet of turmeric powder-a9451025ec911959", "large white ceramic bowl-82395476e05be14d", "box of instant dosa mixture-0c697f34d1fb83cc" ], "reasoning": "1. I determine at 00:03 that the hot plate registers successful button input with a beep sound.\n2. I observe the user press buttons on the hot plate 8 times in quick succession at the beginning of the video: 1 time at 00:03, 3 times at 00:05, 2 times at 00:06, and 2 times at 00:07.\n3. I observe the user press buttons on the hot plate 1 time at 00:42, 01:39, 02:33, 07:11, and 09:16, and 2 times between 08:53-08:54.\n4. I determine the user presses a button on the hot plate 15 times.\n5. I observe the user add the following ingredients to the pan : a chopped up white ingredient from 00:14-00:16, jalapenos (chili-383087b29e839a16) from 00:26-00:29, chopped bits of green garnish from 01:08-01:09, and diced onion halves (onion piece-7cb2e40a420c0c25, onione piece 2-695fc1f216eeae30) from 02:37-02:46.\n6. I observe the user add the following ingredients from the spoon (tablespoon2-676223f5e13ff2a2) to the pan : oil from 03:37-03:45, salt from both 04:42-04:51 and 04:58-05:02, and turmeric from 05:54-06:16.\n7. I determine the user adds 7 unique ingredients .\n8. I subtract 7 from 15 to get 8.", "referenced_object_masks": [ "onion piece-7cb2e40a420c0c25", "onione piece 2-695fc1f216eeae30", "tablespoon2-676223f5e13ff2a2", "chili-383087b29e839a16" ] }, { "key": "P04-20240414-165333:ee31573e74a5ed37c28900e1822c1b60384f4bf0", "video_id": "P04-20240414-165333", "question": "By what number did I increase and decrease the temperature of the burner on the left after setting a temperature of 400?", "answer_choice_0": "800", "answer_choice_1": "600", "answer_choice_2": "400", "answer_choice_3": "200", "answer_choice_4": "100", "answer_id": 3, "answer": "200", "question_type": [ "Egocentric & Spatial Perception", "Reading", "Action Recognition", "Event Occurence" ], "objects_of_interest": [ "spatula-698fd6c88603d750", "chili-383087b29e839a16", "knife-353c0ce699aa17aa", "pressure cooker-0ef98aa54c776891", "tablespoon-6fe1e0520c4bb8a6", "open notebook-10fa332ca686f6d9", "cooling rack-c767cfd63d13daa5", "onion piece-7cb2e40a420c0c25", "onione piece 2-695fc1f216eeae30", "bottle of oil-994ac467e1a4013e", "cap-ee313d56db8cdd01", "tablespoon2-676223f5e13ff2a2", "jar of salt-1bf798cf7a93d107", "packet of turmeric powder-a9451025ec911959", "large white ceramic bowl-82395476e05be14d", "box of instant dosa mixture-0c697f34d1fb83cc" ], "reasoning": "1. First, I watched the video and looked for the moment when the user sets the temperature of the leftmost burner to 400. I found this occurs at 00:42. \n2. Next, I watched the video to see when the user increases and decreases the temperature of the burner on the left and saw that they did it twice, first at 02:33 and second at 07:12.\n3. I observed that at 02:33, the number was increased from 400 to 600, then decreased back to 400 at 07:12, so they were increased and decreased by the same amount.\n4. Therefore, the temperature of the burner on the left increased and decreased by 200 after the user sets the temperature to 400.", "referenced_object_masks": [] }, { "key": "P04-20240414-170520:2eafdf175d614ef368f2fda8a06b09310920b145", "video_id": "P04-20240414-170520", "question": "What is the third item I touch with my left hand after wiping off the scale, not counting the item I am already touching?", "answer_choice_0": "Cookbook", "answer_choice_1": "White plate", "answer_choice_2": "Bag clips", "answer_choice_3": "Cardboard box", "answer_choice_4": "Purple rag", "answer_id": 3, "answer": "Cardboard box", "question_type": [ "Egocentric & Spatial Perception", "Temporal Reasoning", "Object Recognition", "Event Occurence" ], "objects_of_interest": [ "large mixing bowl-7924ca9e89d7c5cc", "plate-9fd8717e4164f5a1", "scale-d9e338a21afb21ea", "purple coloured kitchen rag-3462cec235acdc1a", "plastic sealing clip-5069b3b7b9e3c220", "box of dosa mix-677575d3f430b108", "packet of instant dosa mix powder-51cf98845b26ced7", "larger plate-7114bff13a2bd580", "open notebook-b986d389c6393911", "lid of pressure cooker-6038467a2d16baf7", "regulator of pressure cooker-b7e2a615129b5b22", "pressure cooker-482717f400d9d69c", "tongs-b6603878904b56c7", "baked potato-88a82400e5588a06", "baked potato2-19a1a155f68dd373", "pen-eccaf363c0489fef", "small plastic measuring cup-d8eea38abc5b9452", "measuring cup-2d0070a85bfd307a", "measuring jug2-5efc4d96e772da13", "strainer-91b22f0ebc4c1b61", "whisk-1b361f7ba0ebabf5", "blue coloured plastic chopping board-34095ffab0b61897", "tablespoon-7d53ef999736940e", "tablespoon2-d81e13831e42281b", "piece of ginger-ef2f94a49487df11", "packet-fc472e10061aab1f", "potato masher-89eb0125b3d23bcb", "tablespoon3-4370da841518ed2a", "knife-c78e90fede872ce6", "spatula-0744f41c0ad71906", "jar of salt-c782779f376f93fb", "lid of salt jar-8a9e16e531a38f92", "pan-6435624c87e84bbc", "lid of pan-8ee5f56217da546c" ], "reasoning": "1. I watch the video to identify when the user wipes off the scale (scale-d9e338a21afb21ea), which they do from 00:24-00:26 with the purple rag (purple coloured kitchen rag-3462cec235acdc1a).\n2. During this time, I notice that the user is touching a white bowl (large mixing bowl-7924ca9e89d7c5cc) with their left hand. I discount this item in the list of items touched, since the user is already holding on to it.\n3. I continued to watch and observed that the user next touches the button on the scale (scale-d9e338a21afb21ea) at 00:30, marking the second item touched with the left hand after wiping the scale (scale-d9e338a21afb21ea).\n4. Then, at 00:34, the user touches the small plate (plate-9fd8717e4164f5a1) with the left hand.\n5. Finally, at 00:35, the user touches a cardboard box (box of dosa mix-677575d3f430b108) with their left hand.\n5. The third item touched with the left hand after wiping the scale (scale-d9e338a21afb21ea), therefore, is the cardboard box (box of dosa mix-677575d3f430b108).", "referenced_object_masks": [ "large mixing bowl-7924ca9e89d7c5cc", "box of dosa mix-677575d3f430b108", "purple coloured kitchen rag-3462cec235acdc1a", "scale-d9e338a21afb21ea", "plate-9fd8717e4164f5a1" ] }, { "key": "P04-20240414-170520:54732428a12a5e4279d8f6314a6e7a0424dea4c9", "video_id": "P04-20240414-170520", "question": "How many times do I check the contents of the pressure cooker before fully removing its lid?", "answer_choice_0": "Twice", "answer_choice_1": "3 times", "answer_choice_2": "0 times", "answer_choice_3": "5 times", "answer_choice_4": "Once", "answer_id": 4, "answer": "Once", "question_type": [ "Action Recognition", "Event Occurence", "Object Recognition", "Temporal Reasoning", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "large mixing bowl-7924ca9e89d7c5cc", "plate-9fd8717e4164f5a1", "scale-d9e338a21afb21ea", "purple coloured kitchen rag-3462cec235acdc1a", "plastic sealing clip-5069b3b7b9e3c220", "box of dosa mix-677575d3f430b108", "packet of instant dosa mix powder-51cf98845b26ced7", "larger plate-7114bff13a2bd580", "open notebook-b986d389c6393911", "lid of pressure cooker-6038467a2d16baf7", "regulator of pressure cooker-b7e2a615129b5b22", "pressure cooker-482717f400d9d69c", "tongs-b6603878904b56c7", "baked potato-88a82400e5588a06", "baked potato2-19a1a155f68dd373", "pen-eccaf363c0489fef", "small plastic measuring cup-d8eea38abc5b9452", "measuring cup-2d0070a85bfd307a", "measuring jug2-5efc4d96e772da13", "strainer-91b22f0ebc4c1b61", "whisk-1b361f7ba0ebabf5", "blue coloured plastic chopping board-34095ffab0b61897", "tablespoon-7d53ef999736940e", "tablespoon2-d81e13831e42281b", "piece of ginger-ef2f94a49487df11", "packet-fc472e10061aab1f", "potato masher-89eb0125b3d23bcb", "tablespoon3-4370da841518ed2a", "knife-c78e90fede872ce6", "spatula-0744f41c0ad71906", "jar of salt-c782779f376f93fb", "lid of salt jar-8a9e16e531a38f92", "pan-6435624c87e84bbc", "lid of pan-8ee5f56217da546c" ], "reasoning": "1. I watched the video and at 00:25, I identified the pressure cooker (pressure cooker-482717f400d9d69cc) on the counter.\n2. Then, I continued to watch and saw the user check the contents of the pressure cooker (pressure cooker-482717f400d9d69cc) by lifting the pressure cooker lid (lid of pressure cooker-6038467a2d16baf7) partially but not fully removing it from 00:46-00:51.\n3. The user continues doing other tasks until 01:11-01:19, then they fully remove the lid (lid of pressure cooker-6038467a2d16baf7) from the pressure cooker bottom (pressure cooker-482717f400d9d69c).\n4. Thus, the user only checks the contents of the pressure cooker (pressure cooker-482717f400d9d69c) once before removing the lid.", "referenced_object_masks": [ "pressure cooker-482717f400d9d69c", "lid of pressure cooker-6038467a2d16baf7" ] }, { "key": "P04-20240414-170520:843daa6975191d559d6aa7d5e4ca95c8249bf670", "video_id": "P04-20240414-170520", "question": "After tasting the food in the skillet, what 2 items would I have not grabbed if I grabbed the item on the black stovetop first?", "answer_choice_0": "The potato masher and skillet", "answer_choice_1": "The notebook and pen", "answer_choice_2": "The scale and towel", "answer_choice_3": "The Dosa mix and bowl", "answer_choice_4": "The chef's knife and white plate", "answer_id": 4, "answer": "The chef's knife and white plate", "question_type": [ "Counterfactual", "Object Recognition", "Egocentric & Spatial Perception", "Situational Awareness" ], "objects_of_interest": [ "large mixing bowl-7924ca9e89d7c5cc", "plate-9fd8717e4164f5a1", "scale-d9e338a21afb21ea", "purple coloured kitchen rag-3462cec235acdc1a", "plastic sealing clip-5069b3b7b9e3c220", "box of dosa mix-677575d3f430b108", "packet of instant dosa mix powder-51cf98845b26ced7", "larger plate-7114bff13a2bd580", "open notebook-b986d389c6393911", "lid of pressure cooker-6038467a2d16baf7", "regulator of pressure cooker-b7e2a615129b5b22", "pressure cooker-482717f400d9d69c", "tongs-b6603878904b56c7", "baked potato-88a82400e5588a06", "baked potato2-19a1a155f68dd373", "pen-eccaf363c0489fef", "small plastic measuring cup-d8eea38abc5b9452", "measuring cup-2d0070a85bfd307a", "measuring jug2-5efc4d96e772da13", "strainer-91b22f0ebc4c1b61", "whisk-1b361f7ba0ebabf5", "blue coloured plastic chopping board-34095ffab0b61897", "tablespoon-7d53ef999736940e", "tablespoon2-d81e13831e42281b", "piece of ginger-ef2f94a49487df11", "packet-fc472e10061aab1f", "potato masher-89eb0125b3d23bcb", "tablespoon3-4370da841518ed2a", "knife-c78e90fede872ce6", "spatula-0744f41c0ad71906", "jar of salt-c782779f376f93fb", "lid of salt jar-8a9e16e531a38f92", "pan-6435624c87e84bbc", "lid of pan-8ee5f56217da546c" ], "reasoning": "1. I watched the video to see that the user tasted the food in the skillet (pan-6435624c87e84bbc) from 09:47-09:54.\n2. I then continued to watch to see the user touch the chef's knife (knife-c78e90fede872ce6) at 09:56.\n3. Then, the user grabs the white plate (larger plate-7114bff13a2bd580) with both hands at 09:57.\n4. The user picks the plate (larger plate-7114bff13a2bd580) up and then set it back down before grabbing a potato masher (potato masher-89eb0125b3d23bcb) off of the black stovetop at 10:00.\n5. If they user would have grabbed the potato masher first, they would not have grabbed the knife and plate.", "referenced_object_masks": [ "larger plate-7114bff13a2bd580", "pan-6435624c87e84bbc", "knife-c78e90fede872ce6", "potato masher-89eb0125b3d23bcb" ] }, { "key": "P04-20240414-170520:86610b5ca52dd0ea72e3cfb384baf2462fabda28", "video_id": "P04-20240414-170520", "question": "While pouring the Dosa mix into the bowl, what is half of the number shown on the scale when I pause for the third time?", "answer_choice_0": "12", "answer_choice_1": "30", "answer_choice_2": "25", "answer_choice_3": "35", "answer_choice_4": "20", "answer_id": 2, "answer": "25", "question_type": [ "Action Recognition", "Reading", "Numerical Reasoning", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "large mixing bowl-7924ca9e89d7c5cc", "plate-9fd8717e4164f5a1", "scale-d9e338a21afb21ea", "purple coloured kitchen rag-3462cec235acdc1a", "plastic sealing clip-5069b3b7b9e3c220", "box of dosa mix-677575d3f430b108", "packet of instant dosa mix powder-51cf98845b26ced7", "larger plate-7114bff13a2bd580", "open notebook-b986d389c6393911", "lid of pressure cooker-6038467a2d16baf7", "regulator of pressure cooker-b7e2a615129b5b22", "pressure cooker-482717f400d9d69c", "tongs-b6603878904b56c7", "baked potato-88a82400e5588a06", "baked potato2-19a1a155f68dd373", "pen-eccaf363c0489fef", "small plastic measuring cup-d8eea38abc5b9452", "measuring cup-2d0070a85bfd307a", "measuring jug2-5efc4d96e772da13", "strainer-91b22f0ebc4c1b61", "whisk-1b361f7ba0ebabf5", "blue coloured plastic chopping board-34095ffab0b61897", "tablespoon-7d53ef999736940e", "tablespoon2-d81e13831e42281b", "piece of ginger-ef2f94a49487df11", "packet-fc472e10061aab1f", "potato masher-89eb0125b3d23bcb", "tablespoon3-4370da841518ed2a", "knife-c78e90fede872ce6", "spatula-0744f41c0ad71906", "jar of salt-c782779f376f93fb", "lid of salt jar-8a9e16e531a38f92", "pan-6435624c87e84bbc", "lid of pan-8ee5f56217da546c" ], "reasoning": "1. I watched the video to observe that beginning at 02:05, the user begins pouring the Dosa mix (packet of instant dosa mix powder-51cf98845b26ced7) into the bowl (large mixing bowl-7924ca9e89d7c5cc) that is sitting on the scale (scale-d9e338a21afb21ea).\n2. As the user pours Dosa mix into the bowl (packet of instant dosa mix powder-51cf98845b26ced7, large mixing bowl-7924ca9e89d7c5cc), they pause briefly at 02:12, when the number \"22\" shows on the scale's (scale-d9e338a21afb21ea) screen.\n3. The user then resumes from 02:12-02:13 before pausing again, with \"39\" showing on the scale (scale-d9e338a21afb21ea). This is the second pause.\n4. The user then resumes pouring Dosa powder into the bowl (packet of instant dosa mix powder-51cf98845b26ced7, large mixing bowl-7924ca9e89d7c5cc), pausing for the third time at 02:15, when the scale reads \"50\".\n5. From 02:15-02:20, the user picks up the bowl (large mixing bowl-7924ca9e89d7c5cc) and shakes it before setting it down, at which point the scale reads \"50\" again.\n6. I concluded the reasoning by doing the math, which is 50 / 2 = 25.", "referenced_object_masks": [ "packet of instant dosa mix powder-51cf98845b26ced7", "large mixing bowl-7924ca9e89d7c5cc", "scale-d9e338a21afb21ea" ] }, { "key": "P04-20240414-170520:c1ef928b9ea07d60d4fd1f3c68767b70c0c3d5bb", "video_id": "P04-20240414-170520", "question": "How much longer do I whisk without holding the box in my left hand compared to how long I whisk the Dosa mix while holding the box?", "answer_choice_0": "19 seconds", "answer_choice_1": "39 seconds", "answer_choice_2": "59 seconds", "answer_choice_3": "29 seconds", "answer_choice_4": "49 seconds", "answer_id": 1, "answer": "39 seconds", "question_type": [ "Action Recognition", "Numerical Reasoning", "Egocentric & Spatial Perception", "Temporal Reasoning" ], "objects_of_interest": [ "large mixing bowl-7924ca9e89d7c5cc", "plate-9fd8717e4164f5a1", "scale-d9e338a21afb21ea", "purple coloured kitchen rag-3462cec235acdc1a", "plastic sealing clip-5069b3b7b9e3c220", "box of dosa mix-677575d3f430b108", "packet of instant dosa mix powder-51cf98845b26ced7", "larger plate-7114bff13a2bd580", "open notebook-b986d389c6393911", "lid of pressure cooker-6038467a2d16baf7", "regulator of pressure cooker-b7e2a615129b5b22", "pressure cooker-482717f400d9d69c", "tongs-b6603878904b56c7", "baked potato-88a82400e5588a06", "baked potato2-19a1a155f68dd373", "pen-eccaf363c0489fef", "small plastic measuring cup-d8eea38abc5b9452", "measuring cup-2d0070a85bfd307a", "measuring jug2-5efc4d96e772da13", "strainer-91b22f0ebc4c1b61", "whisk-1b361f7ba0ebabf5", "blue coloured plastic chopping board-34095ffab0b61897", "tablespoon-7d53ef999736940e", "tablespoon2-d81e13831e42281b", "piece of ginger-ef2f94a49487df11", "packet-fc472e10061aab1f", "potato masher-89eb0125b3d23bcb", "tablespoon3-4370da841518ed2a", "knife-c78e90fede872ce6", "spatula-0744f41c0ad71906", "jar of salt-c782779f376f93fb", "lid of salt jar-8a9e16e531a38f92", "pan-6435624c87e84bbc", "lid of pan-8ee5f56217da546c" ], "reasoning": "1. I watched the video until I got to 05:11, when the user begins using a whisk (whisk-1b361f7ba0ebabf5) to stir the Dosa mix (packet of instant dosa mix powder-51cf98845b26ced7) in the bowl (large mixing bowl-7924ca9e89d7c5cc).\n2. I continued to watch while the user whisked the mix (packet of instant dosa mix powder-51cf98845b26ced7), doing so until 05:35, when they pick up the box (box of dosa mix-677575d3f430b108) offscreen, though the sound of them grabbing the box can be heard. This means the user whisked without the box for 24 seconds.\n3. After lifting the box (box of dosa mix-677575d3f430b108) with their left hand, I watched the user continue to whisk until 05:52 when they put the box down. This totaled 17 seconds whisking while holding the box.\n4. I continued watching, and saw that the user doesn't stop whisking until 06:24, totaling 32 more seconds that they whisk without holding the box (box of dosa mix-677575d3f430b108).\n5. To finish, I did the math -- 24 + 32 = 56 seconds without the box (box of dosa mix-677575d3f430b108), minus 17 seconds with the box (box of dosa mix-677575d3f430b108) = 39 more seconds.", "referenced_object_masks": [ "whisk-1b361f7ba0ebabf5", "box of dosa mix-677575d3f430b108", "large mixing bowl-7924ca9e89d7c5cc", "packet of instant dosa mix powder-51cf98845b26ced7" ] }, { "key": "P04-20240414-172534:2f469b82ee80ac9eb70aad3f9ea22bfb68970546", "video_id": "P04-20240414-172534", "question": "How many objects did I touch before I moved the knife and potato masher?", "answer_choice_0": "2", "answer_choice_1": "6", "answer_choice_2": "5", "answer_choice_3": "3", "answer_choice_4": "4", "answer_id": 4, "answer": "4", "question_type": [ "Object Recognition", "Action Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "lid of jar-a4def14d261beb1c", "jar of salt-f64c4d47f02d5318", "olive oil bottle-988e8ab350063d22", "metallic cup-0851a72847fe5509", "potato mash-1b4f93be2792667b", "chopping board-4813ed54390daf04", "wooden spatula-26794f7e13d53c97", "spoon-2c8b4c9339e0043b", "plate-ecb8f08f7325c4b4", "pan-7fc85c566bea3f1e", "green chili-2b5a4ced0da7fd14", "bowl-0cb334b4fb830187", "pan2-7972f76f65e42a82", "purple kitchen rag-d8424066bc01bc32", "packet-cfa32e9866a39b29", "packet of ghee-95d2440914ef16f9" ], "reasoning": "1. At 00:13, I saw the user pick up the knife and potato masher (potato mash-1b4f93be2792667b) and place it on top of the blue cutting board (chopping board-4813ed54390daf04). I try to find the objects the user touched before this.\n2. At 00:06, I saw the user pick up a jar (jar of salt-f64c4d47f02d5318) and its lid (lid of jar-a4def14d261beb1c).\n3. At 00:10, I saw the user pick up a bottle of olive oil (olive oil bottle-988e8ab350063d22).\n4. At 00:11, I saw the user pick up a cup (metallic cup-0851a72847fe5509).\n5. The user didn't touch anymore objects before 00:13.\n5. Counting up the jar (jar of salt-f64c4d47f02d5318), the jar's lid (lid of jar-a4def14d261beb1c), the bottle of olive oil (olive oil bottle-988e8ab350063d22), and the cup (metallic cup-0851a72847fe5509), I come to a total of 4 objects touched.", "referenced_object_masks": [ "metallic cup-0851a72847fe5509", "olive oil bottle-988e8ab350063d22", "chopping board-4813ed54390daf04", "jar of salt-f64c4d47f02d5318", "lid of jar-a4def14d261beb1c", "potato mash-1b4f93be2792667b" ] }, { "key": "P04-20240414-172534:3ae96d17ecc4c6070e532bbc75d0e9e07ae64998", "video_id": "P04-20240414-172534", "question": "Where is the microwave in relationship to me when I am first found carrying the blue cutting board to the sink?", "answer_choice_0": "Right of me", "answer_choice_1": "Left of me", "answer_choice_2": "Behind me", "answer_choice_3": "Above me", "answer_choice_4": "In front of me", "answer_id": 2, "answer": "Behind me", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "lid of jar-a4def14d261beb1c", "jar of salt-f64c4d47f02d5318", "olive oil bottle-988e8ab350063d22", "metallic cup-0851a72847fe5509", "potato mash-1b4f93be2792667b", "chopping board-4813ed54390daf04", "wooden spatula-26794f7e13d53c97", "spoon-2c8b4c9339e0043b", "plate-ecb8f08f7325c4b4", "pan-7fc85c566bea3f1e", "green chili-2b5a4ced0da7fd14", "bowl-0cb334b4fb830187", "pan2-7972f76f65e42a82", "purple kitchen rag-d8424066bc01bc32", "packet-cfa32e9866a39b29", "packet of ghee-95d2440914ef16f9" ], "reasoning": "1. At 00:01 to 00:03, I saw the kitchen without seeing the microwave . But at 00:17, I saw a full view of a black microwave when the user turned toward it. It is outside the kitchen, placed against a wall.\n2. I looked for where the user carries the blue cutting board (chopping board-4813ed54390daf04) for the first time and found it from 00:18-00:20.\n3. I realized that the microwave sits behind the user when they first carry the blue cutting board (chopping board-4813ed54390daf04) to the sink.", "referenced_object_masks": [ "chopping board-4813ed54390daf04" ] }, { "key": "P04-20240414-172534:456b714d68011085eb4d1bbffa452957eb457912", "video_id": "P04-20240414-172534", "question": "Where is the last place I put the food that I scraped off the wooden spoon?", "answer_choice_0": "In the sink", "answer_choice_1": "In my mouth", "answer_choice_2": "In the fridge", "answer_choice_3": "In the trash", "answer_choice_4": "In another dish", "answer_id": 1, "answer": "In my mouth", "question_type": [ "Object Recognition", "Action Recognition", "Egocentric & Spatial Perception", "Temporal Reasoning" ], "objects_of_interest": [ "lid of jar-a4def14d261beb1c", "jar of salt-f64c4d47f02d5318", "olive oil bottle-988e8ab350063d22", "metallic cup-0851a72847fe5509", "potato mash-1b4f93be2792667b", "chopping board-4813ed54390daf04", "wooden spatula-26794f7e13d53c97", "spoon-2c8b4c9339e0043b", "plate-ecb8f08f7325c4b4", "pan-7fc85c566bea3f1e", "green chili-2b5a4ced0da7fd14", "bowl-0cb334b4fb830187", "pan2-7972f76f65e42a82", "purple kitchen rag-d8424066bc01bc32", "packet-cfa32e9866a39b29", "packet of ghee-95d2440914ef16f9" ], "reasoning": "1. At 00:23, I saw the user pick up a wooden spoon with food stuck to it.\n2. At 00:24, I saw the user pick up a table spoon (spoon-2c8b4c9339e0043b).\n3. At 00:25, The user scraped the table spoon (spoon-2c8b4c9339e0043b) against the wooden spoon and ate the excess food off of the tablespoon (spoon-2c8b4c9339e0043b).", "referenced_object_masks": [ "spoon-2c8b4c9339e0043b" ] }, { "key": "P04-20240414-172534:a834a91d8aaaee3f81dea521229cd8cba047f6eb", "video_id": "P04-20240414-172534", "question": "How much time elapsed between closing the jar and turning on the faucet?", "answer_choice_0": "1 minute", "answer_choice_1": "28 seconds", "answer_choice_2": "30 seconds", "answer_choice_3": "54 seconds", "answer_choice_4": "2 seconds", "answer_id": 3, "answer": "54 seconds", "question_type": [ "Action Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "lid of jar-a4def14d261beb1c", "jar of salt-f64c4d47f02d5318", "olive oil bottle-988e8ab350063d22", "metallic cup-0851a72847fe5509", "potato mash-1b4f93be2792667b", "chopping board-4813ed54390daf04", "wooden spatula-26794f7e13d53c97", "spoon-2c8b4c9339e0043b", "plate-ecb8f08f7325c4b4", "pan-7fc85c566bea3f1e", "green chili-2b5a4ced0da7fd14", "bowl-0cb334b4fb830187", "pan2-7972f76f65e42a82", "purple kitchen rag-d8424066bc01bc32", "packet-cfa32e9866a39b29", "packet of ghee-95d2440914ef16f9" ], "reasoning": "1. At 00:06, I saw the user screw the lid (lid of jar-a4def14d261beb1c) onto the jar (jar of salt-f64c4d47f02d5318).\n2. At 00:09. I saw the user place the jar (jar of salt-f64c4d47f02d5318) on the stove.\n3. 01:00, I saw the user turn on the faucet to rinse a skillet .\n4. Approximately 54 seconds elapsed between putting the jar (jar of salt-f64c4d47f02d5318) on the lid (lid of jar-a4def14d261beb1c) and turning on the faucet.", "referenced_object_masks": [ "jar of salt-f64c4d47f02d5318", "lid of jar-a4def14d261beb1c" ] }, { "key": "P04-20240414-172534:b0fddcc4a1c826a6e0cee8b1d300c2f95f805d61", "video_id": "P04-20240414-172534", "question": "How many times did I swipe down with the purple rag as I wiped off the front of the pan?", "answer_choice_0": "10 times", "answer_choice_1": "9 times", "answer_choice_2": "12 times", "answer_choice_3": "11 times", "answer_choice_4": "13 times", "answer_id": 2, "answer": "12 times", "question_type": [ "Counting", "Object Recognition" ], "objects_of_interest": [ "lid of jar-a4def14d261beb1c", "jar of salt-f64c4d47f02d5318", "olive oil bottle-988e8ab350063d22", "metallic cup-0851a72847fe5509", "potato mash-1b4f93be2792667b", "chopping board-4813ed54390daf04", "wooden spatula-26794f7e13d53c97", "spoon-2c8b4c9339e0043b", "plate-ecb8f08f7325c4b4", "pan-7fc85c566bea3f1e", "green chili-2b5a4ced0da7fd14", "bowl-0cb334b4fb830187", "pan2-7972f76f65e42a82", "purple kitchen rag-d8424066bc01bc32", "packet-cfa32e9866a39b29", "packet of ghee-95d2440914ef16f9" ], "reasoning": "1. I watched the video and looked for a time when the user wiped off a pan with a purple rag. I found that the user picked up a purple rag (purple kitchen rag-d8424066bc01bc32) at 01:20 as they were holding a pan (pan2-7972f76f65e42a82).\n2. Then, I counted the number of times that the user swipes down with the purple rag (purple kitchen rag-d8424066bc01bc32) on the pan (pan2-7972f76f65e42a82), all of which occur in fairly quick succession from 01:21-01:29.\n3. I counted that swipes down (i.e., toward the user) on the front side of the pan (pan2-7972f76f65e42a82) occurred specifically at 01:21 (twice), 01:22 (twice), 01:23, 01:24 (twice), 01:25 (twice), 01:26, and 01:27 (twice).\n4. At 01:28, the user flipped the pan (pan2-7972f76f65e42a82) and swiped an additional time, but I did not count this since it was not on the front side. There were no other swipes after this point.\n5. I added all of these up to come up with a total of 12 swipes.", "referenced_object_masks": [ "pan2-7972f76f65e42a82", "purple kitchen rag-d8424066bc01bc32" ] }, { "key": "P04-20240414-173337:0f66c71cd9e6d2c611da8f922b3815b7a1bc1e59", "video_id": "P04-20240414-173337", "question": "While filling up the cup with water, in what relative direction is the stove top?", "answer_choice_0": "7 o'clock", "answer_choice_1": "11 o'clock", "answer_choice_2": "6 o'clock", "answer_choice_3": "3 o'clock", "answer_choice_4": "9 o'clock", "answer_id": 0, "answer": "7 o'clock", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "dosa pan-9debc37335229b5c", "cup1-e2691e55b08ca1a5", "bowl1-94b613eceabf02d5", "spoon1-7f049ac04aa24598", "bottle of oil-ceba746a5e0113cd", "cap of oil bottle-5beb89a08aa55a5d", "stack of metallic cups-6bee8eae0499ec52", "cup2-26b4eb6ba8d09d4c", "purple cloth-8024bdb3a74fba8c", "pen-2f0ec3864ba5f527", "notebook-e3dd7d660ceaa5a0", "packet of ghee-70b8b44895fdf527", "cardboard-5cc88969063d9fb7", "small glass-190b7ceb85e2a60e", "pack of plastic containers-167a92625ea52f36", "ladle-d8e9e6b3f8e37d96", "item1-5d2b5a2c22c40b00", "box of green tea-f7d9ebf95b8bb954", "item2-d5941254b0d6c2f0", "pair of scissors-5c62078990d7afc3", "whisk-342a775f52aac858", "measuring cup-37c08f13e2481f39", "sieve-2c84f8331e2dc9db", "green stick-a2aa6dd9523822b2", "parchment paper-924dbf6f4c040239", "whisk2-e96449e4b3bfda82", "apple cutter-148b203068ea0b4f", "tongs-22d09c57fc15eb19", "serving spoon-a2b29c2c87fa998b", "serving spoon2-76ac9e90ea703ab2", "spatula-f53c7ae51a5ef645", "wooden spatula-8aaee04a423cfa9f", "plastic spatula-ad9de35b944d5cc5", "long spatula-bc374dfd7e63af7d", "brush-2a1c1d7453b201d9", "plate1-acbaa7a5f44c6625", "frying pan-898764d61df18062", "clips-ff878f53966fc958", "weighing scales-0129a36e71ec2091", "plate2-996400a0ef4b706e", "wooden spatula2-3201e984c47f3cfb", "knife-b4b61a958413d951" ], "reasoning": "1. I watched the video from 03:25-03:34, observing the left side of the counter, which, from left to right, contains the scale (weighing scales-0129a36e71ec2091), the stove top , the cutting board , and then the electric cook top .\n2. Then, at 03:46, the user turns to the right to reveal the sink , and a black appliance to the right.\n3. Next, from 03:55-04:05, the user grabs a plastic cup (measuring cup-37c08f13e2481f39) and takes it to the black appliance to the right of the sink, from which they fill the cup (measuring cup-37c08f13e2481f39) with water.\n4. Given the previous layout of the kitchen observed in the video, this places the stovetop at approximately 7 o'clock relative to the user.", "referenced_object_masks": [ "measuring cup-37c08f13e2481f39", "weighing scales-0129a36e71ec2091" ] }, { "key": "P04-20240414-173337:0f863de1cd9702462fcfc10b06b1faebda38ed23", "video_id": "P04-20240414-173337", "question": "From where did I remove the third cooking utensil that was not already out on the counter at the start of the video?", "answer_choice_0": "The drawer beneath the stovetop to the right", "answer_choice_1": "The lower left cabinet, above the silverware slider drawer", "answer_choice_2": "The upper right cabinet, above the dosa cooking pan", "answer_choice_3": "The silverware slider drawer in the far left cabinet", "answer_choice_4": "The upper left cabinet, above the propped wooden cooking board", "answer_id": 1, "answer": "The lower left cabinet, above the silverware slider drawer", "question_type": [ "Counting", "Egocentric & Spatial Perception", "Action Recognition", "Object Recognition" ], "objects_of_interest": [ "dosa pan-9debc37335229b5c", "cup1-e2691e55b08ca1a5", "bowl1-94b613eceabf02d5", "spoon1-7f049ac04aa24598", "bottle of oil-ceba746a5e0113cd", "cap of oil bottle-5beb89a08aa55a5d", "stack of metallic cups-6bee8eae0499ec52", "cup2-26b4eb6ba8d09d4c", "purple cloth-8024bdb3a74fba8c", "pen-2f0ec3864ba5f527", "notebook-e3dd7d660ceaa5a0", "packet of ghee-70b8b44895fdf527", "cardboard-5cc88969063d9fb7", "small glass-190b7ceb85e2a60e", "pack of plastic containers-167a92625ea52f36", "ladle-d8e9e6b3f8e37d96", "item1-5d2b5a2c22c40b00", "box of green tea-f7d9ebf95b8bb954", "item2-d5941254b0d6c2f0", "pair of scissors-5c62078990d7afc3", "whisk-342a775f52aac858", "measuring cup-37c08f13e2481f39", "sieve-2c84f8331e2dc9db", "green stick-a2aa6dd9523822b2", "parchment paper-924dbf6f4c040239", "whisk2-e96449e4b3bfda82", "apple cutter-148b203068ea0b4f", "tongs-22d09c57fc15eb19", "serving spoon-a2b29c2c87fa998b", "serving spoon2-76ac9e90ea703ab2", "spatula-f53c7ae51a5ef645", "wooden spatula-8aaee04a423cfa9f", "plastic spatula-ad9de35b944d5cc5", "long spatula-bc374dfd7e63af7d", "brush-2a1c1d7453b201d9", "plate1-acbaa7a5f44c6625", "frying pan-898764d61df18062", "clips-ff878f53966fc958", "weighing scales-0129a36e71ec2091", "plate2-996400a0ef4b706e", "wooden spatula2-3201e984c47f3cfb", "knife-b4b61a958413d951" ], "reasoning": "1. I watched the video for the instances in which the user removes a cooking utensil that was tucked in a drawer or a cabinet somewhere, with the goal of determining where the third one comes from. The first of these utensils removed from a drawer occurs from 00:15-00:21, when the user removes a metal spoon (spoon1-7f049ac04aa24598) from a sliding silverware drawer in the bottom left cabinet.\n2. I then watched the video for a second chronological instance in which a cooking utensil is removed from a drawer somewhere. This occurs at 00:56 when the user removes a small metallic cup (cup2-26b4eb6ba8d09d4c) from the drawer to the right of the stove. This is a cooking utensil insofar as it is used to weigh an ingredient at 01:56-02:03.\n3. Finally, I watched for a third instance in which a cooking utensil was removed from a drawer. This occurs from 02:30-02:41, when a large ladle (ladle-d8e9e6b3f8e37d96) is removed from the cabinet.\n4. I made note of where this large ladle was removed from: it was the lower left cabinet, but above the silverware sliding drawer.", "referenced_object_masks": [ "ladle-d8e9e6b3f8e37d96", "spoon1-7f049ac04aa24598", "cup2-26b4eb6ba8d09d4c" ] }, { "key": "P04-20240414-173337:3fecb68934457fa646e87344462710dd0069aefd", "video_id": "P04-20240414-173337", "question": "If I had used the first spatula I selected and set out for scraping the dosas, instead of the one I ultimately selected, what material would my dosa-scraping spatula be made of?", "answer_choice_0": "Wood", "answer_choice_1": "Rubber", "answer_choice_2": "Silicone", "answer_choice_3": "Metal", "answer_choice_4": "Plastic", "answer_id": 0, "answer": "Wood", "question_type": [ "Counterfactual", "Object Recognition", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "dosa pan-9debc37335229b5c", "cup1-e2691e55b08ca1a5", "bowl1-94b613eceabf02d5", "spoon1-7f049ac04aa24598", "bottle of oil-ceba746a5e0113cd", "cap of oil bottle-5beb89a08aa55a5d", "stack of metallic cups-6bee8eae0499ec52", "cup2-26b4eb6ba8d09d4c", "purple cloth-8024bdb3a74fba8c", "pen-2f0ec3864ba5f527", "notebook-e3dd7d660ceaa5a0", "packet of ghee-70b8b44895fdf527", "cardboard-5cc88969063d9fb7", "small glass-190b7ceb85e2a60e", "pack of plastic containers-167a92625ea52f36", "ladle-d8e9e6b3f8e37d96", "item1-5d2b5a2c22c40b00", "box of green tea-f7d9ebf95b8bb954", "item2-d5941254b0d6c2f0", "pair of scissors-5c62078990d7afc3", "whisk-342a775f52aac858", "measuring cup-37c08f13e2481f39", "sieve-2c84f8331e2dc9db", "green stick-a2aa6dd9523822b2", "parchment paper-924dbf6f4c040239", "whisk2-e96449e4b3bfda82", "apple cutter-148b203068ea0b4f", "tongs-22d09c57fc15eb19", "serving spoon-a2b29c2c87fa998b", "serving spoon2-76ac9e90ea703ab2", "spatula-f53c7ae51a5ef645", "wooden spatula-8aaee04a423cfa9f", "plastic spatula-ad9de35b944d5cc5", "long spatula-bc374dfd7e63af7d", "brush-2a1c1d7453b201d9", "plate1-acbaa7a5f44c6625", "frying pan-898764d61df18062", "clips-ff878f53966fc958", "weighing scales-0129a36e71ec2091", "plate2-996400a0ef4b706e", "wooden spatula2-3201e984c47f3cfb", "knife-b4b61a958413d951" ], "reasoning": "1. I watched the video for the instance where the user selects a spatula to use on the dosas: I first noticed this occur from 05:14-06:28, where the user has the dough prepared and is searching through cabinets to find a proper spatula.\n2. From 06:15-06:20, I note that, after quite a bit of testing and rifling, the user removes a wooden spoon wooden spatula-8aaee04a423cfa9f) from the drawer and places it on the wooden cutting board . This indicates to me that the user is preparing to use it as a dosa-scraping spatula.\n3. From 06:22-06:28, I watch the user open the other drawer, remove a metal spatula (long spatula-bc374dfd7e63af7d) from that drawer, and put the wooden one wooden spatula-8aaee04a423cfa9f) back in its original drawer, replacing it with the new selection. This indicates to me that this wooden spatula (wooden spatula-8aaee04a423cfa9f)) is the one replaced, confirmed when the user uses the metal one (long spatula-bc374dfd7e63af7d) to scrape the dough starting at 08:56.\n4. I then imagine that I had used the replaced spatula wooden spatula-8aaee04a423cfa9f) instead, and make note of the replaced one's material: it was made of wood, making wood the correct answer.", "referenced_object_masks": [ "wooden spatula-8aaee04a423cfa9f", "long spatula-bc374dfd7e63af7d" ] }, { "key": "P04-20240414-173337:5c47c707d5d2f8612fc58a38deb28f3c49d6c313", "video_id": "P04-20240414-173337", "question": "Before putting oil on the pan to make the second cake, how many times do I tap my spoon on the rim of the metal cup?", "answer_choice_0": "3", "answer_choice_1": "1", "answer_choice_2": "2", "answer_choice_3": "5", "answer_choice_4": "0", "answer_id": 3, "answer": "5", "question_type": [ "Action Recognition", "Counting", "Event Occurence", "Object Recognition" ], "objects_of_interest": [ "dosa pan-9debc37335229b5c", "cup1-e2691e55b08ca1a5", "bowl1-94b613eceabf02d5", "spoon1-7f049ac04aa24598", "bottle of oil-ceba746a5e0113cd", "cap of oil bottle-5beb89a08aa55a5d", "stack of metallic cups-6bee8eae0499ec52", "cup2-26b4eb6ba8d09d4c", "purple cloth-8024bdb3a74fba8c", "pen-2f0ec3864ba5f527", "notebook-e3dd7d660ceaa5a0", "packet of ghee-70b8b44895fdf527", "cardboard-5cc88969063d9fb7", "small glass-190b7ceb85e2a60e", "pack of plastic containers-167a92625ea52f36", "ladle-d8e9e6b3f8e37d96", "item1-5d2b5a2c22c40b00", "box of green tea-f7d9ebf95b8bb954", "item2-d5941254b0d6c2f0", "pair of scissors-5c62078990d7afc3", "whisk-342a775f52aac858", "measuring cup-37c08f13e2481f39", "sieve-2c84f8331e2dc9db", "green stick-a2aa6dd9523822b2", "parchment paper-924dbf6f4c040239", "whisk2-e96449e4b3bfda82", "apple cutter-148b203068ea0b4f", "tongs-22d09c57fc15eb19", "serving spoon-a2b29c2c87fa998b", "serving spoon2-76ac9e90ea703ab2", "spatula-f53c7ae51a5ef645", "wooden spatula-8aaee04a423cfa9f", "plastic spatula-ad9de35b944d5cc5", "long spatula-bc374dfd7e63af7d", "brush-2a1c1d7453b201d9", "plate1-acbaa7a5f44c6625", "frying pan-898764d61df18062", "clips-ff878f53966fc958", "weighing scales-0129a36e71ec2091", "plate2-996400a0ef4b706e", "wooden spatula2-3201e984c47f3cfb", "knife-b4b61a958413d951" ], "reasoning": "1. I watched the video to see the user put the clean dosa pan (dosa pan-9debc37335229b5c) on the electric cooktop from 12:17-12:18, signifying that the second cake is next to be made.\n2. The user then looks left and reveals the two metal cups (cup2-26b4eb6ba8d09d4c, cup1-e2691e55b08ca1a5), one of which has a spoon (spoon1-7f049ac04aa24598) in it.\n3. Next, I saw the user turn left and grab the metal spoon (spoon1-7f049ac04aa24598) from the cup (cup1-e2691e55b08ca1a5) at 12:21.\n4. Next, I slowed down the video to 0.7x and counted that from 12:22-12:23, the user hits the spoon (spoon1-7f049ac04aa24598) off the rim of the cup (cup1-e2691e55b08ca1a5) five times. Thus, the answer is 5.", "referenced_object_masks": [ "cup1-e2691e55b08ca1a5", "spoon1-7f049ac04aa24598", "dosa pan-9debc37335229b5c", "cup2-26b4eb6ba8d09d4c" ] }, { "key": "P04-20240414-173337:7a2a4ae62d4e6ebec0e9495a1ecfe55f5461b476", "video_id": "P04-20240414-173337", "question": "When the electric cooktop beeps for the second time after I remove the pan from it, what is fully visible on the counter directly ahead of me?", "answer_choice_0": "Cutting board", "answer_choice_1": "Bag of dosa mix", "answer_choice_2": "Oil bottle", "answer_choice_3": "Plate", "answer_choice_4": "Frying pan", "answer_id": 1, "answer": "Bag of dosa mix", "question_type": [ "Audio Reasoning", "Action Recognition", "Counting", "Event Occurence", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "dosa pan-9debc37335229b5c", "cup1-e2691e55b08ca1a5", "bowl1-94b613eceabf02d5", "spoon1-7f049ac04aa24598", "bottle of oil-ceba746a5e0113cd", "cap of oil bottle-5beb89a08aa55a5d", "stack of metallic cups-6bee8eae0499ec52", "cup2-26b4eb6ba8d09d4c", "purple cloth-8024bdb3a74fba8c", "pen-2f0ec3864ba5f527", "notebook-e3dd7d660ceaa5a0", "packet of ghee-70b8b44895fdf527", "cardboard-5cc88969063d9fb7", "small glass-190b7ceb85e2a60e", "pack of plastic containers-167a92625ea52f36", "ladle-d8e9e6b3f8e37d96", "item1-5d2b5a2c22c40b00", "box of green tea-f7d9ebf95b8bb954", "item2-d5941254b0d6c2f0", "pair of scissors-5c62078990d7afc3", "whisk-342a775f52aac858", "measuring cup-37c08f13e2481f39", "sieve-2c84f8331e2dc9db", "green stick-a2aa6dd9523822b2", "parchment paper-924dbf6f4c040239", "whisk2-e96449e4b3bfda82", "apple cutter-148b203068ea0b4f", "tongs-22d09c57fc15eb19", "serving spoon-a2b29c2c87fa998b", "serving spoon2-76ac9e90ea703ab2", "spatula-f53c7ae51a5ef645", "wooden spatula-8aaee04a423cfa9f", "plastic spatula-ad9de35b944d5cc5", "long spatula-bc374dfd7e63af7d", "brush-2a1c1d7453b201d9", "plate1-acbaa7a5f44c6625", "frying pan-898764d61df18062", "clips-ff878f53966fc958", "weighing scales-0129a36e71ec2091", "plate2-996400a0ef4b706e", "wooden spatula2-3201e984c47f3cfb", "knife-b4b61a958413d951" ], "reasoning": "1. I watched the video to see the user pull the dosa pan (dosa pan-9debc37335229b5c) off the cooktop at 11:15, starting the plating process.\n2. Immediately after removing the pan (dosa pan-9debc37335229b5c) from the cooktop, the first beep is heard.\n3. At this time, I observed that the user is facing the counter, with the pan obscuring a plate (plate1-acbaa7a5f44c6625) below. Since this object is not fully visible, this does not satisfy the question.\n4. Above the plate (plate1-acbaa7a5f44c6625), I observed the bag of dosa mix sitting on the counter near the wall, immediately ahead of the user.\n5. Because the pan (dosa pan-9debc37335229b5c) obscures the plate (plate1-acbaa7a5f44c6625), the bag of dosa mix is the answer.", "referenced_object_masks": [ "plate1-acbaa7a5f44c6625", "dosa pan-9debc37335229b5c" ] }, { "key": "P04-20240414-173337:88eb8dce347cf740f0244a9756d0b0ea4de0a7f7", "video_id": "P04-20240414-173337", "question": "After spooning oil into the measuring cup, what is the highest number to appear on the digital scale when weighing the first ingredient?", "answer_choice_0": "8", "answer_choice_1": "12", "answer_choice_2": "7", "answer_choice_3": "10", "answer_choice_4": "15", "answer_id": 3, "answer": "10", "question_type": [ "Action Recognition", "Object Recognition", "Reading", "Numerical Reasoning" ], "objects_of_interest": [ "dosa pan-9debc37335229b5c", "cup1-e2691e55b08ca1a5", "bowl1-94b613eceabf02d5", "spoon1-7f049ac04aa24598", "bottle of oil-ceba746a5e0113cd", "cap of oil bottle-5beb89a08aa55a5d", "stack of metallic cups-6bee8eae0499ec52", "cup2-26b4eb6ba8d09d4c", "purple cloth-8024bdb3a74fba8c", "pen-2f0ec3864ba5f527", "notebook-e3dd7d660ceaa5a0", "packet of ghee-70b8b44895fdf527", "cardboard-5cc88969063d9fb7", "small glass-190b7ceb85e2a60e", "pack of plastic containers-167a92625ea52f36", "ladle-d8e9e6b3f8e37d96", "item1-5d2b5a2c22c40b00", "box of green tea-f7d9ebf95b8bb954", "item2-d5941254b0d6c2f0", "pair of scissors-5c62078990d7afc3", "whisk-342a775f52aac858", "measuring cup-37c08f13e2481f39", "sieve-2c84f8331e2dc9db", "green stick-a2aa6dd9523822b2", "parchment paper-924dbf6f4c040239", "whisk2-e96449e4b3bfda82", "apple cutter-148b203068ea0b4f", "tongs-22d09c57fc15eb19", "serving spoon-a2b29c2c87fa998b", "serving spoon2-76ac9e90ea703ab2", "spatula-f53c7ae51a5ef645", "wooden spatula-8aaee04a423cfa9f", "plastic spatula-ad9de35b944d5cc5", "long spatula-bc374dfd7e63af7d", "brush-2a1c1d7453b201d9", "plate1-acbaa7a5f44c6625", "frying pan-898764d61df18062", "clips-ff878f53966fc958", "weighing scales-0129a36e71ec2091", "plate2-996400a0ef4b706e", "wooden spatula2-3201e984c47f3cfb", "knife-b4b61a958413d951" ], "reasoning": "1. I watched the video to see when the user measures the oil (bottle of oil-ceba746a5e0113cd) into the cup (cup1-e2691e55b08ca1a5) with the spoon (spoon1-7f049ac04aa24598). I observed that this happens from 00:29-00:50.\n2. I continued to watch the video after seeing this action to find when the user weighs something on a scale (weighing scales-0129a36e71ec2091).\n3. To this end, I observed the scale (weighing scales-0129a36e71ec2091) come into frame around 01:15.\n4. Then, I watched the user clean a metal bowl (cup2-26b4eb6ba8d09d4c) and place it on the scale (weighing scales-0129a36e71ec2091) from 01:20-01:28.\n5. Next, at 01:44, the user grabs a packet of ghee (packet of ghee-70b8b44895fdf527) from the counter.\n6. From 01:52-01:57, the user pushes some ghee (packet of ghee-70b8b44895fdf527) into the cup (cup2-26b4eb6ba8d09d4c), with the scale reading \"8\".\n7. Next, from 02:05-02:10, the user pushes more ghee (packet of ghee-70b8b44895fdf527) into the cup (cup2-26b4eb6ba8d09d4c), during which the scale moves to read \"10\".\n8. The user then put the ghee (packet of ghee-70b8b44895fdf527) down and moves to write in the notebook (notebook-e3dd7d660ceaa5a0), signifying the end of the weighing process.", "referenced_object_masks": [ "notebook-e3dd7d660ceaa5a0", "cup1-e2691e55b08ca1a5", "spoon1-7f049ac04aa24598", "weighing scales-0129a36e71ec2091", "bottle of oil-ceba746a5e0113cd", "packet of ghee-70b8b44895fdf527", "cup2-26b4eb6ba8d09d4c" ] }, { "key": "P04-20240414-173337:9b8a7006e712dfc989338a20db805750168a8c1c", "video_id": "P04-20240414-173337", "question": "During the sequence in which I measure the ghee, open the cabinet, dry my hands, and write in the notebook for the second time, when does writing in the notebook occur in relation to the other acitivites listed?", "answer_choice_0": "After drying hands, but before measuring the ghee or opening the cabinet", "answer_choice_1": "After drying hands and opening the cabinet, but before measuring the ghee", "answer_choice_2": "After drying hands and measuring the ghee, but before opening the cabinet", "answer_choice_3": "After measuring the ghee, but before drying hands or opening the cabinet", "answer_choice_4": "After opening the cabinet and measuring the ghee, but before drying hands", "answer_id": 3, "answer": "After measuring the ghee, but before drying hands or opening the cabinet", "question_type": [ "Action Recognition", "Temporal Reasoning", "Object Recognition" ], "objects_of_interest": [ "dosa pan-9debc37335229b5c", "cup1-e2691e55b08ca1a5", "bowl1-94b613eceabf02d5", "spoon1-7f049ac04aa24598", "bottle of oil-ceba746a5e0113cd", "cap of oil bottle-5beb89a08aa55a5d", "stack of metallic cups-6bee8eae0499ec52", "cup2-26b4eb6ba8d09d4c", "purple cloth-8024bdb3a74fba8c", "pen-2f0ec3864ba5f527", "notebook-e3dd7d660ceaa5a0", "packet of ghee-70b8b44895fdf527", "cardboard-5cc88969063d9fb7", "small glass-190b7ceb85e2a60e", "pack of plastic containers-167a92625ea52f36", "ladle-d8e9e6b3f8e37d96", "item1-5d2b5a2c22c40b00", "box of green tea-f7d9ebf95b8bb954", "item2-d5941254b0d6c2f0", "pair of scissors-5c62078990d7afc3", "whisk-342a775f52aac858", "measuring cup-37c08f13e2481f39", "sieve-2c84f8331e2dc9db", "green stick-a2aa6dd9523822b2", "parchment paper-924dbf6f4c040239", "whisk2-e96449e4b3bfda82", "apple cutter-148b203068ea0b4f", "tongs-22d09c57fc15eb19", "serving spoon-a2b29c2c87fa998b", "serving spoon2-76ac9e90ea703ab2", "spatula-f53c7ae51a5ef645", "wooden spatula-8aaee04a423cfa9f", "plastic spatula-ad9de35b944d5cc5", "long spatula-bc374dfd7e63af7d", "brush-2a1c1d7453b201d9", "plate1-acbaa7a5f44c6625", "frying pan-898764d61df18062", "clips-ff878f53966fc958", "weighing scales-0129a36e71ec2091", "plate2-996400a0ef4b706e", "wooden spatula2-3201e984c47f3cfb", "knife-b4b61a958413d951" ], "reasoning": "1. I watched the video to identify when the notebook (notebook-e3dd7d660ceaa5a0) is written in for the first time, to have a reference for the later sequence: the user first writes in the notebook from 01:36-01:42.\n2. I watched the video to identify which of the events listed in the question I observed first, so that I would know I was watching the correct sequence: the first event occurs from 01:44-02:03, when the user measures out ghee (packet of ghee-70b8b44895fdf527) into a cup (cup2-26b4eb6ba8d09d4c) by weight.\n3. I continued watching for the second event in the sequence, which occurs from 02:07-02:12, when the user writes in the notebook (notebook-e3dd7d660ceaa5a0) for the second time.\n4. I watched for the third event in the sequence, which occurs from 02:13-02:15, when the user dries their hands with a towel (purple cloth-8024bdb3a74fba8c).\n5. I watched for the fourth event in the sequence, which occurs from 02:15-02:17, when the user opens the lower left cabinet.\n6. I made note of when the second notebook writing occurred in this sequence: it came after the ghee measuring, but before the hand drying or cabinet opening, making C the correct answer.", "referenced_object_masks": [ "purple cloth-8024bdb3a74fba8c", "notebook-e3dd7d660ceaa5a0", "packet of ghee-70b8b44895fdf527", "cup2-26b4eb6ba8d09d4c" ] }, { "key": "P04-20240414-173337:a0cd7ccfcee29404d78a2e17bd1d26c0e6caa522", "video_id": "P04-20240414-173337", "question": "What is the approximate time difference between the cooking of the first dosa and second dosa?", "answer_choice_0": "About 30 seconds", "answer_choice_1": "About 90 seconds", "answer_choice_2": "About 120 seconds", "answer_choice_3": "About 60 seconds", "answer_choice_4": "About 45 seconds", "answer_id": 1, "answer": "About 90 seconds", "question_type": [ "Counting", "Numerical Reasoning", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "dosa pan-9debc37335229b5c", "cup1-e2691e55b08ca1a5", "bowl1-94b613eceabf02d5", "spoon1-7f049ac04aa24598", "bottle of oil-ceba746a5e0113cd", "cap of oil bottle-5beb89a08aa55a5d", "stack of metallic cups-6bee8eae0499ec52", "cup2-26b4eb6ba8d09d4c", "purple cloth-8024bdb3a74fba8c", "pen-2f0ec3864ba5f527", "notebook-e3dd7d660ceaa5a0", "packet of ghee-70b8b44895fdf527", "cardboard-5cc88969063d9fb7", "small glass-190b7ceb85e2a60e", "pack of plastic containers-167a92625ea52f36", "ladle-d8e9e6b3f8e37d96", "item1-5d2b5a2c22c40b00", "box of green tea-f7d9ebf95b8bb954", "item2-d5941254b0d6c2f0", "pair of scissors-5c62078990d7afc3", "whisk-342a775f52aac858", "measuring cup-37c08f13e2481f39", "sieve-2c84f8331e2dc9db", "green stick-a2aa6dd9523822b2", "parchment paper-924dbf6f4c040239", "whisk2-e96449e4b3bfda82", "apple cutter-148b203068ea0b4f", "tongs-22d09c57fc15eb19", "serving spoon-a2b29c2c87fa998b", "serving spoon2-76ac9e90ea703ab2", "spatula-f53c7ae51a5ef645", "wooden spatula-8aaee04a423cfa9f", "plastic spatula-ad9de35b944d5cc5", "long spatula-bc374dfd7e63af7d", "brush-2a1c1d7453b201d9", "plate1-acbaa7a5f44c6625", "frying pan-898764d61df18062", "clips-ff878f53966fc958", "weighing scales-0129a36e71ec2091", "plate2-996400a0ef4b706e", "wooden spatula2-3201e984c47f3cfb", "knife-b4b61a958413d951" ], "reasoning": "1. The batter of the first dosa is drizzled by the user onto the pan (dosa pan-9debc37335229b5c) starting at 07:48, so the first dosa begins cooking at 07:48. I then watched the dosa cook until it is scraped off the pan and onto a plate in pieces from 11:25-11:27 by the user. As the dosa is still technically 'cooking' on the hot plate (dosa pan-9debc37335229b5c) during this time until it is plopped onto the plate , I calculate that the first dosa spent about 3 minutes and 38 seconds cooking, or 218 seconds.\n2. I watch the batter of the second dosa be created on the pan (dosa pan-9debc37335229b5c) starting at 13:01. I then watch the dosa cook, until most of it is scraped off the pan (dosa pan-9debc37335229b5c) and onto the plate (plate2-996400a0ef4b706e) from 15:04-15:10 by the user. As the dosa is still technically 'cooking' on the hot plate (dosa pan-9debc37335229b5c) during this time until it is plopped onto the plate (plate2-996400a0ef4b706e), I calculate that the second dosa spent about 2 minutes and 5 seconds cooking, for a total of about 125 seconds.\n3. I then calculate the difference between 218 and 125 with the equation 218-125=93.\n4. The closes approximation to 93 seconds among the answer choices is D, 90, making D the correct answer.", "referenced_object_masks": [ "dosa pan-9debc37335229b5c", "plate2-996400a0ef4b706e" ] }, { "key": "P04-20240414-173337:b85447411a7b4b327b383aec7b37512bd33201e1", "video_id": "P04-20240414-173337", "question": "When I first start to cook on the dosa pan, what is the difference between how many times I flick water onto the pan and how many spoonfuls of oil I put on the pan?", "answer_choice_0": "3", "answer_choice_1": "1", "answer_choice_2": "0", "answer_choice_3": "2", "answer_choice_4": "4", "answer_id": 1, "answer": "1", "question_type": [ "Action Recognition", "Event Occurence", "Numerical Reasoning", "Counting" ], "objects_of_interest": [ "dosa pan-9debc37335229b5c", "cup1-e2691e55b08ca1a5", "bowl1-94b613eceabf02d5", "spoon1-7f049ac04aa24598", "bottle of oil-ceba746a5e0113cd", "cap of oil bottle-5beb89a08aa55a5d", "stack of metallic cups-6bee8eae0499ec52", "cup2-26b4eb6ba8d09d4c", "purple cloth-8024bdb3a74fba8c", "pen-2f0ec3864ba5f527", "notebook-e3dd7d660ceaa5a0", "packet of ghee-70b8b44895fdf527", "cardboard-5cc88969063d9fb7", "small glass-190b7ceb85e2a60e", "pack of plastic containers-167a92625ea52f36", "ladle-d8e9e6b3f8e37d96", "item1-5d2b5a2c22c40b00", "box of green tea-f7d9ebf95b8bb954", "item2-d5941254b0d6c2f0", "pair of scissors-5c62078990d7afc3", "whisk-342a775f52aac858", "measuring cup-37c08f13e2481f39", "sieve-2c84f8331e2dc9db", "green stick-a2aa6dd9523822b2", "parchment paper-924dbf6f4c040239", "whisk2-e96449e4b3bfda82", "apple cutter-148b203068ea0b4f", "tongs-22d09c57fc15eb19", "serving spoon-a2b29c2c87fa998b", "serving spoon2-76ac9e90ea703ab2", "spatula-f53c7ae51a5ef645", "wooden spatula-8aaee04a423cfa9f", "plastic spatula-ad9de35b944d5cc5", "long spatula-bc374dfd7e63af7d", "brush-2a1c1d7453b201d9", "plate1-acbaa7a5f44c6625", "frying pan-898764d61df18062", "clips-ff878f53966fc958", "weighing scales-0129a36e71ec2091", "plate2-996400a0ef4b706e", "wooden spatula2-3201e984c47f3cfb", "knife-b4b61a958413d951" ], "reasoning": "1. I watched the video and observed the user turn on the dosa pan (dosa pan-9debc37335229b5c) at 06:38. \n2. From 06:45-06:51, the user lifts the cup (cup2-26b4eb6ba8d09d4c) filled with oil towards the pan (dosa pan-9debc37335229b5c) before pulling it back and putting their right hand over the pan (dosa pan-9debc37335229b5c).\n3. At 06:52, the user lifts the plastic cup with water (measuring cup-37c08f13e2481f39) and uses their right hand to flick some water onto the dosa pan (dosa pan-9debc37335229b5c).\n4. Next, from 06:54-06:56, the user flicks water onto the the pan (dosa pan-9debc37335229b5c) a second time.\n5. I then observed the user lift the metal cup (cup2-26b4eb6ba8d09d4c) and add 1 spoonful of oil to the dosa pan (dosa pan-9debc37335229b5c) from 06:57-07:08.\n6, The user then moves on to do another task, meaning they flicked water 2 times and added 1 spoonful of oil to the pan (dosa pan-9debc37335229b5c). Therefore, 2 - 1 = 1.", "referenced_object_masks": [ "measuring cup-37c08f13e2481f39", "cup2-26b4eb6ba8d09d4c", "dosa pan-9debc37335229b5c" ] }, { "key": "P04-20240414-173337:c5cfcfb3c0ef1fe1dcac336ac777afb7b8494366", "video_id": "P04-20240414-173337", "question": "What is the ratio of the number of spoonfuls of olive oil I measured into the cup, to the number of spoonfuls of hot ghee I drizzled onto the first dosa?", "answer_choice_0": "5:7", "answer_choice_1": "1:1", "answer_choice_2": "5:8", "answer_choice_3": "1:2", "answer_choice_4": "5:3", "answer_id": 4, "answer": "5:3", "question_type": [ "Counting", "Action Recognition", "Event Occurence", "Object Recognition" ], "objects_of_interest": [ "dosa pan-9debc37335229b5c", "cup1-e2691e55b08ca1a5", "bowl1-94b613eceabf02d5", "spoon1-7f049ac04aa24598", "bottle of oil-ceba746a5e0113cd", "cap of oil bottle-5beb89a08aa55a5d", "stack of metallic cups-6bee8eae0499ec52", "cup2-26b4eb6ba8d09d4c", "purple cloth-8024bdb3a74fba8c", "pen-2f0ec3864ba5f527", "notebook-e3dd7d660ceaa5a0", "packet of ghee-70b8b44895fdf527", "cardboard-5cc88969063d9fb7", "small glass-190b7ceb85e2a60e", "pack of plastic containers-167a92625ea52f36", "ladle-d8e9e6b3f8e37d96", "item1-5d2b5a2c22c40b00", "box of green tea-f7d9ebf95b8bb954", "item2-d5941254b0d6c2f0", "pair of scissors-5c62078990d7afc3", "whisk-342a775f52aac858", "measuring cup-37c08f13e2481f39", "sieve-2c84f8331e2dc9db", "green stick-a2aa6dd9523822b2", "parchment paper-924dbf6f4c040239", "whisk2-e96449e4b3bfda82", "apple cutter-148b203068ea0b4f", "tongs-22d09c57fc15eb19", "serving spoon-a2b29c2c87fa998b", "serving spoon2-76ac9e90ea703ab2", "spatula-f53c7ae51a5ef645", "wooden spatula-8aaee04a423cfa9f", "plastic spatula-ad9de35b944d5cc5", "long spatula-bc374dfd7e63af7d", "brush-2a1c1d7453b201d9", "plate1-acbaa7a5f44c6625", "frying pan-898764d61df18062", "clips-ff878f53966fc958", "weighing scales-0129a36e71ec2091", "plate2-996400a0ef4b706e", "wooden spatula2-3201e984c47f3cfb", "knife-b4b61a958413d951" ], "reasoning": "1. I first watched to observe the user measuring olive oil (bottle of oil-ceba746a5e0113cd) into the cup (cup1-e2691e55b08ca1a5) and counting the number of spoonfuls they measured out. This measuring occurs from 0:30-0:50, during which time I observed 5 spoonfuls of olive oil being measured out by the user.\n2. I then watched during the cooking of the first dosa for the times where the user handles the cup (cup2-26b4eb6ba8d09d4c) of melted ghee and sprinkles it onto the dosa . The first batch of sprinkling occurs from 08:37-08:50, where I counted 3 sprinklings of ghee . Some sprinklings occur from 08:09-08:25, but these are sprinklings from the olive oil cup (cup1-e2691e55b08ca1a5) and thus, I did not count them.\n3. I took the number of sprinklings I observed in step 1, 5, and ratioed them with the number of ghee sprinklings I observed in step 2, 3, to come to a ratio of 5:3.", "referenced_object_masks": [ "cup1-e2691e55b08ca1a5", "bottle of oil-ceba746a5e0113cd", "cup2-26b4eb6ba8d09d4c" ] }, { "key": "P04-20240414-175337:24a58dd187ba79076a79b70f274541f62cc13b64", "video_id": "P04-20240414-175337", "question": "What is the product of the number of rods on the cooling rack leaning against the back wall behind the stovetop and the number of scoops of scrambled egg I put on the plate?", "answer_choice_0": "12", "answer_choice_1": "16", "answer_choice_2": "14", "answer_choice_3": "9", "answer_choice_4": "6", "answer_id": 1, "answer": "16", "question_type": [ "Counting", "Object Recognition", "Numerical Reasoning" ], "objects_of_interest": [ "long spatula-db162eded46b7ff5", "frying pan-c89fd59c44994570", "wooden spatula-08e1677994e5e4df", "cup-1c965b87f51f3af9", "cup2-40ade0a98169a5e9", "spoon-ea7d7fc4d075b4ff", "ladle-4947bf2df550ce7a", "bowl-74327afceccb541a", "knife-8f9ae69353ae5266", "spatula3-61170648df5d8534", "spatula4-b8deb305e94ac8ee", "spatula5-88caf1d5806ade7b", "parchment paper-091d7e6d61a0180d", "electric hob-c7bfb36be907fcd3", "pan2-e73367f41a448874", "tablespoon-41f514ca54322316", "lid of pan-79e747849fbe286b" ], "reasoning": "1. I watched the video to see when the user is standing at the black stovetop , which first occurs from 09:02-09:04.\n2. At this spot, the cooling rack is visible against the wall behind the stovetop , and I counted 8 rods on it.\n3. Then, beginning at 09:02, I watched the user scoop eggs onto the plate .\n4. They make the first scoop from 09:04-09:08\n5. Next, the user makes a second scoop from 09:09-09:13. \n6. I continued to watch to see if the user did anymore scoops, observing them put the lid (lid of pan-79e747849fbe286b) back on the pan (pan2-e73367f41a448874) from 09:17-09:21. The user didn't do scoop any more egg.\n7. I concluded by doing the math, which is 8 x 2 = 16.", "referenced_object_masks": [ "pan2-e73367f41a448874", "lid of pan-79e747849fbe286b" ] }, { "key": "P04-20240414-175337:5d370d8731d9aa831f8b61900ff37f7a20d378a7", "video_id": "P04-20240414-175337", "question": "What is the quotient of the number the user initially sets on the electric cooktop divided by the number of bag clips found on the counter?", "answer_choice_0": "200", "answer_choice_1": "250", "answer_choice_2": "125", "answer_choice_3": "50", "answer_choice_4": "300", "answer_id": 4, "answer": "300", "question_type": [ "Numerical Reasoning", "Reading", "Object Recognition" ], "objects_of_interest": [ "long spatula-db162eded46b7ff5", "frying pan-c89fd59c44994570", "wooden spatula-08e1677994e5e4df", "cup-1c965b87f51f3af9", "cup2-40ade0a98169a5e9", "spoon-ea7d7fc4d075b4ff", "ladle-4947bf2df550ce7a", "bowl-74327afceccb541a", "knife-8f9ae69353ae5266", "spatula3-61170648df5d8534", "spatula4-b8deb305e94ac8ee", "spatula5-88caf1d5806ade7b", "parchment paper-091d7e6d61a0180d", "electric hob-c7bfb36be907fcd3", "pan2-e73367f41a448874", "tablespoon-41f514ca54322316", "lid of pan-79e747849fbe286b" ], "reasoning": "1. I watched the video and noted at 00:13 that there are 2 white bag clips sitting on the counter immediately to the left of the black stovetop (electric hob-c7bfb36be907fcd3).\n2. I watched the user return to the electric cooktop (electric hob-c7bfb36be907fcd3) upon which they place a frying pan (frying pan-c89fd59c44994570) at 00:17. From 00:21-00:23, the user inputs a time on the cooktop (electric hob-c7bfb36be907fcd3): \"600\".\n3. I then did the final math, which is 600 / 2 = 300.", "referenced_object_masks": [ "frying pan-c89fd59c44994570", "electric hob-c7bfb36be907fcd3" ] }, { "key": "P04-20240414-175337:99d49a38e21b48b9832ab973908e88d08eba2937", "video_id": "P04-20240414-175337", "question": "After turning the cooktop on for the second pancake, what is the third item I touch with my right hand?", "answer_choice_0": "Frying pan", "answer_choice_1": "Ladle", "answer_choice_2": "Electric cooktop", "answer_choice_3": "Spatula", "answer_choice_4": "Chef's knife", "answer_id": 0, "answer": "Frying pan", "question_type": [ "Event Occurence", "Egocentric & Spatial Perception", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "long spatula-db162eded46b7ff5", "frying pan-c89fd59c44994570", "wooden spatula-08e1677994e5e4df", "cup-1c965b87f51f3af9", "cup2-40ade0a98169a5e9", "spoon-ea7d7fc4d075b4ff", "ladle-4947bf2df550ce7a", "bowl-74327afceccb541a", "knife-8f9ae69353ae5266", "spatula3-61170648df5d8534", "spatula4-b8deb305e94ac8ee", "spatula5-88caf1d5806ade7b", "parchment paper-091d7e6d61a0180d", "electric hob-c7bfb36be907fcd3", "pan2-e73367f41a448874", "tablespoon-41f514ca54322316", "lid of pan-79e747849fbe286b" ], "reasoning": "1. I watched the video to find that from 01:05-04:07, the user cooks the first pancake .\n2. I continued to watch the video to see that the user turns the cooktop (electric hob-c7bfb36be907fcd3) on from 05:15-05:17. It remains on until 06:27, when the user ladles batter onto the cooktop (electric hob-c7bfb36be907fcd3) to make a second pancake. \n3. After doing this, I observed the first thing the user touched with their right hand was the metal spatula (long spatula-db162eded46b7ff5) from 05:19-05:33. \n4. Next, I saw the user switch the metal spatula (long spatula-db162eded46b7ff5) to their left hand before picking up the wooden spatula (wooden spatula-08e1677994e5e4df) at 05:34. This is the second item touched with the right hand.\n5. The user continued scraping the spatula (long spatula-db162eded46b7ff5) until putting it down and grabbing the frying pan handle (frying pan-c89fd59c44994570) at 05:44. Thus, the frying pan (frying pan-c89fd59c44994570) is the third item grabbed with the right hand after starting the second pancake.", "referenced_object_masks": [ "frying pan-c89fd59c44994570", "electric hob-c7bfb36be907fcd3", "long spatula-db162eded46b7ff5", "wooden spatula-08e1677994e5e4df" ] }, { "key": "P04-20240414-175337:a0d03a7df0c24ebd6e2de6f2b7dec3f17183a9a2", "video_id": "P04-20240414-175337", "question": "How much time elapses between the first time I flip a section of the pancake to the second time I flip a section?", "answer_choice_0": "20 seconds", "answer_choice_1": "11 seconds", "answer_choice_2": "13 seconds", "answer_choice_3": "7 seconds", "answer_choice_4": "12 seconds", "answer_id": 2, "answer": "13 seconds", "question_type": [ "Object Recognition", "Temporal Reasoning", "Numerical Reasoning", "Action Recognition" ], "objects_of_interest": [ "long spatula-db162eded46b7ff5", "frying pan-c89fd59c44994570", "wooden spatula-08e1677994e5e4df", "cup-1c965b87f51f3af9", "cup2-40ade0a98169a5e9", "spoon-ea7d7fc4d075b4ff", "ladle-4947bf2df550ce7a", "bowl-74327afceccb541a", "knife-8f9ae69353ae5266", "spatula3-61170648df5d8534", "spatula4-b8deb305e94ac8ee", "spatula5-88caf1d5806ade7b", "parchment paper-091d7e6d61a0180d", "electric hob-c7bfb36be907fcd3", "pan2-e73367f41a448874", "tablespoon-41f514ca54322316", "lid of pan-79e747849fbe286b" ], "reasoning": "1. I watched the video to see the user ladle pancake mix onto the frying pan (frying pan-c89fd59c44994570) from 01:21-01:32.\n2. The user then lets it sit on the pan (frying pan-c89fd59c44994570) until 03:34, when the user flips 1 segment of the pancake .\n3. I then continued to watch while the user lets the pancake sit like that until they do the next flip of a different segment at 03:47. \n4. I then did the reasoning to determine that 13 seconds lapsed between flips.", "referenced_object_masks": [ "frying pan-c89fd59c44994570" ] }, { "key": "P04-20240414-175337:f77f30b0d79f27b8e8199be4e249de2d4580fce3", "video_id": "P04-20240414-175337", "question": "If I put the pan on the right burner of the electric cooktop, what utensil would be sitting immediately to the right?", "answer_choice_0": "Ladle", "answer_choice_1": "Measuring cup", "answer_choice_2": "Chef's knife", "answer_choice_3": "Potato masher", "answer_choice_4": "Spatula", "answer_id": 3, "answer": "Potato masher", "question_type": [ "Counterfactual", "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "long spatula-db162eded46b7ff5", "frying pan-c89fd59c44994570", "wooden spatula-08e1677994e5e4df", "cup-1c965b87f51f3af9", "cup2-40ade0a98169a5e9", "spoon-ea7d7fc4d075b4ff", "ladle-4947bf2df550ce7a", "bowl-74327afceccb541a", "knife-8f9ae69353ae5266", "spatula3-61170648df5d8534", "spatula4-b8deb305e94ac8ee", "spatula5-88caf1d5806ade7b", "parchment paper-091d7e6d61a0180d", "electric hob-c7bfb36be907fcd3", "pan2-e73367f41a448874", "tablespoon-41f514ca54322316", "lid of pan-79e747849fbe286b" ], "reasoning": "1. I watched the video to get a complete view of the electric cooktop (electric hob-c7bfb36be907fcd3) which is first shown at 00:48. I find it at 02:10 when the user moves over to look at a plate with a knife (knife-8f9ae69353ae5266) and other items .\n2. At 02:10, I see that a potato masher is immediately to the right of the right panel of the cooktop (electric hob-c7bfb36be907fcd3).\n3. If I were to move the pan (frying pan-c89fd59c44994570) to the right panel, the potato masher would be immediately to the right.", "referenced_object_masks": [ "frying pan-c89fd59c44994570", "knife-8f9ae69353ae5266", "electric hob-c7bfb36be907fcd3" ] }, { "key": "P04-20240414-181030:132ece3ac09bcab3e316fcd549891c09565c1779", "video_id": "P04-20240414-181030", "question": "What two monocolored things in the kitchen make up the color of the liquid inside the spray bottle on the left?", "answer_choice_0": "Round fruit and cutting board", "answer_choice_1": "Switch and sponge", "answer_choice_2": "Plate and switch", "answer_choice_3": "Pepper and switch", "answer_choice_4": "Cutting board and switch", "answer_id": 1, "answer": "Switch and sponge", "question_type": [ "Counterfactual", "Object Recognition" ], "objects_of_interest": [ "purple cloth-2ecc5549a4746c56", "weighing scales-55dd2003d1969a5f", "jar of salt-2a05921fb760e71d", "item1-34378ca39a5617aa", "packet of ghee-372efe8c7be66c3a", "pen-31c6522ac684dcc0", "cap of pen-4db0a3a941597422", "packet of black mustard seeds-c25307b936b665a8", "clip-cfb24c5eff912dba", "clip2-83536067b889d759", "packet2-7688f3f3a2c69e8f", "packet of dosa mix-a71d5fd6edbbb783", "dosa mix box-6e035ff67b496b86", "pressure cooker-7c74a195ac8e1cc2", "plate1-18f787b91d7d0106", "wooden spatula-d264abb18e2fddc7", "plastic spatula-c98e52b321fed743", "dosa pan-eeb2c75add404dba", "spoon1-1d0dae078790531a", "cup1-fa774d0154a6733c", "cup2-0507800e5f9b4fb8", "plate3-f9f9ce6a81f1a79e", "hot plate-700230dd06c5dd42", "measuring cup-545d1df3579a4337", "pair of scissors-2b33b1a0ff8b41cb", "brush-919360b3e0e3bed8", "cut pieces of green chilli-1916001d86ab31cf", "frying pan-137e4cb90cbda9f6", "bag of food-a33fd71692a2c0b7", "chopping board-f1ad6202fba03f2d", "notebook-c4b5a1fd20e390b9", "plastic container-adaa989ccd347ef9", "lid of plastic container-a905a0b0a6cb5660", "bottle of oil-2e28508099abcc7c", "peanut butter jar-e48baf3ca94b2b92", "piece of paper-d8beb72af19399ce", "regulator-8aecd3c81bfea98d", "stack of plates-6e317b6de7a900c2", "lid of pressure cooker-01a86d4eec780791", "sponge-c3d6897129b87d79", "soap dispenser-b9d65cf98c56b306", "rubber gasket-a44fd44270739bf7", "spoon2-011ae20db470d296", "ladle-a8829f23c6b94edd", "bowl1-965bce17be556d85", "whisk-a1c748e809670631", "potato masher-7cf29fdeab72c453", "metal slice-43c47739991bce87", "knife-12c8414924ac7da3", "tongs-ed19c7ae8c6c5e09", "plate2-edcf5a5822fe6f54", "chopping board2-a2818e83730c4587", "utensil holder-617f540235e7b12e", "small spoon holder-50751b22c4d756e0", "sponge2-de534bf91d1bc2b1", "measuring jug-5f02d2a4e1a8386c", "black tea towel-2ba4a9c457d79d55", "stack of plastic boxes-df05207d91c0cb4d" ], "reasoning": "1. I watched the video to look for a spray bottle or multiple of them (as the question mentions \"on the left\") and found two spray bottles right next to each other at 02:18.\n2. The color of the liquid inside the spray bottle on the left is orange.\n3. You have to combine red and yellow to get orange.\n4. I watched the video to find something red and saw a red switch on the wall at 02:00.\n5. I watched the video to find something yellow and saw the yellow sponge (sponge-c3d6897129b87d79) at 05:08.\n6. You would combine the color of the switch (red) and the color of the sponge (sponge-c3d6897129b87d79) (yellow) to create the color orange, which is the color of the liquid inside the spray bottle .", "referenced_object_masks": [ "sponge-c3d6897129b87d79" ] }, { "key": "P04-20240414-181030:352525463da501279b1e3ea24766acb37a5eceb3", "video_id": "P04-20240414-181030", "question": "What item did I rinse in the sink but not clean?", "answer_choice_0": "Spatula", "answer_choice_1": "Whisk", "answer_choice_2": "Frying Pan", "answer_choice_3": "Potato Masher", "answer_choice_4": "Cutting board", "answer_id": 2, "answer": "Frying Pan", "question_type": [ "Egocentric & Spatial Perception", "Action Recognition", "Event Occurence", "Object Recognition" ], "objects_of_interest": [ "purple cloth-2ecc5549a4746c56", "weighing scales-55dd2003d1969a5f", "jar of salt-2a05921fb760e71d", "item1-34378ca39a5617aa", "packet of ghee-372efe8c7be66c3a", "pen-31c6522ac684dcc0", "cap of pen-4db0a3a941597422", "packet of black mustard seeds-c25307b936b665a8", "clip-cfb24c5eff912dba", "clip2-83536067b889d759", "packet2-7688f3f3a2c69e8f", "packet of dosa mix-a71d5fd6edbbb783", "dosa mix box-6e035ff67b496b86", "pressure cooker-7c74a195ac8e1cc2", "plate1-18f787b91d7d0106", "wooden spatula-d264abb18e2fddc7", "plastic spatula-c98e52b321fed743", "dosa pan-eeb2c75add404dba", "spoon1-1d0dae078790531a", "cup1-fa774d0154a6733c", "cup2-0507800e5f9b4fb8", "plate3-f9f9ce6a81f1a79e", "hot plate-700230dd06c5dd42", "measuring cup-545d1df3579a4337", "pair of scissors-2b33b1a0ff8b41cb", "brush-919360b3e0e3bed8", "cut pieces of green chilli-1916001d86ab31cf", "frying pan-137e4cb90cbda9f6", "bag of food-a33fd71692a2c0b7", "chopping board-f1ad6202fba03f2d", "notebook-c4b5a1fd20e390b9", "plastic container-adaa989ccd347ef9", "lid of plastic container-a905a0b0a6cb5660", "bottle of oil-2e28508099abcc7c", "peanut butter jar-e48baf3ca94b2b92", "piece of paper-d8beb72af19399ce", "regulator-8aecd3c81bfea98d", "stack of plates-6e317b6de7a900c2", "lid of pressure cooker-01a86d4eec780791", "sponge-c3d6897129b87d79", "soap dispenser-b9d65cf98c56b306", "rubber gasket-a44fd44270739bf7", "spoon2-011ae20db470d296", "ladle-a8829f23c6b94edd", "bowl1-965bce17be556d85", "whisk-a1c748e809670631", "potato masher-7cf29fdeab72c453", "metal slice-43c47739991bce87", "knife-12c8414924ac7da3", "tongs-ed19c7ae8c6c5e09", "plate2-edcf5a5822fe6f54", "chopping board2-a2818e83730c4587", "utensil holder-617f540235e7b12e", "small spoon holder-50751b22c4d756e0", "sponge2-de534bf91d1bc2b1", "measuring jug-5f02d2a4e1a8386c", "black tea towel-2ba4a9c457d79d55", "stack of plastic boxes-df05207d91c0cb4d" ], "reasoning": "1. First, I watched the video and saw a frying pan (dosa pan-eeb2c75add404dba) getting rinsed but not washed between 01:40-01:48, then left in the sink .\n2. The frying pan (dosa pan-eeb2c75add404dba) was then pulled out of the sink at 04:21, with the user still not washing it.\n3. Then, at 19:56, the frying pan (dosa pan-eeb2c75add404dba) was put back in the sink , with the user still not washing it.\n4. So, the only item that was rinsed but not cleaned is the frying pan (dosa pan-eeb2c75add404dba).", "referenced_object_masks": [ "dosa pan-eeb2c75add404dba" ] }, { "key": "P04-20240414-181030:65ab1d9482d91ada39e025f3347abf44d9f79f36", "video_id": "P04-20240414-181030", "question": "Where is the packet of ghee when it's first seen in relation to where I put away the third thing I touch?", "answer_choice_0": "Above on the counter and to my left", "answer_choice_1": "Above on the counter and to my right", "answer_choice_2": "Below on the counter and to my right", "answer_choice_3": "Behind me on the floor", "answer_choice_4": "Below on the counter and to my left", "answer_id": 1, "answer": "Above on the counter and to my right", "question_type": [ "Action Recognition", "Counting", "Event Occurence", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "purple cloth-2ecc5549a4746c56", "weighing scales-55dd2003d1969a5f", "jar of salt-2a05921fb760e71d", "item1-34378ca39a5617aa", "packet of ghee-372efe8c7be66c3a", "pen-31c6522ac684dcc0", "cap of pen-4db0a3a941597422", "packet of black mustard seeds-c25307b936b665a8", "clip-cfb24c5eff912dba", "clip2-83536067b889d759", "packet2-7688f3f3a2c69e8f", "packet of dosa mix-a71d5fd6edbbb783", "dosa mix box-6e035ff67b496b86", "pressure cooker-7c74a195ac8e1cc2", "plate1-18f787b91d7d0106", "wooden spatula-d264abb18e2fddc7", "plastic spatula-c98e52b321fed743", "dosa pan-eeb2c75add404dba", "spoon1-1d0dae078790531a", "cup1-fa774d0154a6733c", "cup2-0507800e5f9b4fb8", "plate3-f9f9ce6a81f1a79e", "hot plate-700230dd06c5dd42", "measuring cup-545d1df3579a4337", "pair of scissors-2b33b1a0ff8b41cb", "brush-919360b3e0e3bed8", "cut pieces of green chilli-1916001d86ab31cf", "frying pan-137e4cb90cbda9f6", "bag of food-a33fd71692a2c0b7", "chopping board-f1ad6202fba03f2d", "notebook-c4b5a1fd20e390b9", "plastic container-adaa989ccd347ef9", "lid of plastic container-a905a0b0a6cb5660", "bottle of oil-2e28508099abcc7c", "peanut butter jar-e48baf3ca94b2b92", "piece of paper-d8beb72af19399ce", "regulator-8aecd3c81bfea98d", "stack of plates-6e317b6de7a900c2", "lid of pressure cooker-01a86d4eec780791", "sponge-c3d6897129b87d79", "soap dispenser-b9d65cf98c56b306", "rubber gasket-a44fd44270739bf7", "spoon2-011ae20db470d296", "ladle-a8829f23c6b94edd", "bowl1-965bce17be556d85", "whisk-a1c748e809670631", "potato masher-7cf29fdeab72c453", "metal slice-43c47739991bce87", "knife-12c8414924ac7da3", "tongs-ed19c7ae8c6c5e09", "plate2-edcf5a5822fe6f54", "chopping board2-a2818e83730c4587", "utensil holder-617f540235e7b12e", "small spoon holder-50751b22c4d756e0", "sponge2-de534bf91d1bc2b1", "measuring jug-5f02d2a4e1a8386c", "black tea towel-2ba4a9c457d79d55", "stack of plastic boxes-df05207d91c0cb4d" ], "reasoning": "1. I watched the user enter the kitchen at 00:00. \n2. I see the packet of ghee (packet of ghee-372efe8c7be66c3a) on the counter at 00:01, in a white package with blue, to the left of the stove top. \n2. I saw the user first touch the purple towel (purple cloth-2ecc5549a4746c56) at 00:02. \n3. I saw the user then touch the scale (weighing scales-55dd2003d1969a5f) at 00:03. This is the second item touched.\n4. At 00:10, I saw the user touch the jar of salt (jar of salt-2a05921fb760e71d). The jar of salt (jar of salt-2a05921fb760e71d) was the third thing the user touched. \n5. From 00:10 to 00:12, I watched the user bend down and put away the jar of salt (jar of salt-2a05921fb760e71d) in the cabinet below where they wiped the scale. This means the packet of ghee (packet of ghee-372efe8c7be66c3a) is now above and to the right of the jar of salt (jar of salt-2a05921fb760e71d).", "referenced_object_masks": [ "weighing scales-55dd2003d1969a5f", "packet of ghee-372efe8c7be66c3a", "jar of salt-2a05921fb760e71d", "purple cloth-2ecc5549a4746c56" ] }, { "key": "P04-20240414-181030:8ad9b396102cf95317ab637f10deb32e1d152786", "video_id": "P04-20240414-181030", "question": "After scraping the blue cutting board over the trash, what was not one of the first 4 items I soaped?", "answer_choice_0": "Tongs", "answer_choice_1": "Blue cutting board", "answer_choice_2": "Spatula", "answer_choice_3": "Metal spoon", "answer_choice_4": "Wooden spoon", "answer_id": 2, "answer": "Spatula", "question_type": [ "Action Recognition", "Event Occurence", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "purple cloth-2ecc5549a4746c56", "weighing scales-55dd2003d1969a5f", "jar of salt-2a05921fb760e71d", "item1-34378ca39a5617aa", "packet of ghee-372efe8c7be66c3a", "pen-31c6522ac684dcc0", "cap of pen-4db0a3a941597422", "packet of black mustard seeds-c25307b936b665a8", "clip-cfb24c5eff912dba", "clip2-83536067b889d759", "packet2-7688f3f3a2c69e8f", "packet of dosa mix-a71d5fd6edbbb783", "dosa mix box-6e035ff67b496b86", "pressure cooker-7c74a195ac8e1cc2", "plate1-18f787b91d7d0106", "wooden spatula-d264abb18e2fddc7", "plastic spatula-c98e52b321fed743", "dosa pan-eeb2c75add404dba", "spoon1-1d0dae078790531a", "cup1-fa774d0154a6733c", "cup2-0507800e5f9b4fb8", "plate3-f9f9ce6a81f1a79e", "hot plate-700230dd06c5dd42", "measuring cup-545d1df3579a4337", "pair of scissors-2b33b1a0ff8b41cb", "brush-919360b3e0e3bed8", "cut pieces of green chilli-1916001d86ab31cf", "frying pan-137e4cb90cbda9f6", "bag of food-a33fd71692a2c0b7", "chopping board-f1ad6202fba03f2d", "notebook-c4b5a1fd20e390b9", "plastic container-adaa989ccd347ef9", "lid of plastic container-a905a0b0a6cb5660", "bottle of oil-2e28508099abcc7c", "peanut butter jar-e48baf3ca94b2b92", "piece of paper-d8beb72af19399ce", "regulator-8aecd3c81bfea98d", "stack of plates-6e317b6de7a900c2", "lid of pressure cooker-01a86d4eec780791", "sponge-c3d6897129b87d79", "soap dispenser-b9d65cf98c56b306", "rubber gasket-a44fd44270739bf7", "spoon2-011ae20db470d296", "ladle-a8829f23c6b94edd", "bowl1-965bce17be556d85", "whisk-a1c748e809670631", "potato masher-7cf29fdeab72c453", "metal slice-43c47739991bce87", "knife-12c8414924ac7da3", "tongs-ed19c7ae8c6c5e09", "plate2-edcf5a5822fe6f54", "chopping board2-a2818e83730c4587", "utensil holder-617f540235e7b12e", "small spoon holder-50751b22c4d756e0", "sponge2-de534bf91d1bc2b1", "measuring jug-5f02d2a4e1a8386c", "black tea towel-2ba4a9c457d79d55", "stack of plastic boxes-df05207d91c0cb4d" ], "reasoning": "1. I watched for the user to scrape the blue cutting board (chopping board2-a2818e83730c4587) over the trash. I saw the user do this from 12:53 to 12:57. \n2. From 13:00 to 13:27, I watched the user soap the blue cutting board (chopping board2-a2818e83730c4587). This is the first item. \n3. From 13:28 to 13:39, I saw the user soap a wooden spoon in the sink. This is the second item.\n4. From 13:40 to 13:54, I watched the user soap a metal spoon (spoon2-011ae20db470d296). This is the third item.\n5. From 13:56 to 14:06, I saw the user soap tongs (tongs-ed19c7ae8c6c5e09). This is the fourth item.", "referenced_object_masks": [ "tongs-ed19c7ae8c6c5e09", "chopping board2-a2818e83730c4587", "spoon2-011ae20db470d296" ] }, { "key": "P04-20240414-181030:95609e931d0fdbbf8abfc0fce88db556033fcfe5", "video_id": "P04-20240414-181030", "question": "How much time passes from when I first touch the purple towel to when I begin rinsing the tongs?", "answer_choice_0": "15:40", "answer_choice_1": "15:00", "answer_choice_2": "14:20", "answer_choice_3": "14:40", "answer_choice_4": "14:10", "answer_id": 3, "answer": "14:40", "question_type": [ "Action Recognition", "Counting", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "purple cloth-2ecc5549a4746c56", "weighing scales-55dd2003d1969a5f", "jar of salt-2a05921fb760e71d", "item1-34378ca39a5617aa", "packet of ghee-372efe8c7be66c3a", "pen-31c6522ac684dcc0", "cap of pen-4db0a3a941597422", "packet of black mustard seeds-c25307b936b665a8", "clip-cfb24c5eff912dba", "clip2-83536067b889d759", "packet2-7688f3f3a2c69e8f", "packet of dosa mix-a71d5fd6edbbb783", "dosa mix box-6e035ff67b496b86", "pressure cooker-7c74a195ac8e1cc2", "plate1-18f787b91d7d0106", "wooden spatula-d264abb18e2fddc7", "plastic spatula-c98e52b321fed743", "dosa pan-eeb2c75add404dba", "spoon1-1d0dae078790531a", "cup1-fa774d0154a6733c", "cup2-0507800e5f9b4fb8", "plate3-f9f9ce6a81f1a79e", "hot plate-700230dd06c5dd42", "measuring cup-545d1df3579a4337", "pair of scissors-2b33b1a0ff8b41cb", "brush-919360b3e0e3bed8", "cut pieces of green chilli-1916001d86ab31cf", "frying pan-137e4cb90cbda9f6", "bag of food-a33fd71692a2c0b7", "chopping board-f1ad6202fba03f2d", "notebook-c4b5a1fd20e390b9", "plastic container-adaa989ccd347ef9", "lid of plastic container-a905a0b0a6cb5660", "bottle of oil-2e28508099abcc7c", "peanut butter jar-e48baf3ca94b2b92", "piece of paper-d8beb72af19399ce", "regulator-8aecd3c81bfea98d", "stack of plates-6e317b6de7a900c2", "lid of pressure cooker-01a86d4eec780791", "sponge-c3d6897129b87d79", "soap dispenser-b9d65cf98c56b306", "rubber gasket-a44fd44270739bf7", "spoon2-011ae20db470d296", "ladle-a8829f23c6b94edd", "bowl1-965bce17be556d85", "whisk-a1c748e809670631", "potato masher-7cf29fdeab72c453", "metal slice-43c47739991bce87", "knife-12c8414924ac7da3", "tongs-ed19c7ae8c6c5e09", "plate2-edcf5a5822fe6f54", "chopping board2-a2818e83730c4587", "utensil holder-617f540235e7b12e", "small spoon holder-50751b22c4d756e0", "sponge2-de534bf91d1bc2b1", "measuring jug-5f02d2a4e1a8386c", "black tea towel-2ba4a9c457d79d55", "stack of plastic boxes-df05207d91c0cb4d" ], "reasoning": "1. I watched for the user to touch the purple towel (purple cloth-2ecc5549a4746c56). At 00:06, I saw the user touch the purple towel (purple cloth-2ecc5549a4746c56) and use it to wipe the scale (weighing scales-55dd2003d1969a5f). \n2. I watched for the user to rinse the tongs (tongs-ed19c7ae8c6c5e09). I saw the user begin rinsing the tongs (tongs-ed19c7ae8c6c5e09) at 14:46. \n3. I subtracted 0:0:06 from 14:46, and got 14:40.", "referenced_object_masks": [ "weighing scales-55dd2003d1969a5f", "tongs-ed19c7ae8c6c5e09", "purple cloth-2ecc5549a4746c56" ] }, { "key": "P04-20240414-181030:a900af2d166f10fe02e04aa7eafeb464f995263a", "video_id": "P04-20240414-181030", "question": "If I multiplied the number of plates I washed by 50, what number would I get?", "answer_choice_0": "300", "answer_choice_1": "250", "answer_choice_2": "100", "answer_choice_3": "500", "answer_choice_4": "150", "answer_id": 4, "answer": "150", "question_type": [ "Numerical Reasoning", "Object Recognition", "Counterfactual", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "purple cloth-2ecc5549a4746c56", "weighing scales-55dd2003d1969a5f", "jar of salt-2a05921fb760e71d", "item1-34378ca39a5617aa", "packet of ghee-372efe8c7be66c3a", "pen-31c6522ac684dcc0", "cap of pen-4db0a3a941597422", "packet of black mustard seeds-c25307b936b665a8", "clip-cfb24c5eff912dba", "clip2-83536067b889d759", "packet2-7688f3f3a2c69e8f", "packet of dosa mix-a71d5fd6edbbb783", "dosa mix box-6e035ff67b496b86", "pressure cooker-7c74a195ac8e1cc2", "plate1-18f787b91d7d0106", "wooden spatula-d264abb18e2fddc7", "plastic spatula-c98e52b321fed743", "dosa pan-eeb2c75add404dba", "spoon1-1d0dae078790531a", "cup1-fa774d0154a6733c", "cup2-0507800e5f9b4fb8", "plate3-f9f9ce6a81f1a79e", "hot plate-700230dd06c5dd42", "measuring cup-545d1df3579a4337", "pair of scissors-2b33b1a0ff8b41cb", "brush-919360b3e0e3bed8", "cut pieces of green chilli-1916001d86ab31cf", "frying pan-137e4cb90cbda9f6", "bag of food-a33fd71692a2c0b7", "chopping board-f1ad6202fba03f2d", "notebook-c4b5a1fd20e390b9", "plastic container-adaa989ccd347ef9", "lid of plastic container-a905a0b0a6cb5660", "bottle of oil-2e28508099abcc7c", "peanut butter jar-e48baf3ca94b2b92", "piece of paper-d8beb72af19399ce", "regulator-8aecd3c81bfea98d", "stack of plates-6e317b6de7a900c2", "lid of pressure cooker-01a86d4eec780791", "sponge-c3d6897129b87d79", "soap dispenser-b9d65cf98c56b306", "rubber gasket-a44fd44270739bf7", "spoon2-011ae20db470d296", "ladle-a8829f23c6b94edd", "bowl1-965bce17be556d85", "whisk-a1c748e809670631", "potato masher-7cf29fdeab72c453", "metal slice-43c47739991bce87", "knife-12c8414924ac7da3", "tongs-ed19c7ae8c6c5e09", "plate2-edcf5a5822fe6f54", "chopping board2-a2818e83730c4587", "utensil holder-617f540235e7b12e", "small spoon holder-50751b22c4d756e0", "sponge2-de534bf91d1bc2b1", "measuring jug-5f02d2a4e1a8386c", "black tea towel-2ba4a9c457d79d55", "stack of plastic boxes-df05207d91c0cb4d" ], "reasoning": "1. First, I watched the video to establish how many plates were washed, and saw the first plate (plate3-f9f9ce6a81f1a79e) being washed at 16:11.\n2. I then saw a second plate (plate1-18f787b91d7d0106) being washed at 16:49.\n3. I then saw a third plate (plate2-edcf5a5822fe6f54) being washed at 17:45.\n4. This makes 3 plates total.\n5. 50x3=150", "referenced_object_masks": [ "plate1-18f787b91d7d0106", "plate3-f9f9ce6a81f1a79e", "plate2-edcf5a5822fe6f54" ] }, { "key": "P04-20240414-181030:c4efebc108237021dec3c9968f75543a664e1bdf", "video_id": "P04-20240414-181030", "question": "What percentage of my open bags of ingredients did I not put a plastic clamp on?", "answer_choice_0": "25%", "answer_choice_1": "75%", "answer_choice_2": "50%", "answer_choice_3": "66.66%", "answer_choice_4": "33.33%", "answer_id": 4, "answer": "33.33%", "question_type": [ "Egocentric & Spatial Perception", "Action Recognition", "Event Occurence", "Object Recognition", "Counterfactual" ], "objects_of_interest": [ "purple cloth-2ecc5549a4746c56", "weighing scales-55dd2003d1969a5f", "jar of salt-2a05921fb760e71d", "item1-34378ca39a5617aa", "packet of ghee-372efe8c7be66c3a", "pen-31c6522ac684dcc0", "cap of pen-4db0a3a941597422", "packet of black mustard seeds-c25307b936b665a8", "clip-cfb24c5eff912dba", "clip2-83536067b889d759", "packet2-7688f3f3a2c69e8f", "packet of dosa mix-a71d5fd6edbbb783", "dosa mix box-6e035ff67b496b86", "pressure cooker-7c74a195ac8e1cc2", "plate1-18f787b91d7d0106", "wooden spatula-d264abb18e2fddc7", "plastic spatula-c98e52b321fed743", "dosa pan-eeb2c75add404dba", "spoon1-1d0dae078790531a", "cup1-fa774d0154a6733c", "cup2-0507800e5f9b4fb8", "plate3-f9f9ce6a81f1a79e", "hot plate-700230dd06c5dd42", "measuring cup-545d1df3579a4337", "pair of scissors-2b33b1a0ff8b41cb", "brush-919360b3e0e3bed8", "cut pieces of green chilli-1916001d86ab31cf", "frying pan-137e4cb90cbda9f6", "bag of food-a33fd71692a2c0b7", "chopping board-f1ad6202fba03f2d", "notebook-c4b5a1fd20e390b9", "plastic container-adaa989ccd347ef9", "lid of plastic container-a905a0b0a6cb5660", "bottle of oil-2e28508099abcc7c", "peanut butter jar-e48baf3ca94b2b92", "piece of paper-d8beb72af19399ce", "regulator-8aecd3c81bfea98d", "stack of plates-6e317b6de7a900c2", "lid of pressure cooker-01a86d4eec780791", "sponge-c3d6897129b87d79", "soap dispenser-b9d65cf98c56b306", "rubber gasket-a44fd44270739bf7", "spoon2-011ae20db470d296", "ladle-a8829f23c6b94edd", "bowl1-965bce17be556d85", "whisk-a1c748e809670631", "potato masher-7cf29fdeab72c453", "metal slice-43c47739991bce87", "knife-12c8414924ac7da3", "tongs-ed19c7ae8c6c5e09", "plate2-edcf5a5822fe6f54", "chopping board2-a2818e83730c4587", "utensil holder-617f540235e7b12e", "small spoon holder-50751b22c4d756e0", "sponge2-de534bf91d1bc2b1", "measuring jug-5f02d2a4e1a8386c", "black tea towel-2ba4a9c457d79d55", "stack of plastic boxes-df05207d91c0cb4d" ], "reasoning": "1. I watched the video to see when a plastic clamp was put on an \"open bag of ingredients\" and saw this happen for the first time with the user put the first plastic clamp (clip-cfb24c5eff912dba) on the first open bag of ingredients (packet of black mustard seeds-c25307b936b665a8) at 00:45.\n2. Then, I saw the second plastic clamp (clip2-83536067b889d759) go on the second open bag of ingredients (packet2-7688f3f3a2c69e8f) at 00:51.\n3. Finally, I noticed that the third open bag of ingredients (packet of dosa mix-a71d5fd6edbbb783) did not have a clamp put on it, it was simply rolled up and put in its box (dosa mix box-6e035ff67b496b86) at 01:06 by the user.\n4. So, out of the 3 open bags of ingredients, only 1 of them did not have a plastic clamp put on it, which comes out to 33.33%.", "referenced_object_masks": [ "packet of black mustard seeds-c25307b936b665a8", "clip2-83536067b889d759", "packet2-7688f3f3a2c69e8f", "dosa mix box-6e035ff67b496b86", "packet of dosa mix-a71d5fd6edbbb783", "clip-cfb24c5eff912dba" ] }, { "key": "P04-20240414-181030:e162dce89a942397e15fd72a1361d7d94cd34718", "video_id": "P04-20240414-181030", "question": "What did I put away before putting the scale away?", "answer_choice_0": "Pack of black mustard seeds", "answer_choice_1": "Pressure cooker", "answer_choice_2": "Pen", "answer_choice_3": "Jar of salt", "answer_choice_4": "Packet of ghee", "answer_id": 4, "answer": "Packet of ghee", "question_type": [ "Action Recognition", "Event Occurence", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "purple cloth-2ecc5549a4746c56", "weighing scales-55dd2003d1969a5f", "jar of salt-2a05921fb760e71d", "item1-34378ca39a5617aa", "packet of ghee-372efe8c7be66c3a", "pen-31c6522ac684dcc0", "cap of pen-4db0a3a941597422", "packet of black mustard seeds-c25307b936b665a8", "clip-cfb24c5eff912dba", "clip2-83536067b889d759", "packet2-7688f3f3a2c69e8f", "packet of dosa mix-a71d5fd6edbbb783", "dosa mix box-6e035ff67b496b86", "pressure cooker-7c74a195ac8e1cc2", "plate1-18f787b91d7d0106", "wooden spatula-d264abb18e2fddc7", "plastic spatula-c98e52b321fed743", "dosa pan-eeb2c75add404dba", "spoon1-1d0dae078790531a", "cup1-fa774d0154a6733c", "cup2-0507800e5f9b4fb8", "plate3-f9f9ce6a81f1a79e", "hot plate-700230dd06c5dd42", "measuring cup-545d1df3579a4337", "pair of scissors-2b33b1a0ff8b41cb", "brush-919360b3e0e3bed8", "cut pieces of green chilli-1916001d86ab31cf", "frying pan-137e4cb90cbda9f6", "bag of food-a33fd71692a2c0b7", "chopping board-f1ad6202fba03f2d", "notebook-c4b5a1fd20e390b9", "plastic container-adaa989ccd347ef9", "lid of plastic container-a905a0b0a6cb5660", "bottle of oil-2e28508099abcc7c", "peanut butter jar-e48baf3ca94b2b92", "piece of paper-d8beb72af19399ce", "regulator-8aecd3c81bfea98d", "stack of plates-6e317b6de7a900c2", "lid of pressure cooker-01a86d4eec780791", "sponge-c3d6897129b87d79", "soap dispenser-b9d65cf98c56b306", "rubber gasket-a44fd44270739bf7", "spoon2-011ae20db470d296", "ladle-a8829f23c6b94edd", "bowl1-965bce17be556d85", "whisk-a1c748e809670631", "potato masher-7cf29fdeab72c453", "metal slice-43c47739991bce87", "knife-12c8414924ac7da3", "tongs-ed19c7ae8c6c5e09", "plate2-edcf5a5822fe6f54", "chopping board2-a2818e83730c4587", "utensil holder-617f540235e7b12e", "small spoon holder-50751b22c4d756e0", "sponge2-de534bf91d1bc2b1", "measuring jug-5f02d2a4e1a8386c", "black tea towel-2ba4a9c457d79d55", "stack of plastic boxes-df05207d91c0cb4d" ], "reasoning": "1. I watched for the user to put away items. The first item I saw the user put away was a jar of salt (jar of salt-2a05921fb760e71d) from 00:10 to 00:12. \n2. The second item I watched the user put away was a packet of ghee (packet of ghee-372efe8c7be66c3a) between 00:17 to 00:20. \n3. From 00:26 to 00:30, I watched the user put away the scale (weighing scales-55dd2003d1969a5f). The item the user put away before this was the packet of ghee (packet of ghee-372efe8c7be66c3a).", "referenced_object_masks": [ "packet of ghee-372efe8c7be66c3a", "weighing scales-55dd2003d1969a5f", "jar of salt-2a05921fb760e71d" ] }, { "key": "P04-20240414-181030:e2c206102eed0e0215f67d661c89374af5ebc2f2", "video_id": "P04-20240414-181030", "question": "Not counting wooden spoons or ladles, how many spoons did I clean?", "answer_choice_0": "6", "answer_choice_1": "4", "answer_choice_2": "9", "answer_choice_3": "13", "answer_choice_4": "2", "answer_id": 1, "answer": "4", "question_type": [ "Egocentric & Spatial Perception", "Counting", "Action Recognition", "Object Recognition" ], "objects_of_interest": [ "purple cloth-2ecc5549a4746c56", "weighing scales-55dd2003d1969a5f", "jar of salt-2a05921fb760e71d", "item1-34378ca39a5617aa", "packet of ghee-372efe8c7be66c3a", "pen-31c6522ac684dcc0", "cap of pen-4db0a3a941597422", "packet of black mustard seeds-c25307b936b665a8", "clip-cfb24c5eff912dba", "clip2-83536067b889d759", "packet2-7688f3f3a2c69e8f", "packet of dosa mix-a71d5fd6edbbb783", "dosa mix box-6e035ff67b496b86", "pressure cooker-7c74a195ac8e1cc2", "plate1-18f787b91d7d0106", "wooden spatula-d264abb18e2fddc7", "plastic spatula-c98e52b321fed743", "dosa pan-eeb2c75add404dba", "spoon1-1d0dae078790531a", "cup1-fa774d0154a6733c", "cup2-0507800e5f9b4fb8", "plate3-f9f9ce6a81f1a79e", "hot plate-700230dd06c5dd42", "measuring cup-545d1df3579a4337", "pair of scissors-2b33b1a0ff8b41cb", "brush-919360b3e0e3bed8", "cut pieces of green chilli-1916001d86ab31cf", "frying pan-137e4cb90cbda9f6", "bag of food-a33fd71692a2c0b7", "chopping board-f1ad6202fba03f2d", "notebook-c4b5a1fd20e390b9", "plastic container-adaa989ccd347ef9", "lid of plastic container-a905a0b0a6cb5660", "bottle of oil-2e28508099abcc7c", "peanut butter jar-e48baf3ca94b2b92", "piece of paper-d8beb72af19399ce", "regulator-8aecd3c81bfea98d", "stack of plates-6e317b6de7a900c2", "lid of pressure cooker-01a86d4eec780791", "sponge-c3d6897129b87d79", "soap dispenser-b9d65cf98c56b306", "rubber gasket-a44fd44270739bf7", "spoon2-011ae20db470d296", "ladle-a8829f23c6b94edd", "bowl1-965bce17be556d85", "whisk-a1c748e809670631", "potato masher-7cf29fdeab72c453", "metal slice-43c47739991bce87", "knife-12c8414924ac7da3", "tongs-ed19c7ae8c6c5e09", "plate2-edcf5a5822fe6f54", "chopping board2-a2818e83730c4587", "utensil holder-617f540235e7b12e", "small spoon holder-50751b22c4d756e0", "sponge2-de534bf91d1bc2b1", "measuring jug-5f02d2a4e1a8386c", "black tea towel-2ba4a9c457d79d55", "stack of plastic boxes-df05207d91c0cb4d" ], "reasoning": "1. I watched the video to see how many spoons were cleaned, and I saw the first spoon (spoon2-011ae20db470d296) being cleaned at 10:25.\n2. At 10:40, the user begins to clean a ladle (ladle-a8829f23c6b94edd). Since the question specifies no ladles, I do not count this.\n3. At 13:29, the user begins to clean a wooden spoon (wooden spatula-d264abb18e2fddc7). Since the question specifies no wooden spoons, I do not count this either.\n4. I then saw a second spoon (spoon1-1d0dae078790531a) being cleaned at 13:51.\n5. I then saw a third spoon being cleaned at 14:13.\n6. I then saw a fourth spoon being cleaned at 14:25.\n7. I continued to watch to see whether any more spoons were cleaned. Once determining that the user did not clean any more, I concluded that 4 spoons total were cleaned.", "referenced_object_masks": [ "wooden spatula-d264abb18e2fddc7", "ladle-a8829f23c6b94edd", "spoon1-1d0dae078790531a", "spoon2-011ae20db470d296" ] }, { "key": "P04-20240414-181030:feabe8fdedd3ceb8e6903da3996b3843f6c1ee26", "video_id": "P04-20240414-181030", "question": "When I'm standing at the sink, if the sink is at 12 o'clock, what clock position would I have to turn to face where I wiped the scale?", "answer_choice_0": "7 o'clock", "answer_choice_1": "11 o'clock", "answer_choice_2": "2 o'clock", "answer_choice_3": "9 o'clock", "answer_choice_4": "6 o'clock", "answer_id": 0, "answer": "7 o'clock", "question_type": [ "Action Recognition", "Event Occurence", "Situational Awareness", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "purple cloth-2ecc5549a4746c56", "weighing scales-55dd2003d1969a5f", "jar of salt-2a05921fb760e71d", "item1-34378ca39a5617aa", "packet of ghee-372efe8c7be66c3a", "pen-31c6522ac684dcc0", "cap of pen-4db0a3a941597422", "packet of black mustard seeds-c25307b936b665a8", "clip-cfb24c5eff912dba", "clip2-83536067b889d759", "packet2-7688f3f3a2c69e8f", "packet of dosa mix-a71d5fd6edbbb783", "dosa mix box-6e035ff67b496b86", "pressure cooker-7c74a195ac8e1cc2", "plate1-18f787b91d7d0106", "wooden spatula-d264abb18e2fddc7", "plastic spatula-c98e52b321fed743", "dosa pan-eeb2c75add404dba", "spoon1-1d0dae078790531a", "cup1-fa774d0154a6733c", "cup2-0507800e5f9b4fb8", "plate3-f9f9ce6a81f1a79e", "hot plate-700230dd06c5dd42", "measuring cup-545d1df3579a4337", "pair of scissors-2b33b1a0ff8b41cb", "brush-919360b3e0e3bed8", "cut pieces of green chilli-1916001d86ab31cf", "frying pan-137e4cb90cbda9f6", "bag of food-a33fd71692a2c0b7", "chopping board-f1ad6202fba03f2d", "notebook-c4b5a1fd20e390b9", "plastic container-adaa989ccd347ef9", "lid of plastic container-a905a0b0a6cb5660", "bottle of oil-2e28508099abcc7c", "peanut butter jar-e48baf3ca94b2b92", "piece of paper-d8beb72af19399ce", "regulator-8aecd3c81bfea98d", "stack of plates-6e317b6de7a900c2", "lid of pressure cooker-01a86d4eec780791", "sponge-c3d6897129b87d79", "soap dispenser-b9d65cf98c56b306", "rubber gasket-a44fd44270739bf7", "spoon2-011ae20db470d296", "ladle-a8829f23c6b94edd", "bowl1-965bce17be556d85", "whisk-a1c748e809670631", "potato masher-7cf29fdeab72c453", "metal slice-43c47739991bce87", "knife-12c8414924ac7da3", "tongs-ed19c7ae8c6c5e09", "plate2-edcf5a5822fe6f54", "chopping board2-a2818e83730c4587", "utensil holder-617f540235e7b12e", "small spoon holder-50751b22c4d756e0", "sponge2-de534bf91d1bc2b1", "measuring jug-5f02d2a4e1a8386c", "black tea towel-2ba4a9c457d79d55", "stack of plastic boxes-df05207d91c0cb4d" ], "reasoning": "1. I watched for the user to wipe the scale (weighing scales-55dd2003d1969a5f) with the purple towel (purple cloth-2ecc5549a4746c56). I saw the user do this at 00:06. \n2. At 00:14, I watched the user turn slightly to their right and the sink comes into view. \n3. if the user were standing at the sink, the place where they wiped the scale (weighing scales-55dd2003d1969a5f) with the towel (purple cloth-2ecc5549a4746c56) would be in the 7 o'clock position.", "referenced_object_masks": [ "weighing scales-55dd2003d1969a5f", "purple cloth-2ecc5549a4746c56" ] }, { "key": "P04-20240415-175615:297f9fc8430e2eaae0b20ea0df6b8e136530ffff", "video_id": "P04-20240415-175615", "question": "If I placed the portable burner directly to the left of the stovetop at the halfway point of the video, with its lengthwise edge abutting the left side of the stove, which of the following items would not be competing with the portable burner for space?", "answer_choice_0": "The bag of brown powder", "answer_choice_1": "The rice bag", "answer_choice_2": "The notebook", "answer_choice_3": "The food scale", "answer_choice_4": "The covered pot", "answer_id": 4, "answer": "The covered pot", "question_type": [ "Egocentric & Spatial Perception", "Counterfactual", "Object Recognition", "Situational Awareness", "Temporal Reasoning" ], "objects_of_interest": [ "towel-bb81734dcb0fefb7", "chopping board-2db62a7cdb7e1e33", "knife-4100c0833d8ed206", "tablespoon-33190f152b8c6d64", "tablespoon2-f5edc00e9306a9c6", "bottle of olive oil-e121f234af8bb2d4", "saucepan-eabf1412611fd752", "bag-d6ff78597c31ca9c", "metallic tumbler-ff33bd04cbcaa47d", "metalic cup-8d0d2f50af8d900f", "weighing scale-f8aeea8253d12ae4", "plastic container-4c9984bc3c9d79ee", "bottle of oil2-4a40a6d886b67d72", "garam masala powder packet-4a62e4a1a13a8356", "rice bag-88c30c4db5abd9a0", "garlic-f2e741177dbd9fa4", "ginger-61c0b2e6a3e58909", "onion-c7b2ceecdf5af7f1", "chili-73caa3d78fe83dde", "container2-a70c3da20660430b", "container3-f21d7e45117dbe19", "lid-6dd3409ce3807d6c", "pepper-d3962cd68f38a74c", "lemon-6c197df4cd691e46", "pen-0c622a7de1f23000", "plate-c3e9b6d83f8a8504", "black plastic container box-d1227ba8a2f0c56a", "lid2-7238e4b5987c4ef9", "plate2-4a49834ce6d0e936", "spice bag-e7114e489fb3c9db", "spice bag2-1491417e8ca26a8a", "spice bag3-d29a9d7dd883e21f", "plastic bag of cashews-2db3ce7eda53883f", "bag5-1c51fe6be09b68b7", "tablespoon3-5af7494a04121ed7", "tablespoon4-517f87a292530bc5", "notebook-01551372823e1b54", "cap of pen-786e694857ab4581", "garlic clove-c8b083de7080c19d" ], "reasoning": "1. I first determine the halfway point of the video through simple mathematics. The video is 15 minutes and 18 seconds long, or 918 seconds - I divide this by 2 to get a halfway mark of 459 seconds, or 7 minutes and 39 seconds - the halfway point of the video is therefore at the 07:39 mark.\n2. I observe the state of the left side of the counter at around 07:39, and imagine that the portable burner were placed just to the left of the stovetop. From here, I look at the items on that part of the counter, and determine what would be competing for space with the portable burner if they remained.\n3. On the left side of the countertop, I see a green bag of rice (rice bag-88c30c4db5abd9a0), 2 spice bags (bag5-1c51fe6be09b68b7), a notebook and pen (notebook-01551372823e1b54, pen-0c622a7de1f23000), a food scale (weighing scale-f8aeea8253d12ae4), 2 metal racks , a plate (plate-c3e9b6d83f8a8504), and a wooden cutting board . Based on the size of the portable burner , I note that all of these items would be competing with the burner for space.\n4. I compare the items I observed on the left side of the countertop of the answer choices, and determine that the only one that the portable burner would not overlap would be the covered pot (saucepan-eabf1412611fd752) which sits on the stovetop, making D the correct answer.", "referenced_object_masks": [ "bag5-1c51fe6be09b68b7", "saucepan-eabf1412611fd752", "rice bag-88c30c4db5abd9a0", "plate-c3e9b6d83f8a8504", "weighing scale-f8aeea8253d12ae4", "notebook-01551372823e1b54", "pen-0c622a7de1f23000" ] }, { "key": "P04-20240415-175615:4a566165147e6e148891bbec485be139adb48181", "video_id": "P04-20240415-175615", "question": "If you multiplied the number of times I pumped the soap dispenser when first washing my hands, by the number of times I touched the faucet handles in front of me when first washing my hands, what would the answer be?", "answer_choice_0": "15", "answer_choice_1": "18", "answer_choice_2": "6", "answer_choice_3": "8", "answer_choice_4": "4", "answer_id": 2, "answer": "6", "question_type": [ "Numerical Reasoning", "Counting", "Action Recognition", "Object Recognition" ], "objects_of_interest": [ "towel-bb81734dcb0fefb7", "chopping board-2db62a7cdb7e1e33", "knife-4100c0833d8ed206", "tablespoon-33190f152b8c6d64", "tablespoon2-f5edc00e9306a9c6", "bottle of olive oil-e121f234af8bb2d4", "saucepan-eabf1412611fd752", "bag-d6ff78597c31ca9c", "metallic tumbler-ff33bd04cbcaa47d", "metalic cup-8d0d2f50af8d900f", "weighing scale-f8aeea8253d12ae4", "plastic container-4c9984bc3c9d79ee", "bottle of oil2-4a40a6d886b67d72", "garam masala powder packet-4a62e4a1a13a8356", "rice bag-88c30c4db5abd9a0", "garlic-f2e741177dbd9fa4", "ginger-61c0b2e6a3e58909", "onion-c7b2ceecdf5af7f1", "chili-73caa3d78fe83dde", "container2-a70c3da20660430b", "container3-f21d7e45117dbe19", "lid-6dd3409ce3807d6c", "pepper-d3962cd68f38a74c", "lemon-6c197df4cd691e46", "pen-0c622a7de1f23000", "plate-c3e9b6d83f8a8504", "black plastic container box-d1227ba8a2f0c56a", "lid2-7238e4b5987c4ef9", "plate2-4a49834ce6d0e936", "spice bag-e7114e489fb3c9db", "spice bag2-1491417e8ca26a8a", "spice bag3-d29a9d7dd883e21f", "plastic bag of cashews-2db3ce7eda53883f", "bag5-1c51fe6be09b68b7", "tablespoon3-5af7494a04121ed7", "tablespoon4-517f87a292530bc5", "notebook-01551372823e1b54", "cap of pen-786e694857ab4581", "garlic clove-c8b083de7080c19d" ], "reasoning": "1. I found the timestamp where the user first washed their hands at 00:06, and saw that they reached for a faucet handle.\n2. The user then turned the faucet, and ran their hands under water, before reaching for the soap dispenser at 00:09.\n3. The user pumped the dispenser handle 3 times, and resumed washing their hands before reaching for the faucet handle to turn off the water at 00:26.\n4. If we multiply the number of times the user touched the handle by the number of times they pumped the soap dispenser, it would look like an equation of 2 multiplied by 3, which equals 6.", "referenced_object_masks": [] }, { "key": "P04-20240415-175615:7d601f37c057f14574473e216c0792fc5549731a", "video_id": "P04-20240415-175615", "question": "When do I chop the onions on the cutting board?", "answer_choice_0": "Before chopping the garlic, chopping the peppers, and peeling the garlic", "answer_choice_1": "Before peeling the garlic and chopping the ginger, but after chopping the peppers", "answer_choice_2": "Before chopping the ginger, but after chopping the peppers and peeling the garlic", "answer_choice_3": "Before chopping the peppers, but after chopping the ginger and peeling the garlic", "answer_choice_4": "Before chopping the peppers and peeling the garlic, but after chopping the ginger", "answer_id": 4, "answer": "Before chopping the peppers and peeling the garlic, but after chopping the ginger", "question_type": [ "Temporal Reasoning", "Object Recognition", "Event Occurence", "Action Recognition" ], "objects_of_interest": [ "towel-bb81734dcb0fefb7", "chopping board-2db62a7cdb7e1e33", "knife-4100c0833d8ed206", "tablespoon-33190f152b8c6d64", "tablespoon2-f5edc00e9306a9c6", "bottle of olive oil-e121f234af8bb2d4", "saucepan-eabf1412611fd752", "bag-d6ff78597c31ca9c", "metallic tumbler-ff33bd04cbcaa47d", "metalic cup-8d0d2f50af8d900f", "weighing scale-f8aeea8253d12ae4", "plastic container-4c9984bc3c9d79ee", "bottle of oil2-4a40a6d886b67d72", "garam masala powder packet-4a62e4a1a13a8356", "rice bag-88c30c4db5abd9a0", "garlic-f2e741177dbd9fa4", "ginger-61c0b2e6a3e58909", "onion-c7b2ceecdf5af7f1", "chili-73caa3d78fe83dde", "container2-a70c3da20660430b", "container3-f21d7e45117dbe19", "lid-6dd3409ce3807d6c", "pepper-d3962cd68f38a74c", "lemon-6c197df4cd691e46", "pen-0c622a7de1f23000", "plate-c3e9b6d83f8a8504", "black plastic container box-d1227ba8a2f0c56a", "lid2-7238e4b5987c4ef9", "plate2-4a49834ce6d0e936", "spice bag-e7114e489fb3c9db", "spice bag2-1491417e8ca26a8a", "spice bag3-d29a9d7dd883e21f", "plastic bag of cashews-2db3ce7eda53883f", "bag5-1c51fe6be09b68b7", "tablespoon3-5af7494a04121ed7", "tablespoon4-517f87a292530bc5", "notebook-01551372823e1b54", "cap of pen-786e694857ab4581", "garlic clove-c8b083de7080c19d" ], "reasoning": "1. I observe that the answer choices require me to find when the onion (onion-c7b2ceecdf5af7f1), peppers (chili-73caa3d78fe83dde, pepper-d3962cd68f38a74c), and ginger (ginger-61c0b2e6a3e58909) are chopped, and when the garlic (garlic clove-c8b083de7080c19d) is peeled, in chronological order. I therefore watch the video to determine the first event in this chronology, and observe it from 10:20-12:09, when the user chops the ginger (ginger-61c0b2e6a3e58909).\n2. I watch the video for the second chronological event, which occurs from 12:21-13:38, when the user chops the onion (onion-c7b2ceecdf5af7f1).\n3. I watch the video for the third chronological event, which occurs from 13:39-14:00, where the user chops the peppers (chili-73caa3d78fe83dde, pepper-d3962cd68f38a74c).\n4. I watch the video for the fourth chronological event, which occurs from 14:10-14:42, where the user peels the garlic clove (garlic clove-c8b083de7080c19d).\n5. I compare this chronology to the answer choices, and find that C gets it correct.", "referenced_object_masks": [ "ginger-61c0b2e6a3e58909", "chili-73caa3d78fe83dde", "pepper-d3962cd68f38a74c", "onion-c7b2ceecdf5af7f1", "garlic clove-c8b083de7080c19d" ] }, { "key": "P04-20240415-175615:ac5f4ac995417902b9ab9be9e9e34a4e3a48e15a", "video_id": "P04-20240415-175615", "question": "Is there a common purpose to what I'm doing from 12:37-12:45 and what I'm doing from 14:15-14:40?", "answer_choice_0": "Yes, I'm dicing vegetables", "answer_choice_1": "Yes, I'm arranging ingredients on a plate", "answer_choice_2": "No, I'm grilling vegetables and cutting vegetables", "answer_choice_3": "Yes, I'm peeling layers off vegetables", "answer_choice_4": "No, I'm putting an item into the fridge and taking and item out", "answer_id": 3, "answer": "Yes, I'm peeling layers off vegetables", "question_type": [ "Action Recognition", "Goal Reasoning", "Object Recognition", "Temporal Reasoning", "State Changes", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "towel-bb81734dcb0fefb7", "chopping board-2db62a7cdb7e1e33", "knife-4100c0833d8ed206", "tablespoon-33190f152b8c6d64", "tablespoon2-f5edc00e9306a9c6", "bottle of olive oil-e121f234af8bb2d4", "saucepan-eabf1412611fd752", "bag-d6ff78597c31ca9c", "metallic tumbler-ff33bd04cbcaa47d", "metalic cup-8d0d2f50af8d900f", "weighing scale-f8aeea8253d12ae4", "plastic container-4c9984bc3c9d79ee", "bottle of oil2-4a40a6d886b67d72", "garam masala powder packet-4a62e4a1a13a8356", "rice bag-88c30c4db5abd9a0", "garlic-f2e741177dbd9fa4", "ginger-61c0b2e6a3e58909", "onion-c7b2ceecdf5af7f1", "chili-73caa3d78fe83dde", "container2-a70c3da20660430b", "container3-f21d7e45117dbe19", "lid-6dd3409ce3807d6c", "pepper-d3962cd68f38a74c", "lemon-6c197df4cd691e46", "pen-0c622a7de1f23000", "plate-c3e9b6d83f8a8504", "black plastic container box-d1227ba8a2f0c56a", "lid2-7238e4b5987c4ef9", "plate2-4a49834ce6d0e936", "spice bag-e7114e489fb3c9db", "spice bag2-1491417e8ca26a8a", "spice bag3-d29a9d7dd883e21f", "plastic bag of cashews-2db3ce7eda53883f", "bag5-1c51fe6be09b68b7", "tablespoon3-5af7494a04121ed7", "tablespoon4-517f87a292530bc5", "notebook-01551372823e1b54", "cap of pen-786e694857ab4581", "garlic clove-c8b083de7080c19d" ], "reasoning": "1. I look at the 12:37-12:45 timecode, and I notice that the user is peeling an onion half , the product of a cut onion (onion-c7b2ceecdf5af7f1).\n2. I look at the 14:15-14:40 timecode, and I notice that the user is peeling a clove of garlic (garlic clove-c8b083de7080c19d).\n3. Both of these timecodes showcase peeling layers off vegetables.\n4. Peeling layers off vegetables is, therefore, the answer.", "referenced_object_masks": [ "garlic clove-c8b083de7080c19d", "onion-c7b2ceecdf5af7f1" ] }, { "key": "P04-20240415-175615:b7735b164f336af417af98809afc933ff3dc0d2f", "video_id": "P04-20240415-175615", "question": "When I first take items from the container of vegetables, fruit, and fungi, which ingredient do I place furthest from me?", "answer_choice_0": "Garlic", "answer_choice_1": "Mushroom", "answer_choice_2": "Lemon", "answer_choice_3": "Pepper", "answer_choice_4": "Onion", "answer_id": 2, "answer": "Lemon", "question_type": [ "Object Recognition", "Egocentric & Spatial Perception", "Situational Awareness", "Action Recognition" ], "objects_of_interest": [ "towel-bb81734dcb0fefb7", "chopping board-2db62a7cdb7e1e33", "knife-4100c0833d8ed206", "tablespoon-33190f152b8c6d64", "tablespoon2-f5edc00e9306a9c6", "bottle of olive oil-e121f234af8bb2d4", "saucepan-eabf1412611fd752", "bag-d6ff78597c31ca9c", "metallic tumbler-ff33bd04cbcaa47d", "metalic cup-8d0d2f50af8d900f", "weighing scale-f8aeea8253d12ae4", "plastic container-4c9984bc3c9d79ee", "bottle of oil2-4a40a6d886b67d72", "garam masala powder packet-4a62e4a1a13a8356", "rice bag-88c30c4db5abd9a0", "garlic-f2e741177dbd9fa4", "ginger-61c0b2e6a3e58909", "onion-c7b2ceecdf5af7f1", "chili-73caa3d78fe83dde", "container2-a70c3da20660430b", "container3-f21d7e45117dbe19", "lid-6dd3409ce3807d6c", "pepper-d3962cd68f38a74c", "lemon-6c197df4cd691e46", "pen-0c622a7de1f23000", "plate-c3e9b6d83f8a8504", "black plastic container box-d1227ba8a2f0c56a", "lid2-7238e4b5987c4ef9", "plate2-4a49834ce6d0e936", "spice bag-e7114e489fb3c9db", "spice bag2-1491417e8ca26a8a", "spice bag3-d29a9d7dd883e21f", "plastic bag of cashews-2db3ce7eda53883f", "bag5-1c51fe6be09b68b7", "tablespoon3-5af7494a04121ed7", "tablespoon4-517f87a292530bc5", "notebook-01551372823e1b54", "cap of pen-786e694857ab4581", "garlic clove-c8b083de7080c19d" ], "reasoning": "1. I saw a container (container3-f21d7e45117dbe19) with bell pepper halves , a green pepper (pepper-d3962cd68f38a74c), a mushroom , and a lemon (lemon-6c197df4cd691e46) at 03:11, after the lid (lid-6dd3409ce3807d6c) is removed from the container.\n2. This is the container spoken of in the question, since it contains vegetables, fruit, and fungi.\n3. The user took a pepper (pepper-d3962cd68f38a74c) out first and placed it from 03:13-03:14.\n4. The user took a lemon (lemon-6c197df4cd691e46) second and placed it from 03:15-03:16.\n5. I notice that the lemon (lemon-6c197df4cd691e46) is slightly further away from the user than the pepper (pepper-d3962cd68f38a74c).\n6. Therefore, the answer is lemon.", "referenced_object_masks": [ "container3-f21d7e45117dbe19", "lid-6dd3409ce3807d6c", "pepper-d3962cd68f38a74c", "lemon-6c197df4cd691e46" ] }, { "key": "P04-20240415-175615:d3e94ff2c052a734f74e59726b1615935153579f", "video_id": "P04-20240415-175615", "question": "If the first container I put back in the fridge were placed one shelf lower, but in the same location on that shelf, replacing whatever would be in that spot, what color would the food to its immediate right be?", "answer_choice_0": "Red", "answer_choice_1": "Green", "answer_choice_2": "Yellow", "answer_choice_3": "White", "answer_choice_4": "Orange", "answer_id": 1, "answer": "Green", "question_type": [ "Egocentric & Spatial Perception", "Event Occurence", "Counterfactual", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "towel-bb81734dcb0fefb7", "chopping board-2db62a7cdb7e1e33", "knife-4100c0833d8ed206", "tablespoon-33190f152b8c6d64", "tablespoon2-f5edc00e9306a9c6", "bottle of olive oil-e121f234af8bb2d4", "saucepan-eabf1412611fd752", "bag-d6ff78597c31ca9c", "metallic tumbler-ff33bd04cbcaa47d", "metalic cup-8d0d2f50af8d900f", "weighing scale-f8aeea8253d12ae4", "plastic container-4c9984bc3c9d79ee", "bottle of oil2-4a40a6d886b67d72", "garam masala powder packet-4a62e4a1a13a8356", "rice bag-88c30c4db5abd9a0", "garlic-f2e741177dbd9fa4", "ginger-61c0b2e6a3e58909", "onion-c7b2ceecdf5af7f1", "chili-73caa3d78fe83dde", "container2-a70c3da20660430b", "container3-f21d7e45117dbe19", "lid-6dd3409ce3807d6c", "pepper-d3962cd68f38a74c", "lemon-6c197df4cd691e46", "pen-0c622a7de1f23000", "plate-c3e9b6d83f8a8504", "black plastic container box-d1227ba8a2f0c56a", "lid2-7238e4b5987c4ef9", "plate2-4a49834ce6d0e936", "spice bag-e7114e489fb3c9db", "spice bag2-1491417e8ca26a8a", "spice bag3-d29a9d7dd883e21f", "plastic bag of cashews-2db3ce7eda53883f", "bag5-1c51fe6be09b68b7", "tablespoon3-5af7494a04121ed7", "tablespoon4-517f87a292530bc5", "notebook-01551372823e1b54", "cap of pen-786e694857ab4581", "garlic clove-c8b083de7080c19d" ], "reasoning": "1. I watch the video to determine when the user places something back into the fridge that they previously took out. To do so, I first note the objects that the user takes out of the fridge before putting something back in - I watch the user take out a plastic packet of food (bag-d6ff78597c31ca9c) from 01:27-01:34, and then, from 02:46-03:09, I watch the user take out a chili pepper (chili-73caa3d78fe83dde), a tupperware container (container2-a70c3da20660430b), and a second, larger tupperware (container3-f21d7e45117dbe19).\n2. From 03:21-03:24, I observe the user place the second tupperware (container3-f21d7e45117dbe19) back into the fridge. I observe the location that the user places it back into - the top left of 2 shelves - and counterfactually imagine that it was placed 1 shelf lower.\n3. I observe the food on the second shelf, and note a bag of green food on the middle second shelf and a tupperware with a brown piece of food to its left. Putting the first container (container3-f21d7e45117dbe19) 1 shelf down would replace the tupperware with the brown piece of food and locate it immediately to the left of the bag of green food .\n4. I therefore note that the food to the right of the proposed location of the tupperware (container3-f21d7e45117dbe19) would be green, making C the correct answer.", "referenced_object_masks": [ "container3-f21d7e45117dbe19", "bag-d6ff78597c31ca9c", "chili-73caa3d78fe83dde", "container2-a70c3da20660430b" ] }, { "key": "P04-20240415-175615:d5021ba990cf3c88ecedd73b7ab686853eb7cfda", "video_id": "P04-20240415-175615", "question": "What is the product of the number of dry spices I place on the plate before the cashews, and the number of ginger chunks I slide onto the wooden cutting board?", "answer_choice_0": "36", "answer_choice_1": "27", "answer_choice_2": "33", "answer_choice_3": "44", "answer_choice_4": "52", "answer_id": 3, "answer": "44", "question_type": [ "Counting", "Numerical Reasoning", "Object Recognition", "Event Occurence", "Action Recognition" ], "objects_of_interest": [ "towel-bb81734dcb0fefb7", "chopping board-2db62a7cdb7e1e33", "knife-4100c0833d8ed206", "tablespoon-33190f152b8c6d64", "tablespoon2-f5edc00e9306a9c6", "bottle of olive oil-e121f234af8bb2d4", "saucepan-eabf1412611fd752", "bag-d6ff78597c31ca9c", "metallic tumbler-ff33bd04cbcaa47d", "metalic cup-8d0d2f50af8d900f", "weighing scale-f8aeea8253d12ae4", "plastic container-4c9984bc3c9d79ee", "bottle of oil2-4a40a6d886b67d72", "garam masala powder packet-4a62e4a1a13a8356", "rice bag-88c30c4db5abd9a0", "garlic-f2e741177dbd9fa4", "ginger-61c0b2e6a3e58909", "onion-c7b2ceecdf5af7f1", "chili-73caa3d78fe83dde", "container2-a70c3da20660430b", "container3-f21d7e45117dbe19", "lid-6dd3409ce3807d6c", "pepper-d3962cd68f38a74c", "lemon-6c197df4cd691e46", "pen-0c622a7de1f23000", "plate-c3e9b6d83f8a8504", "black plastic container box-d1227ba8a2f0c56a", "lid2-7238e4b5987c4ef9", "plate2-4a49834ce6d0e936", "spice bag-e7114e489fb3c9db", "spice bag2-1491417e8ca26a8a", "spice bag3-d29a9d7dd883e21f", "plastic bag of cashews-2db3ce7eda53883f", "bag5-1c51fe6be09b68b7", "tablespoon3-5af7494a04121ed7", "tablespoon4-517f87a292530bc5", "notebook-01551372823e1b54", "cap of pen-786e694857ab4581", "garlic clove-c8b083de7080c19d" ], "reasoning": "1. I first find the point in the video where the user is placing dry spices on a plate . This begins at 04:54, when the user removes a large, dry leaf from the first spice bag (spice bag-e7114e489fb3c9db) and puts it on a plate . I count this as 1 dry spice.\n2. Next, I watch the user place a long, thin, brown dry spice from the second spice bag (spice bag2-1491417e8ca26a8a) onto the plate at 05:25. I now count 2 dry spices on the plate.\n3. Next, I watch the user place 4 cloves in quick succession onto the plate from the third spice bag (spice bag3-d29a9d7dd883e21f) from 05:32-05:44. I now count a total of 6 dry spices on the plate.\n4. Next, I watch the user place 5 more dry spices from the fourth spice bag onto the plate from 05:57-06:26. I now count 11 dry spices on the plate.\n5. At 06:40, the user begins placing cashews from the cashew bag (plastic bag of cashews-2db3ce7eda53883f) onto the plate , so I stop counting spices at a total of 11.\n6. Next, I observe when the user is chopping ginger (ginger-61c0b2e6a3e58909), which occurs from 10:20-12:09. I observe the user stop chopping, and slide 4 large chunks of ginger onto the wooden cutting board, at 12:10. Thus, I use 4 for the next part of the equation.\n7. To answer the question, I multiply the 11 dry spices by the 4 ginger chunks, to get a product of 44, making the answer E.", "referenced_object_masks": [ "spice bag3-d29a9d7dd883e21f", "spice bag2-1491417e8ca26a8a", "spice bag-e7114e489fb3c9db", "ginger-61c0b2e6a3e58909", "plastic bag of cashews-2db3ce7eda53883f" ] }, { "key": "P04-20240415-175615:e3decf9459aa99042953b7b77d47b31530090d4c", "video_id": "P04-20240415-175615", "question": "What are the fingers of my left hand doing, and what surface do they touch between 03:27 and 03:30?", "answer_choice_0": "Wiping linoleum", "answer_choice_1": "Rolling metal", "answer_choice_2": "Tapping wood", "answer_choice_3": "Gripping plastic", "answer_choice_4": "Scrubbing granite", "answer_id": 2, "answer": "Tapping wood", "question_type": [ "Action Recognition", "Situational Awareness", "Temporal Reasoning", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "towel-bb81734dcb0fefb7", "chopping board-2db62a7cdb7e1e33", "knife-4100c0833d8ed206", "tablespoon-33190f152b8c6d64", "tablespoon2-f5edc00e9306a9c6", "bottle of olive oil-e121f234af8bb2d4", "saucepan-eabf1412611fd752", "bag-d6ff78597c31ca9c", "metallic tumbler-ff33bd04cbcaa47d", "metalic cup-8d0d2f50af8d900f", "weighing scale-f8aeea8253d12ae4", "plastic container-4c9984bc3c9d79ee", "bottle of oil2-4a40a6d886b67d72", "garam masala powder packet-4a62e4a1a13a8356", "rice bag-88c30c4db5abd9a0", "garlic-f2e741177dbd9fa4", "ginger-61c0b2e6a3e58909", "onion-c7b2ceecdf5af7f1", "chili-73caa3d78fe83dde", "container2-a70c3da20660430b", "container3-f21d7e45117dbe19", "lid-6dd3409ce3807d6c", "pepper-d3962cd68f38a74c", "lemon-6c197df4cd691e46", "pen-0c622a7de1f23000", "plate-c3e9b6d83f8a8504", "black plastic container box-d1227ba8a2f0c56a", "lid2-7238e4b5987c4ef9", "plate2-4a49834ce6d0e936", "spice bag-e7114e489fb3c9db", "spice bag2-1491417e8ca26a8a", "spice bag3-d29a9d7dd883e21f", "plastic bag of cashews-2db3ce7eda53883f", "bag5-1c51fe6be09b68b7", "tablespoon3-5af7494a04121ed7", "tablespoon4-517f87a292530bc5", "notebook-01551372823e1b54", "cap of pen-786e694857ab4581", "garlic clove-c8b083de7080c19d" ], "reasoning": "1. I found the beginning of the demarcated time stamp at 03:27, and identified the left hand in the video.\n2. Both hands are on different surfaces, with the right hand placed between the notebook (notebook-01551372823e1b54) and the scale (weighing scale-f8aeea8253d12ae4) on the hard surface of the counter, and the left hand placed on the wooden slab, with both hands tapping their fingers until 03:30.", "referenced_object_masks": [ "weighing scale-f8aeea8253d12ae4", "notebook-01551372823e1b54" ] }, { "key": "P04-20240415-175615:ecaf1fff23e3a2027c17539c4153d5bc2d585e72", "video_id": "P04-20240415-175615", "question": "If the soap I washed my hands with was sitting at the opposite diagonal corner of the sink, which of the following items would it be closest to?", "answer_choice_0": "The ceramic mugs", "answer_choice_1": "The spray bottle", "answer_choice_2": "The rectangular carton", "answer_choice_3": "The sponge", "answer_choice_4": "The metal bottle", "answer_id": 2, "answer": "The rectangular carton", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Counterfactual", "Situational Awareness" ], "objects_of_interest": [ "towel-bb81734dcb0fefb7", "chopping board-2db62a7cdb7e1e33", "knife-4100c0833d8ed206", "tablespoon-33190f152b8c6d64", "tablespoon2-f5edc00e9306a9c6", "bottle of olive oil-e121f234af8bb2d4", "saucepan-eabf1412611fd752", "bag-d6ff78597c31ca9c", "metallic tumbler-ff33bd04cbcaa47d", "metalic cup-8d0d2f50af8d900f", "weighing scale-f8aeea8253d12ae4", "plastic container-4c9984bc3c9d79ee", "bottle of oil2-4a40a6d886b67d72", "garam masala powder packet-4a62e4a1a13a8356", "rice bag-88c30c4db5abd9a0", "garlic-f2e741177dbd9fa4", "ginger-61c0b2e6a3e58909", "onion-c7b2ceecdf5af7f1", "chili-73caa3d78fe83dde", "container2-a70c3da20660430b", "container3-f21d7e45117dbe19", "lid-6dd3409ce3807d6c", "pepper-d3962cd68f38a74c", "lemon-6c197df4cd691e46", "pen-0c622a7de1f23000", "plate-c3e9b6d83f8a8504", "black plastic container box-d1227ba8a2f0c56a", "lid2-7238e4b5987c4ef9", "plate2-4a49834ce6d0e936", "spice bag-e7114e489fb3c9db", "spice bag2-1491417e8ca26a8a", "spice bag3-d29a9d7dd883e21f", "plastic bag of cashews-2db3ce7eda53883f", "bag5-1c51fe6be09b68b7", "tablespoon3-5af7494a04121ed7", "tablespoon4-517f87a292530bc5", "notebook-01551372823e1b54", "cap of pen-786e694857ab4581", "garlic clove-c8b083de7080c19d" ], "reasoning": "1. I observe the user washing their hands at 00:09, and note the location of the dish soap they use, which is located at the back left corner of the sink.\n2. I imagine that the dish soap is in the opposite diagonal corner of its true location, which would place it in the front right corner of the sink.\n3. I note the location of many objects in and around the sink. These items include two mugs in the left sink basin, a spray bottle in the back left corner of the sink to the direct right of the true location of the dish soap , a spray bottle with yellow liquid on the left side of the windowsill, and a metal bottle on the right side of the windowsill. A dish drying tray is stretched over the right sink basin, and on top of it, there is a yellow sponge , and a white basket with a blue bottle in it . A rectangular carton of liquid sits in the bottom right corner of the sink.\n4. I compare the objects I observed in step 3 and their locations to the answer choices, and observe that the rectangular carton would be the closest to the dish soap in its new theoretical location, making E the correct answer.", "referenced_object_masks": [] }, { "key": "P04-20240415-175615:fabab51ee4a92a853a8ea32bee0f43084cf60ff4", "video_id": "P04-20240415-175615", "question": "If I added double to the original amount of masala powder that was placed on the plate on the scale, what would be the weight of the readout?", "answer_choice_0": "3", "answer_choice_1": "8", "answer_choice_2": "12", "answer_choice_3": "6", "answer_choice_4": "4", "answer_id": 3, "answer": "6", "question_type": [ "Object Recognition", "Counterfactual", "Reading", "Numerical Reasoning" ], "objects_of_interest": [ "towel-bb81734dcb0fefb7", "chopping board-2db62a7cdb7e1e33", "knife-4100c0833d8ed206", "tablespoon-33190f152b8c6d64", "tablespoon2-f5edc00e9306a9c6", "bottle of olive oil-e121f234af8bb2d4", "saucepan-eabf1412611fd752", "bag-d6ff78597c31ca9c", "metallic tumbler-ff33bd04cbcaa47d", "metalic cup-8d0d2f50af8d900f", "weighing scale-f8aeea8253d12ae4", "plastic container-4c9984bc3c9d79ee", "bottle of oil2-4a40a6d886b67d72", "garam masala powder packet-4a62e4a1a13a8356", "rice bag-88c30c4db5abd9a0", "garlic-f2e741177dbd9fa4", "ginger-61c0b2e6a3e58909", "onion-c7b2ceecdf5af7f1", "chili-73caa3d78fe83dde", "container2-a70c3da20660430b", "container3-f21d7e45117dbe19", "lid-6dd3409ce3807d6c", "pepper-d3962cd68f38a74c", "lemon-6c197df4cd691e46", "pen-0c622a7de1f23000", "plate-c3e9b6d83f8a8504", "black plastic container box-d1227ba8a2f0c56a", "lid2-7238e4b5987c4ef9", "plate2-4a49834ce6d0e936", "spice bag-e7114e489fb3c9db", "spice bag2-1491417e8ca26a8a", "spice bag3-d29a9d7dd883e21f", "plastic bag of cashews-2db3ce7eda53883f", "bag5-1c51fe6be09b68b7", "tablespoon3-5af7494a04121ed7", "tablespoon4-517f87a292530bc5", "notebook-01551372823e1b54", "cap of pen-786e694857ab4581", "garlic clove-c8b083de7080c19d" ], "reasoning": "1. I found the timestamp at 08:38 where the user began to pour masala powder (garam masala powder packet-4a62e4a1a13a8356) onto a plate (plate-c3e9b6d83f8a8504) placed on a scale (weighing scale-f8aeea8253d12ae4).\n2. As the user finishes pouring, we see that the readout of the scale (weighing scale-f8aeea8253d12ae4) says \"3\".\n3. If we multiply the number from the readout by two in order to give us double, the readout would say \"6\".", "referenced_object_masks": [ "weighing scale-f8aeea8253d12ae4", "garam masala powder packet-4a62e4a1a13a8356", "plate-c3e9b6d83f8a8504" ] }, { "key": "P04-20240415-181212:06e03f7b2485210b502547ddc1c1456afd26f0a0", "video_id": "P04-20240415-181212", "question": "In terms of degrees, in what approximate position, moving clockwise, is the black water cooler from where I stood to measure rice?", "answer_choice_0": "90 degrees", "answer_choice_1": "130 degrees", "answer_choice_2": "45 degrees", "answer_choice_3": "270 degrees", "answer_choice_4": "180 degrees", "answer_id": 1, "answer": "130 degrees", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "pen-2197924ad0b25f75", "pen cap-b725d3eaaf76a803", "plate1-b1f7a8242bdd7994", "pestle-83de0b35c6c6062e", "mortar-a26024ec3cc0fab1", "purple cloth-be0017ea6c5417df", "knife1-1f86de904a928d34", "spoon1-0a4cbb05816be8c9", "chopping board-9f174495248e5157", "plate2-934b976353f45629", "cup1-b2e7119c74cb976d", "bowl1-798fd9872fd4f94a", "bag of rice-858e02e9eb9418df", "measuring jug-94128b9a8974109d", "plastic measuring jug-0bc063d5332bba06", "water cooler's drip tray-be4df49e15d9ed50", "pressure cooker-8dd279a2bed14952", "hot plate-a62d3efb6a026f66", "lid of pressure cooker-38ab35a6a30b86d5", "bottle of oil-d008e55c6eb26983", "cap of oil-9bd98807c70d897c", "spoon2-40b5bc6f06118f8a", "packet of ghee-f663817c38022db1", "parchment paper-b25a600ca24ad160", "wooden spatula-335cb2965abbc23b", "plug of hot plate-a2c6a8cd81314d65", "notebook-bdd55a1897ad0ccd", "bowl2-160a6dc2bb0e083d", "plastic box-569e210d84423050", "lid of plastic box-2579c75b2ad15c5a", "jar of salt-bc15930a9c311300", "lid of salt jar-c423d8108fbf7477", "spoon3-5fb95fa38cfcc10e", "half of lemon-5b167c1ccdc0ed7b", "lid of plastic box2-c30af3b8bbf16219", "packet1-6a29896e9978f395", "packet2-3110b451969b6284", "head of garlic-a4721bea37a8b83a", "weighing scales-da539e28c2386599" ], "reasoning": "1. I watched the video in order to figure out where the user is located as they measured rice, and then determined where the black water cooler was located in relation to them at that time.\n2. First, I noticed the user opened the bag of rice (bag of rice-858e02e9eb9418df) at 04:55.\n3. Then, I saw the user scooped rice and reached into the green bag (bag of rice-858e02e9eb9418df) at 04:57. They then measured the rice on the measuring scale (weighing scales-da539e28c2386599) at 05:05.\n4. I took note of the user's place, which is to the left of the stove top .\n5. Then, I looked for the water cooler . At 05:53, I noticed the user turned from standing in front of the wooden counter to the right, and then walked forward toward the water cooler .\n6. Thinking of the original location where the user scooped rice , which was to the left of the wooden counter, I determined the user would need to turn clockwise approximately 130 degrees in order to find the water cooler .", "referenced_object_masks": [ "weighing scales-da539e28c2386599", "bag of rice-858e02e9eb9418df" ] }, { "key": "P04-20240415-181212:163b97b71abacd50ef56706dfb3d9dd9cbe9f373", "video_id": "P04-20240415-181212", "question": "What is the last item placed in the sink and where was it located when originally I picked it up?", "answer_choice_0": "Cutting board, Left of hot plate", "answer_choice_1": "Plate, right of hot plate", "answer_choice_2": "Measuring cup, metal rack", "answer_choice_3": "Cutting board, black countertop", "answer_choice_4": "Plate, left of hot plate", "answer_id": 0, "answer": "Cutting board, Left of hot plate", "question_type": [ "Action Recognition", "State Changes", "Situational Awareness", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "pen-2197924ad0b25f75", "pen cap-b725d3eaaf76a803", "plate1-b1f7a8242bdd7994", "pestle-83de0b35c6c6062e", "mortar-a26024ec3cc0fab1", "purple cloth-be0017ea6c5417df", "knife1-1f86de904a928d34", "spoon1-0a4cbb05816be8c9", "chopping board-9f174495248e5157", "plate2-934b976353f45629", "cup1-b2e7119c74cb976d", "bowl1-798fd9872fd4f94a", "bag of rice-858e02e9eb9418df", "measuring jug-94128b9a8974109d", "plastic measuring jug-0bc063d5332bba06", "water cooler's drip tray-be4df49e15d9ed50", "pressure cooker-8dd279a2bed14952", "hot plate-a62d3efb6a026f66", "lid of pressure cooker-38ab35a6a30b86d5", "bottle of oil-d008e55c6eb26983", "cap of oil-9bd98807c70d897c", "spoon2-40b5bc6f06118f8a", "packet of ghee-f663817c38022db1", "parchment paper-b25a600ca24ad160", "wooden spatula-335cb2965abbc23b", "plug of hot plate-a2c6a8cd81314d65", "notebook-bdd55a1897ad0ccd", "bowl2-160a6dc2bb0e083d", "plastic box-569e210d84423050", "lid of plastic box-2579c75b2ad15c5a", "jar of salt-bc15930a9c311300", "lid of salt jar-c423d8108fbf7477", "spoon3-5fb95fa38cfcc10e", "half of lemon-5b167c1ccdc0ed7b", "lid of plastic box2-c30af3b8bbf16219", "packet1-6a29896e9978f395", "packet2-3110b451969b6284", "head of garlic-a4721bea37a8b83a", "weighing scales-da539e28c2386599" ], "reasoning": "1. First, I looked for the final item in the video to be placed inside the sink .\n2. I noticed the user grabbed the cutting board (chopping board-9f174495248e5157) and placed it into the sink at 26:05 and it was the last item placed, as no other items are placed into the sink after this point.\n3. Then, I looked to find out where the cutting board (chopping board-9f174495248e5157) was originally located. I found it immediately at the start of the video, at 00:00, to the left of the hot plate (hot plate-a62d3efb6a026f66) and on top of the wooden counter .\n4. Putting this all together to figure out the final answer, I determined that the final item to placed in the sink was the cutting board (chopping board-9f174495248e5157), and it was originally located to the left of the hot plate (hot plate-a62d3efb6a026f66).", "referenced_object_masks": [ "chopping board-9f174495248e5157", "hot plate-a62d3efb6a026f66" ] }, { "key": "P04-20240415-181212:2d0146bdc508dc84606b0c53e6db5a5e2fd4a6f4", "video_id": "P04-20240415-181212", "question": "What is the third vessel that I put rice into?", "answer_choice_0": "The white bowl", "answer_choice_1": "The bag of rice", "answer_choice_2": "The small silver cup", "answer_choice_3": "The saucepan", "answer_choice_4": "The glass measuring cup", "answer_id": 1, "answer": "The bag of rice", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Numerical Reasoning" ], "objects_of_interest": [ "pen-2197924ad0b25f75", "pen cap-b725d3eaaf76a803", "plate1-b1f7a8242bdd7994", "pestle-83de0b35c6c6062e", "mortar-a26024ec3cc0fab1", "purple cloth-be0017ea6c5417df", "knife1-1f86de904a928d34", "spoon1-0a4cbb05816be8c9", "chopping board-9f174495248e5157", "plate2-934b976353f45629", "cup1-b2e7119c74cb976d", "bowl1-798fd9872fd4f94a", "bag of rice-858e02e9eb9418df", "measuring jug-94128b9a8974109d", "plastic measuring jug-0bc063d5332bba06", "water cooler's drip tray-be4df49e15d9ed50", "pressure cooker-8dd279a2bed14952", "hot plate-a62d3efb6a026f66", "lid of pressure cooker-38ab35a6a30b86d5", "bottle of oil-d008e55c6eb26983", "cap of oil-9bd98807c70d897c", "spoon2-40b5bc6f06118f8a", "packet of ghee-f663817c38022db1", "parchment paper-b25a600ca24ad160", "wooden spatula-335cb2965abbc23b", "plug of hot plate-a2c6a8cd81314d65", "notebook-bdd55a1897ad0ccd", "bowl2-160a6dc2bb0e083d", "plastic box-569e210d84423050", "lid of plastic box-2579c75b2ad15c5a", "jar of salt-bc15930a9c311300", "lid of salt jar-c423d8108fbf7477", "spoon3-5fb95fa38cfcc10e", "half of lemon-5b167c1ccdc0ed7b", "lid of plastic box2-c30af3b8bbf16219", "packet1-6a29896e9978f395", "packet2-3110b451969b6284", "head of garlic-a4721bea37a8b83a", "weighing scales-da539e28c2386599" ], "reasoning": "1. I watched for the user to pick up the bag of rice (bag of rice-858e02e9eb9418df), which they do at 4:51. Since the rice is already in the bag, this does not count as one of the vessels.\n2. At 4:30, the user had picked up a small silver cup (bag of rice-858e02e9eb9418df), which they then use to scoop rice from the bag (bag of rice-858e02e9eb9418df) at 4:57. This is the first separate vessel.\n3. At 4:34, the user had poured the rice into a white bowl (bowl1-798fd9872fd4f94a) retrieved from the cupboard, and pours rice from the small silver cup into it at 5:05. This counts as the second vessel.\n4. At 5:27, the user pours some of the scooped rice back into the bag of rice (bag of rice-858e02e9eb9418df), meaning that it counts as the third vessel.", "referenced_object_masks": [ "bowl1-798fd9872fd4f94a", "bag of rice-858e02e9eb9418df" ] }, { "key": "P04-20240415-181212:2e50c7d340adaebfee459cae7ef2b24bb23ba5f9", "video_id": "P04-20240415-181212", "question": "How many times do I scrape the spoon on the pestle after mixing the food in the mortar with the same spoon?", "answer_choice_0": "6 times", "answer_choice_1": "8 times", "answer_choice_2": "5 times", "answer_choice_3": "7 times", "answer_choice_4": "9 times", "answer_id": 3, "answer": "7 times", "question_type": [ "Action Recognition", "Counting", "Object Recognition" ], "objects_of_interest": [ "pen-2197924ad0b25f75", "pen cap-b725d3eaaf76a803", "plate1-b1f7a8242bdd7994", "pestle-83de0b35c6c6062e", "mortar-a26024ec3cc0fab1", "purple cloth-be0017ea6c5417df", "knife1-1f86de904a928d34", "spoon1-0a4cbb05816be8c9", "chopping board-9f174495248e5157", "plate2-934b976353f45629", "cup1-b2e7119c74cb976d", "bowl1-798fd9872fd4f94a", "bag of rice-858e02e9eb9418df", "measuring jug-94128b9a8974109d", "plastic measuring jug-0bc063d5332bba06", "water cooler's drip tray-be4df49e15d9ed50", "pressure cooker-8dd279a2bed14952", "hot plate-a62d3efb6a026f66", "lid of pressure cooker-38ab35a6a30b86d5", "bottle of oil-d008e55c6eb26983", "cap of oil-9bd98807c70d897c", "spoon2-40b5bc6f06118f8a", "packet of ghee-f663817c38022db1", "parchment paper-b25a600ca24ad160", "wooden spatula-335cb2965abbc23b", "plug of hot plate-a2c6a8cd81314d65", "notebook-bdd55a1897ad0ccd", "bowl2-160a6dc2bb0e083d", "plastic box-569e210d84423050", "lid of plastic box-2579c75b2ad15c5a", "jar of salt-bc15930a9c311300", "lid of salt jar-c423d8108fbf7477", "spoon3-5fb95fa38cfcc10e", "half of lemon-5b167c1ccdc0ed7b", "lid of plastic box2-c30af3b8bbf16219", "packet1-6a29896e9978f395", "packet2-3110b451969b6284", "head of garlic-a4721bea37a8b83a", "weighing scales-da539e28c2386599" ], "reasoning": "1. I noticed the user mixed food in the mortar (mortar-a26024ec3cc0fab1) at 03:58. \n2. I noticed the user brought up the spoon (spoon1-0a4cbb05816be8c9) to the pestle (pestle-83de0b35c6c6062e) at 04:00. \n3. I noticed the user clanked the spoon (spoon1-0a4cbb05816be8c9) for a total of 7 times from 04:01-04:04.", "referenced_object_masks": [ "spoon1-0a4cbb05816be8c9", "mortar-a26024ec3cc0fab1", "pestle-83de0b35c6c6062e" ] }, { "key": "P04-20240415-181212:4029818548d1a717d35cf143c1e02f441862f55d", "video_id": "P04-20240415-181212", "question": "How much time passes between when I first pick up the mortar and pestle and when I put it down for the last time?", "answer_choice_0": "15:36", "answer_choice_1": "12:49", "answer_choice_2": "18:24", "answer_choice_3": "24:37", "answer_choice_4": "2:33", "answer_id": 3, "answer": "24:37", "question_type": [ "Object Recognition", "Temporal Reasoning", "Numerical Reasoning" ], "objects_of_interest": [ "pen-2197924ad0b25f75", "pen cap-b725d3eaaf76a803", "plate1-b1f7a8242bdd7994", "pestle-83de0b35c6c6062e", "mortar-a26024ec3cc0fab1", "purple cloth-be0017ea6c5417df", "knife1-1f86de904a928d34", "spoon1-0a4cbb05816be8c9", "chopping board-9f174495248e5157", "plate2-934b976353f45629", "cup1-b2e7119c74cb976d", "bowl1-798fd9872fd4f94a", "bag of rice-858e02e9eb9418df", "measuring jug-94128b9a8974109d", "plastic measuring jug-0bc063d5332bba06", "water cooler's drip tray-be4df49e15d9ed50", "pressure cooker-8dd279a2bed14952", "hot plate-a62d3efb6a026f66", "lid of pressure cooker-38ab35a6a30b86d5", "bottle of oil-d008e55c6eb26983", "cap of oil-9bd98807c70d897c", "spoon2-40b5bc6f06118f8a", "packet of ghee-f663817c38022db1", "parchment paper-b25a600ca24ad160", "wooden spatula-335cb2965abbc23b", "plug of hot plate-a2c6a8cd81314d65", "notebook-bdd55a1897ad0ccd", "bowl2-160a6dc2bb0e083d", "plastic box-569e210d84423050", "lid of plastic box-2579c75b2ad15c5a", "jar of salt-bc15930a9c311300", "lid of salt jar-c423d8108fbf7477", "spoon3-5fb95fa38cfcc10e", "half of lemon-5b167c1ccdc0ed7b", "lid of plastic box2-c30af3b8bbf16219", "packet1-6a29896e9978f395", "packet2-3110b451969b6284", "head of garlic-a4721bea37a8b83a", "weighing scales-da539e28c2386599" ], "reasoning": "1. I watched for the user to pick up the mortar (mortar-a26024ec3cc0fab1) and pestle (pestle-83de0b35c6c6062e) for the first time, which occurs at 0:45.\n2. I watched for the user to put down the mortar (mortar-a26024ec3cc0fab1) and pestle (pestle-83de0b35c6c6062e) for a final time, which occurs at 25:22.\n3. The time between the two time stamps is 24:37.", "referenced_object_masks": [ "pestle-83de0b35c6c6062e", "mortar-a26024ec3cc0fab1" ] }, { "key": "P04-20240415-181212:7801489c025bd15b41e62ca40299bca412814c0c", "video_id": "P04-20240415-181212", "question": "How does the ratio of blue lights to red lights on the water cooler change from when it is dispensing to when it is idle?", "answer_choice_0": "2:3 to 3:2", "answer_choice_1": "3:1 to 2:5", "answer_choice_2": "3:2 to 1:1", "answer_choice_3": "2:1 to 1:1", "answer_choice_4": "1:1 to 1:3", "answer_id": 3, "answer": "2:1 to 1:1", "question_type": [ "Action Recognition", "State Changes", "Numerical Reasoning", "Object Recognition", "Counting" ], "objects_of_interest": [ "pen-2197924ad0b25f75", "pen cap-b725d3eaaf76a803", "plate1-b1f7a8242bdd7994", "pestle-83de0b35c6c6062e", "mortar-a26024ec3cc0fab1", "purple cloth-be0017ea6c5417df", "knife1-1f86de904a928d34", "spoon1-0a4cbb05816be8c9", "chopping board-9f174495248e5157", "plate2-934b976353f45629", "cup1-b2e7119c74cb976d", "bowl1-798fd9872fd4f94a", "bag of rice-858e02e9eb9418df", "measuring jug-94128b9a8974109d", "plastic measuring jug-0bc063d5332bba06", "water cooler's drip tray-be4df49e15d9ed50", "pressure cooker-8dd279a2bed14952", "hot plate-a62d3efb6a026f66", "lid of pressure cooker-38ab35a6a30b86d5", "bottle of oil-d008e55c6eb26983", "cap of oil-9bd98807c70d897c", "spoon2-40b5bc6f06118f8a", "packet of ghee-f663817c38022db1", "parchment paper-b25a600ca24ad160", "wooden spatula-335cb2965abbc23b", "plug of hot plate-a2c6a8cd81314d65", "notebook-bdd55a1897ad0ccd", "bowl2-160a6dc2bb0e083d", "plastic box-569e210d84423050", "lid of plastic box-2579c75b2ad15c5a", "jar of salt-bc15930a9c311300", "lid of salt jar-c423d8108fbf7477", "spoon3-5fb95fa38cfcc10e", "half of lemon-5b167c1ccdc0ed7b", "lid of plastic box2-c30af3b8bbf16219", "packet1-6a29896e9978f395", "packet2-3110b451969b6284", "head of garlic-a4721bea37a8b83a", "weighing scales-da539e28c2386599" ], "reasoning": "1. I watched for the user to use the water cooler , which they do at 6:07, pressing the button to dispense water. \n2. I noted that when the water is dispensing, there are 4 blue lights and 2 red lights lit on the face of the water cooler .\n3. At 6:12, the user turns the dispenser off, and there are 2 blue lights and 2 red lights lit on the face of the water cooler .\n4. The ratio while the water cooler is dispensing is 2:1, and while it is idle, 1:1.", "referenced_object_masks": [] }, { "key": "P04-20240415-181212:8b419972ced1ffa5e5029db738812a04ec0db81a", "video_id": "P04-20240415-181212", "question": "What would I need to do to get from standing and facing the yellow dishtowel to standing and facing the purple dishtowel in the first minute?", "answer_choice_0": "Turn left, walk three steps", "answer_choice_1": "Turn right, walk a step, turn left", "answer_choice_2": "Turn 45 degrees left, walk past the sink", "answer_choice_3": "Turn right, walk to the sink, 45 degrees left", "answer_choice_4": "Turn 180 degrees right", "answer_id": 3, "answer": "Turn right, walk to the sink, 45 degrees left", "question_type": [ "Egocentric & Spatial Perception", "Situational Awareness", "Object Recognition" ], "objects_of_interest": [ "pen-2197924ad0b25f75", "pen cap-b725d3eaaf76a803", "plate1-b1f7a8242bdd7994", "pestle-83de0b35c6c6062e", "mortar-a26024ec3cc0fab1", "purple cloth-be0017ea6c5417df", "knife1-1f86de904a928d34", "spoon1-0a4cbb05816be8c9", "chopping board-9f174495248e5157", "plate2-934b976353f45629", "cup1-b2e7119c74cb976d", "bowl1-798fd9872fd4f94a", "bag of rice-858e02e9eb9418df", "measuring jug-94128b9a8974109d", "plastic measuring jug-0bc063d5332bba06", "water cooler's drip tray-be4df49e15d9ed50", "pressure cooker-8dd279a2bed14952", "hot plate-a62d3efb6a026f66", "lid of pressure cooker-38ab35a6a30b86d5", "bottle of oil-d008e55c6eb26983", "cap of oil-9bd98807c70d897c", "spoon2-40b5bc6f06118f8a", "packet of ghee-f663817c38022db1", "parchment paper-b25a600ca24ad160", "wooden spatula-335cb2965abbc23b", "plug of hot plate-a2c6a8cd81314d65", "notebook-bdd55a1897ad0ccd", "bowl2-160a6dc2bb0e083d", "plastic box-569e210d84423050", "lid of plastic box-2579c75b2ad15c5a", "jar of salt-bc15930a9c311300", "lid of salt jar-c423d8108fbf7477", "spoon3-5fb95fa38cfcc10e", "half of lemon-5b167c1ccdc0ed7b", "lid of plastic box2-c30af3b8bbf16219", "packet1-6a29896e9978f395", "packet2-3110b451969b6284", "head of garlic-a4721bea37a8b83a", "weighing scales-da539e28c2386599" ], "reasoning": "1. I looked for the yellow dishtowel , which appears at the far left of the counter at 0:02.\n2. I looked for the purple dishtowel (purple cloth-be0017ea6c5417df), which can be seen in the bend of the corner to the left of the sink at 0:33.\n3. To go from facing the yellow towel to facing the purple towel (purple cloth-be0017ea6c5417df), the user would need to turn right, walk to the sink, and turn 45 degrees to the left.", "referenced_object_masks": [ "purple cloth-be0017ea6c5417df" ] }, { "key": "P04-20240415-181212:be0e1d164a3ef0923a2dcff310af5be0f0ab81c0", "video_id": "P04-20240415-181212", "question": "What is the temperature on the left side of the convection oven after I squeeze lemon juice into a spoon?", "answer_choice_0": "450", "answer_choice_1": "300", "answer_choice_2": "350", "answer_choice_3": "400", "answer_choice_4": "250", "answer_id": 3, "answer": "400", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Reading", "Object Recognition" ], "objects_of_interest": [ "pen-2197924ad0b25f75", "pen cap-b725d3eaaf76a803", "plate1-b1f7a8242bdd7994", "pestle-83de0b35c6c6062e", "mortar-a26024ec3cc0fab1", "purple cloth-be0017ea6c5417df", "knife1-1f86de904a928d34", "spoon1-0a4cbb05816be8c9", "chopping board-9f174495248e5157", "plate2-934b976353f45629", "cup1-b2e7119c74cb976d", "bowl1-798fd9872fd4f94a", "bag of rice-858e02e9eb9418df", "measuring jug-94128b9a8974109d", "plastic measuring jug-0bc063d5332bba06", "water cooler's drip tray-be4df49e15d9ed50", "pressure cooker-8dd279a2bed14952", "hot plate-a62d3efb6a026f66", "lid of pressure cooker-38ab35a6a30b86d5", "bottle of oil-d008e55c6eb26983", "cap of oil-9bd98807c70d897c", "spoon2-40b5bc6f06118f8a", "packet of ghee-f663817c38022db1", "parchment paper-b25a600ca24ad160", "wooden spatula-335cb2965abbc23b", "plug of hot plate-a2c6a8cd81314d65", "notebook-bdd55a1897ad0ccd", "bowl2-160a6dc2bb0e083d", "plastic box-569e210d84423050", "lid of plastic box-2579c75b2ad15c5a", "jar of salt-bc15930a9c311300", "lid of salt jar-c423d8108fbf7477", "spoon3-5fb95fa38cfcc10e", "half of lemon-5b167c1ccdc0ed7b", "lid of plastic box2-c30af3b8bbf16219", "packet1-6a29896e9978f395", "packet2-3110b451969b6284", "head of garlic-a4721bea37a8b83a", "weighing scales-da539e28c2386599" ], "reasoning": "1. I watched for the user to pick up the half of a lemon (half of lemon-5b167c1ccdc0ed7b), which occurs at 20:40.\n2. The user then proceeds to squeeze lemon juice into a spoon at 20:44.\n3. I watched for the next clear view of the left side of the convection oven, which occurs at 20:50.\n4. The temperature on the left side of the convection oven at 20:50 is 400.", "referenced_object_masks": [ "half of lemon-5b167c1ccdc0ed7b" ] }, { "key": "P04-20240415-181212:c13d7cc5383248608bc3b93b6d991772ed1cde22", "video_id": "P04-20240415-181212", "question": "How many times do I brush the seasonings into the pressure cooker from the white plate for the first time?", "answer_choice_0": "9 times", "answer_choice_1": "10 times", "answer_choice_2": "12 times", "answer_choice_3": "8 times", "answer_choice_4": "11 times", "answer_id": 4, "answer": "11 times", "question_type": [ "Action Recognition", "Counting", "Object Recognition", "State Changes" ], "objects_of_interest": [ "pen-2197924ad0b25f75", "pen cap-b725d3eaaf76a803", "plate1-b1f7a8242bdd7994", "pestle-83de0b35c6c6062e", "mortar-a26024ec3cc0fab1", "purple cloth-be0017ea6c5417df", "knife1-1f86de904a928d34", "spoon1-0a4cbb05816be8c9", "chopping board-9f174495248e5157", "plate2-934b976353f45629", "cup1-b2e7119c74cb976d", "bowl1-798fd9872fd4f94a", "bag of rice-858e02e9eb9418df", "measuring jug-94128b9a8974109d", "plastic measuring jug-0bc063d5332bba06", "water cooler's drip tray-be4df49e15d9ed50", "pressure cooker-8dd279a2bed14952", "hot plate-a62d3efb6a026f66", "lid of pressure cooker-38ab35a6a30b86d5", "bottle of oil-d008e55c6eb26983", "cap of oil-9bd98807c70d897c", "spoon2-40b5bc6f06118f8a", "packet of ghee-f663817c38022db1", "parchment paper-b25a600ca24ad160", "wooden spatula-335cb2965abbc23b", "plug of hot plate-a2c6a8cd81314d65", "notebook-bdd55a1897ad0ccd", "bowl2-160a6dc2bb0e083d", "plastic box-569e210d84423050", "lid of plastic box-2579c75b2ad15c5a", "jar of salt-bc15930a9c311300", "lid of salt jar-c423d8108fbf7477", "spoon3-5fb95fa38cfcc10e", "half of lemon-5b167c1ccdc0ed7b", "lid of plastic box2-c30af3b8bbf16219", "packet1-6a29896e9978f395", "packet2-3110b451969b6284", "head of garlic-a4721bea37a8b83a", "weighing scales-da539e28c2386599" ], "reasoning": "1. I watched the video and looked for a moment with seasonings on a white plate. I found this moment at 04:16, when the user picks up the plate (plate2-934b976353f45629) and brings it toward the pressure cooker (pressure cooker-8dd279a2bed14952) at 10:33.\n2. The user grabbed the wooden spatula (wooden spatula-335cb2965abbc23b) at 10:35.\n3. Then, I watched as the user brushed the seasonings into the pressure cooker (pressure cooker-8dd279a2bed14952) at 10:43, 10:44, 10:45, 10:47, 10:48, 10:50, 10:51, 10:52, 10:52 again, 10:53, and 10:53 again.\n4. Adding these up, I arrived at a total of 11 times.", "referenced_object_masks": [ "pressure cooker-8dd279a2bed14952", "wooden spatula-335cb2965abbc23b", "plate2-934b976353f45629" ] }, { "key": "P04-20240415-181212:ed06da043bed7f64375ae1eef2b000c636597bd1", "video_id": "P04-20240415-181212", "question": "After I scoop the peas 4 times, how many times do I go down the page with the pen before writing?", "answer_choice_0": "2 times", "answer_choice_1": "4 times", "answer_choice_2": "1 time", "answer_choice_3": "3 times", "answer_choice_4": "5 times", "answer_id": 2, "answer": "1 time", "question_type": [ "Action Recognition", "Counting", "Object Recognition" ], "objects_of_interest": [ "pen-2197924ad0b25f75", "pen cap-b725d3eaaf76a803", "plate1-b1f7a8242bdd7994", "pestle-83de0b35c6c6062e", "mortar-a26024ec3cc0fab1", "purple cloth-be0017ea6c5417df", "knife1-1f86de904a928d34", "spoon1-0a4cbb05816be8c9", "chopping board-9f174495248e5157", "plate2-934b976353f45629", "cup1-b2e7119c74cb976d", "bowl1-798fd9872fd4f94a", "bag of rice-858e02e9eb9418df", "measuring jug-94128b9a8974109d", "plastic measuring jug-0bc063d5332bba06", "water cooler's drip tray-be4df49e15d9ed50", "pressure cooker-8dd279a2bed14952", "hot plate-a62d3efb6a026f66", "lid of pressure cooker-38ab35a6a30b86d5", "bottle of oil-d008e55c6eb26983", "cap of oil-9bd98807c70d897c", "spoon2-40b5bc6f06118f8a", "packet of ghee-f663817c38022db1", "parchment paper-b25a600ca24ad160", "wooden spatula-335cb2965abbc23b", "plug of hot plate-a2c6a8cd81314d65", "notebook-bdd55a1897ad0ccd", "bowl2-160a6dc2bb0e083d", "plastic box-569e210d84423050", "lid of plastic box-2579c75b2ad15c5a", "jar of salt-bc15930a9c311300", "lid of salt jar-c423d8108fbf7477", "spoon3-5fb95fa38cfcc10e", "half of lemon-5b167c1ccdc0ed7b", "lid of plastic box2-c30af3b8bbf16219", "packet1-6a29896e9978f395", "packet2-3110b451969b6284", "head of garlic-a4721bea37a8b83a", "weighing scales-da539e28c2386599" ], "reasoning": "1. I watched the video and looked for a time when the user scooped peas. I found the first 3 instances of this, as the user scoops peas from a plastic box (plastic box-569e210d84423050) at 13:35, 13:39, and 13:44.\n2. I noticed the user scooped the peas for the 4th time at 13:49.\n3. Then, I noticed the user grabbed the pen (pen-2197924ad0b25f75) and placed their hand on a notebook (notebook-bdd55a1897ad0ccd) at 13:56.\n4. I noticed the user's hand going up the front page of the notebook (notebook-bdd55a1897ad0ccd) with the pen (pen-2197924ad0b25f75) at 13:57.\n5. I noticed the user's hand went back down the front page of the notebook (notebook-bdd55a1897ad0ccd) with the pen (pen-2197924ad0b25f75) at 13:57.\n6. I noticed the user's hand went back up the front page of the notebook (notebook-bdd55a1897ad0ccd) with the pen (pen-2197924ad0b25f75) and wrote at 13:59.\n7. Since the user only went down one time, I determined that to be the answer.", "referenced_object_masks": [ "notebook-bdd55a1897ad0ccd", "plastic box-569e210d84423050", "pen-2197924ad0b25f75" ] }, { "key": "P04-20240415-184003:1d1186d66d2df2df693ac02632f781a4e0652027", "video_id": "P04-20240415-184003", "question": "How many times throughout the video do I grab the purple rag to dry my hands?", "answer_choice_0": "Twice", "answer_choice_1": "Four times", "answer_choice_2": "Three times", "answer_choice_3": "Six times", "answer_choice_4": "One time", "answer_id": 1, "answer": "Four times", "question_type": [ "Object Recognition", "Action Recognition", "Counting", "Event Occurence" ], "objects_of_interest": [ "mug1-bc42b8b4245cca68", "mug2-2fed1fcc04d6657d", "glass-5f0690a1864ecddc", "carton of milk-12ec887ce8367ee0", "water cooler's drip tray-d9d3de2126debcbd", "sponge-9dbc2671c7ef8878", "measuring jug-859522538fc68034", "knife-4754276585807924", "spoon1-4e4a44c7a0a65c24", "spoon2-531733cc037ca5f3", "spoon3-6413b53a66d4a2f1", "purple cloth-bc298e9dd628f207", "metallic cup-9b7b19e462bd15eb", "chopping board-fab81e2ae0a666e4", "cooling rack-53ff57d481c4b7dc", "wooden spatula-60f3ad35616358e3", "pressure cooker-0246013723857daf", "hot plate-bdb2f36f470baabe", "plate1-90441383d9ec2ef5", "plate2-0c1ebe243bbb7415", "bowl1-08bc348b54211814", "bowl2-d6ef8746af1cfbf3", "mug3-c8797ffdf6d0c0d8", "spoon4-3a3e82e9d3746297", "sink plug-4fe2ab06f3d3aade", "spoon5-7f4bca22b3f00984", "spoon6-5678aae24053cfe5", "pestle-16c66a31b9e2f57d", "mortar-69fddd7526c01079", "regulator of pressure cooker-769d6c5b28da8f98", "lid of pressure cooker-1c8ec329ef372c28", "plate3-d06e3a2b3c05a3bd", "serving spoon-c9c9f96a634adf87", "parchment paper-b304d09d2f756172", "white spoon-58bb40a8807be721", "weighing scales-24aa1e611d7eef8c", "stack of plastic boxes-efbb3fffad4e439d", "spoon7-8824aea87e11043e", "pen-2b6413786463fa00", "cap of pen-d1abcc7c527de322", "notebook-2204dcb867bc80b7" ], "reasoning": "1. I watched for the instances the user grabbed the purple rag (purple cloth-bc298e9dd628f207) throughout the video. The first instance occurs at 03:02. \n2. The second time happens at 04:33 and lasts until 04:39. \n3. The user puts the purple rag (purple cloth-bc298e9dd628f207) down at 04:40, and doesn't grab it again until 08:52, where he dries his hands for the third time. \n4. The user puts the purple rag (purple cloth-bc298e9dd628f207) on the counter again at 09:08. The user grabs the rag (purple cloth-bc298e9dd628f207) to dry his hands for the fourth time at 13:27 and puts it back on top of the counter at 13:44. \n5. I continued to watch the video until the end to see if there was another instance but there was none. Therefore, the last time the user grabs the purple rag (purple cloth-bc298e9dd628f207) to dry his hands happened at the 13:24 mark, which is his fourth time.", "referenced_object_masks": [ "purple cloth-bc298e9dd628f207" ] }, { "key": "P04-20240415-184003:1e3980b7f9ac4933fc5d239a46789245f89aa5d0", "video_id": "P04-20240415-184003", "question": "What is the difference between the number of buttons on the stove vent and the number of electrical panels directly in front of the wooden cutting board?", "answer_choice_0": "3", "answer_choice_1": "2", "answer_choice_2": "5", "answer_choice_3": "1", "answer_choice_4": "4", "answer_id": 0, "answer": "3", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counting", "Numerical Reasoning" ], "objects_of_interest": [ "mug1-bc42b8b4245cca68", "mug2-2fed1fcc04d6657d", "glass-5f0690a1864ecddc", "carton of milk-12ec887ce8367ee0", "water cooler's drip tray-d9d3de2126debcbd", "sponge-9dbc2671c7ef8878", "measuring jug-859522538fc68034", "knife-4754276585807924", "spoon1-4e4a44c7a0a65c24", "spoon2-531733cc037ca5f3", "spoon3-6413b53a66d4a2f1", "purple cloth-bc298e9dd628f207", "metallic cup-9b7b19e462bd15eb", "chopping board-fab81e2ae0a666e4", "cooling rack-53ff57d481c4b7dc", "wooden spatula-60f3ad35616358e3", "pressure cooker-0246013723857daf", "hot plate-bdb2f36f470baabe", "plate1-90441383d9ec2ef5", "plate2-0c1ebe243bbb7415", "bowl1-08bc348b54211814", "bowl2-d6ef8746af1cfbf3", "mug3-c8797ffdf6d0c0d8", "spoon4-3a3e82e9d3746297", "sink plug-4fe2ab06f3d3aade", "spoon5-7f4bca22b3f00984", "spoon6-5678aae24053cfe5", "pestle-16c66a31b9e2f57d", "mortar-69fddd7526c01079", "regulator of pressure cooker-769d6c5b28da8f98", "lid of pressure cooker-1c8ec329ef372c28", "plate3-d06e3a2b3c05a3bd", "serving spoon-c9c9f96a634adf87", "parchment paper-b304d09d2f756172", "white spoon-58bb40a8807be721", "weighing scales-24aa1e611d7eef8c", "stack of plastic boxes-efbb3fffad4e439d", "spoon7-8824aea87e11043e", "pen-2b6413786463fa00", "cap of pen-d1abcc7c527de322", "notebook-2204dcb867bc80b7" ], "reasoning": "1. I locate the vent at 00:00 above the countertop in front of the user.\n2. At 00:02, I identify 2 electrical panels above the wooden cutting board to the left of the user.\n3. At 02:58 I identify 5 buttons on the vent in front of the user.\n4. I subtract 2 from 5 to get 3.", "referenced_object_masks": [] }, { "key": "P04-20240415-184003:4c517c91f5f21a69f5e7979eb1f34799ccd154fd", "video_id": "P04-20240415-184003", "question": "What is the 7th item I touch with my right hand while standing in front of the sink for the first time?", "answer_choice_0": "Faucet lever", "answer_choice_1": "Knife", "answer_choice_2": "Sponge", "answer_choice_3": "Measuring cup", "answer_choice_4": "Mug", "answer_id": 3, "answer": "Measuring cup", "question_type": [ "Object Recognition", "Action Recognition", "Counting", "Event Occurence" ], "objects_of_interest": [ "mug1-bc42b8b4245cca68", "mug2-2fed1fcc04d6657d", "glass-5f0690a1864ecddc", "carton of milk-12ec887ce8367ee0", "water cooler's drip tray-d9d3de2126debcbd", "sponge-9dbc2671c7ef8878", "measuring jug-859522538fc68034", "knife-4754276585807924", "spoon1-4e4a44c7a0a65c24", "spoon2-531733cc037ca5f3", "spoon3-6413b53a66d4a2f1", "purple cloth-bc298e9dd628f207", "metallic cup-9b7b19e462bd15eb", "chopping board-fab81e2ae0a666e4", "cooling rack-53ff57d481c4b7dc", "wooden spatula-60f3ad35616358e3", "pressure cooker-0246013723857daf", "hot plate-bdb2f36f470baabe", "plate1-90441383d9ec2ef5", "plate2-0c1ebe243bbb7415", "bowl1-08bc348b54211814", "bowl2-d6ef8746af1cfbf3", "mug3-c8797ffdf6d0c0d8", "spoon4-3a3e82e9d3746297", "sink plug-4fe2ab06f3d3aade", "spoon5-7f4bca22b3f00984", "spoon6-5678aae24053cfe5", "pestle-16c66a31b9e2f57d", "mortar-69fddd7526c01079", "regulator of pressure cooker-769d6c5b28da8f98", "lid of pressure cooker-1c8ec329ef372c28", "plate3-d06e3a2b3c05a3bd", "serving spoon-c9c9f96a634adf87", "parchment paper-b304d09d2f756172", "white spoon-58bb40a8807be721", "weighing scales-24aa1e611d7eef8c", "stack of plastic boxes-efbb3fffad4e439d", "spoon7-8824aea87e11043e", "pen-2b6413786463fa00", "cap of pen-d1abcc7c527de322", "notebook-2204dcb867bc80b7" ], "reasoning": "1. I located the moment the user stands in front of the sink for the first time at 00:05. \n2. I continued to watch to identify the items the user touches with the right hand and counted until I reached the 7th item. \n3. The first item the user touches with the right hand is a white mug (mug1-bc42b8b4245cca68) at 00:06. The second item is glass (glass-5f0690a1864ecddc) at 00:12. The third item is the drip tray of a water cooler (water cooler's drip tray-d9d3de2126debcbd) at 00:18. The fourth item is the yellow sponge (sponge-9dbc2671c7ef8878) at 00:21. The fifth item the user touches with the right hand is the faucet lever at 00:23. The sixth item is the faucet lever again at 00:24. Finally, the seventh item is a measuring cup (measuring jug-859522538fc68034) at 00:26.", "referenced_object_masks": [ "mug1-bc42b8b4245cca68", "glass-5f0690a1864ecddc", "measuring jug-859522538fc68034", "sponge-9dbc2671c7ef8878", "water cooler's drip tray-d9d3de2126debcbd" ] }, { "key": "P04-20240415-184003:6f52a67f8dc59089399216a44ce9a80d269e65e5", "video_id": "P04-20240415-184003", "question": "Where is the trash can in relation to me if I turn 180 degrees while standing in front of the sink?", "answer_choice_0": "In front", "answer_choice_1": "Diagonally to the right", "answer_choice_2": "Diagonally to the left", "answer_choice_3": "Straight across", "answer_choice_4": "To the left", "answer_id": 2, "answer": "Diagonally to the left", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "mug1-bc42b8b4245cca68", "mug2-2fed1fcc04d6657d", "glass-5f0690a1864ecddc", "carton of milk-12ec887ce8367ee0", "water cooler's drip tray-d9d3de2126debcbd", "sponge-9dbc2671c7ef8878", "measuring jug-859522538fc68034", "knife-4754276585807924", "spoon1-4e4a44c7a0a65c24", "spoon2-531733cc037ca5f3", "spoon3-6413b53a66d4a2f1", "purple cloth-bc298e9dd628f207", "metallic cup-9b7b19e462bd15eb", "chopping board-fab81e2ae0a666e4", "cooling rack-53ff57d481c4b7dc", "wooden spatula-60f3ad35616358e3", "pressure cooker-0246013723857daf", "hot plate-bdb2f36f470baabe", "plate1-90441383d9ec2ef5", "plate2-0c1ebe243bbb7415", "bowl1-08bc348b54211814", "bowl2-d6ef8746af1cfbf3", "mug3-c8797ffdf6d0c0d8", "spoon4-3a3e82e9d3746297", "sink plug-4fe2ab06f3d3aade", "spoon5-7f4bca22b3f00984", "spoon6-5678aae24053cfe5", "pestle-16c66a31b9e2f57d", "mortar-69fddd7526c01079", "regulator of pressure cooker-769d6c5b28da8f98", "lid of pressure cooker-1c8ec329ef372c28", "plate3-d06e3a2b3c05a3bd", "serving spoon-c9c9f96a634adf87", "parchment paper-b304d09d2f756172", "white spoon-58bb40a8807be721", "weighing scales-24aa1e611d7eef8c", "stack of plastic boxes-efbb3fffad4e439d", "spoon7-8824aea87e11043e", "pen-2b6413786463fa00", "cap of pen-d1abcc7c527de322", "notebook-2204dcb867bc80b7" ], "reasoning": "1. I watched the video from the beginning to locate the trash can in the kitchen. At 05:04, the user turns around and drops something in the trash can, which is located next to the dining area in the kitchen that is against the wall.\n2. At 05:06, the user turns back around to the right, which causes them to be face to face with the sink.\n3. This information helped me conclude that if the user is standing in front of the sink and turns 180 degrees in either direction, the trash can will be located diagonally to their left.", "referenced_object_masks": [] }, { "key": "P04-20240415-184003:7d817441850cf8df622b5b3e8d21b9b5eed53cc0", "video_id": "P04-20240415-184003", "question": "What is the third item I scrub after putting the measuring cup to dry?", "answer_choice_0": "Second spoon", "answer_choice_1": "First spoon", "answer_choice_2": "Cutting board", "answer_choice_3": "Mug", "answer_choice_4": "Knife", "answer_id": 0, "answer": "Second spoon", "question_type": [ "Object Recognition", "Action Recognition", "Counting", "Event Occurence", "Temporal Reasoning" ], "objects_of_interest": [ "mug1-bc42b8b4245cca68", "mug2-2fed1fcc04d6657d", "glass-5f0690a1864ecddc", "carton of milk-12ec887ce8367ee0", "water cooler's drip tray-d9d3de2126debcbd", "sponge-9dbc2671c7ef8878", "measuring jug-859522538fc68034", "knife-4754276585807924", "spoon1-4e4a44c7a0a65c24", "spoon2-531733cc037ca5f3", "spoon3-6413b53a66d4a2f1", "purple cloth-bc298e9dd628f207", "metallic cup-9b7b19e462bd15eb", "chopping board-fab81e2ae0a666e4", "cooling rack-53ff57d481c4b7dc", "wooden spatula-60f3ad35616358e3", "pressure cooker-0246013723857daf", "hot plate-bdb2f36f470baabe", "plate1-90441383d9ec2ef5", "plate2-0c1ebe243bbb7415", "bowl1-08bc348b54211814", "bowl2-d6ef8746af1cfbf3", "mug3-c8797ffdf6d0c0d8", "spoon4-3a3e82e9d3746297", "sink plug-4fe2ab06f3d3aade", "spoon5-7f4bca22b3f00984", "spoon6-5678aae24053cfe5", "pestle-16c66a31b9e2f57d", "mortar-69fddd7526c01079", "regulator of pressure cooker-769d6c5b28da8f98", "lid of pressure cooker-1c8ec329ef372c28", "plate3-d06e3a2b3c05a3bd", "serving spoon-c9c9f96a634adf87", "parchment paper-b304d09d2f756172", "white spoon-58bb40a8807be721", "weighing scales-24aa1e611d7eef8c", "stack of plastic boxes-efbb3fffad4e439d", "spoon7-8824aea87e11043e", "pen-2b6413786463fa00", "cap of pen-d1abcc7c527de322", "notebook-2204dcb867bc80b7" ], "reasoning": "1. I located the moment the user puts the measuring cup (measuring jug-859522538fc68034) to dry after rinsing it at 02:05.\n2. I watched for the first item the user scrubbed after putting the measuring cup (measuring jug-859522538fc68034) to dry. The user grabs a knife (knife-4754276585807924) first at 02:07, and scrubs it from 02:09 to 02:18.\n3. The user then grabs a spoon (spoon1-4e4a44c7a0a65c24) at 02:20 then scrubs it with the sponge (sponge-9dbc2671c7ef8878) from 02:21 to 02:28. \n4. The third item the user grabs from the sink to wash is a second spoon (spoon2-531733cc037ca5f3) at 02:30, and they scrub it from 02:31 to 02:38. 5. Therefore, the item the user scrubs third after putting the measuring cup (measuring jug-859522538fc68034) to dry is the second spoon (spoon2-531733cc037ca5f3).", "referenced_object_masks": [ "sponge-9dbc2671c7ef8878", "knife-4754276585807924", "measuring jug-859522538fc68034", "spoon1-4e4a44c7a0a65c24", "spoon2-531733cc037ca5f3" ] }, { "key": "P04-20240415-184003:8a95152034d6d3df7aa376da50df0972ecc602f8", "video_id": "P04-20240415-184003", "question": "What object did I remove from a drawer right before putting the wooden spatula in the sink?", "answer_choice_0": "Mortar", "answer_choice_1": "Parchment paper", "answer_choice_2": "White spoon", "answer_choice_3": "Purple cloth", "answer_choice_4": "Pestle", "answer_id": 2, "answer": "White spoon", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "mug1-bc42b8b4245cca68", "mug2-2fed1fcc04d6657d", "glass-5f0690a1864ecddc", "carton of milk-12ec887ce8367ee0", "water cooler's drip tray-d9d3de2126debcbd", "sponge-9dbc2671c7ef8878", "measuring jug-859522538fc68034", "knife-4754276585807924", "spoon1-4e4a44c7a0a65c24", "spoon2-531733cc037ca5f3", "spoon3-6413b53a66d4a2f1", "purple cloth-bc298e9dd628f207", "metallic cup-9b7b19e462bd15eb", "chopping board-fab81e2ae0a666e4", "cooling rack-53ff57d481c4b7dc", "wooden spatula-60f3ad35616358e3", "pressure cooker-0246013723857daf", "hot plate-bdb2f36f470baabe", "plate1-90441383d9ec2ef5", "plate2-0c1ebe243bbb7415", "bowl1-08bc348b54211814", "bowl2-d6ef8746af1cfbf3", "mug3-c8797ffdf6d0c0d8", "spoon4-3a3e82e9d3746297", "sink plug-4fe2ab06f3d3aade", "spoon5-7f4bca22b3f00984", "spoon6-5678aae24053cfe5", "pestle-16c66a31b9e2f57d", "mortar-69fddd7526c01079", "regulator of pressure cooker-769d6c5b28da8f98", "lid of pressure cooker-1c8ec329ef372c28", "plate3-d06e3a2b3c05a3bd", "serving spoon-c9c9f96a634adf87", "parchment paper-b304d09d2f756172", "white spoon-58bb40a8807be721", "weighing scales-24aa1e611d7eef8c", "stack of plastic boxes-efbb3fffad4e439d", "spoon7-8824aea87e11043e", "pen-2b6413786463fa00", "cap of pen-d1abcc7c527de322", "notebook-2204dcb867bc80b7" ], "reasoning": "1. I locate the wooden spatula (wooden spatula-60f3ad35616358e3) at 04:47 to the right of the wooden cutting board .\n2. I observe the user place the wooden spatula (wooden spatula-60f3ad35616358e3) into the sink at 15:46.\n3. I rewind until observing the user remove a white spoon (white spoon-58bb40a8807be721) from the drawer at 15:34.\n4. I watch until 15:46 to determine no other object is removed from a drawer .", "referenced_object_masks": [ "wooden spatula-60f3ad35616358e3", "white spoon-58bb40a8807be721" ] }, { "key": "P04-20240415-184003:92a09a003c900dbf6ffd71e0945c42a2be3715d7", "video_id": "P04-20240415-184003", "question": "After I put a spoon into the rice I served, what are the first two objects I grab?", "answer_choice_0": "Pen and pen cap", "answer_choice_1": ", Notebook and pen cap", "answer_choice_2": "Weighing scale and pen", "answer_choice_3": "Weighing scale and notebook", "answer_choice_4": "Notebook and pen", "answer_id": 0, "answer": "Pen and pen cap", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counting" ], "objects_of_interest": [ "mug1-bc42b8b4245cca68", "mug2-2fed1fcc04d6657d", "glass-5f0690a1864ecddc", "carton of milk-12ec887ce8367ee0", "water cooler's drip tray-d9d3de2126debcbd", "sponge-9dbc2671c7ef8878", "measuring jug-859522538fc68034", "knife-4754276585807924", "spoon1-4e4a44c7a0a65c24", "spoon2-531733cc037ca5f3", "spoon3-6413b53a66d4a2f1", "purple cloth-bc298e9dd628f207", "metallic cup-9b7b19e462bd15eb", "chopping board-fab81e2ae0a666e4", "cooling rack-53ff57d481c4b7dc", "wooden spatula-60f3ad35616358e3", "pressure cooker-0246013723857daf", "hot plate-bdb2f36f470baabe", "plate1-90441383d9ec2ef5", "plate2-0c1ebe243bbb7415", "bowl1-08bc348b54211814", "bowl2-d6ef8746af1cfbf3", "mug3-c8797ffdf6d0c0d8", "spoon4-3a3e82e9d3746297", "sink plug-4fe2ab06f3d3aade", "spoon5-7f4bca22b3f00984", "spoon6-5678aae24053cfe5", "pestle-16c66a31b9e2f57d", "mortar-69fddd7526c01079", "regulator of pressure cooker-769d6c5b28da8f98", "lid of pressure cooker-1c8ec329ef372c28", "plate3-d06e3a2b3c05a3bd", "serving spoon-c9c9f96a634adf87", "parchment paper-b304d09d2f756172", "white spoon-58bb40a8807be721", "weighing scales-24aa1e611d7eef8c", "stack of plastic boxes-efbb3fffad4e439d", "spoon7-8824aea87e11043e", "pen-2b6413786463fa00", "cap of pen-d1abcc7c527de322", "notebook-2204dcb867bc80b7" ], "reasoning": "1. From 15:51 to 16:20 I observe the user serving rice onto a plate (plate3-d06e3a2b3c05a3bd) with the white spoon (white spoon-58bb40a8807be721).\n2. I observe the user take a spoon (spoon7-8824aea87e11043e) from the drawer at 16:52.\n3. I determine the user puts the spoon (spoon7-8824aea87e11043e) into the rice on their plate (plate3-d06e3a2b3c05a3bd) at 16:59.\n4. At 17:03, I observe the user grab the pen (pen-2b6413786463fa00) with their left hand and the pen cap (cap of pen-d1abcc7c527de322) with their right.", "referenced_object_masks": [ "white spoon-58bb40a8807be721", "pen-2b6413786463fa00", "plate3-d06e3a2b3c05a3bd", "spoon7-8824aea87e11043e", "cap of pen-d1abcc7c527de322" ] }, { "key": "P04-20240415-184003:d2d7190920305650afa33c2bc59397871912f011", "video_id": "P04-20240415-184003", "question": "If I switch the positions of the yellow and orange rags with the purple one while I'm serving the cooked rice, where would the pressure cooker regulator be in relation to the yellow and orange rags?", "answer_choice_0": "To the left of the yellow and orange rags", "answer_choice_1": "Upper left corner of the yellow and orange rags", "answer_choice_2": "Upper right corner of yellow and orange rags", "answer_choice_3": "Behind the yellow and orange rags", "answer_choice_4": "Under the yellow and orange rag", "answer_id": 1, "answer": "Upper left corner of the yellow and orange rags", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Event Occurence", "Counterfactual" ], "objects_of_interest": [ "mug1-bc42b8b4245cca68", "mug2-2fed1fcc04d6657d", "glass-5f0690a1864ecddc", "carton of milk-12ec887ce8367ee0", "water cooler's drip tray-d9d3de2126debcbd", "sponge-9dbc2671c7ef8878", "measuring jug-859522538fc68034", "knife-4754276585807924", "spoon1-4e4a44c7a0a65c24", "spoon2-531733cc037ca5f3", "spoon3-6413b53a66d4a2f1", "purple cloth-bc298e9dd628f207", "metallic cup-9b7b19e462bd15eb", "chopping board-fab81e2ae0a666e4", "cooling rack-53ff57d481c4b7dc", "wooden spatula-60f3ad35616358e3", "pressure cooker-0246013723857daf", "hot plate-bdb2f36f470baabe", "plate1-90441383d9ec2ef5", "plate2-0c1ebe243bbb7415", "bowl1-08bc348b54211814", "bowl2-d6ef8746af1cfbf3", "mug3-c8797ffdf6d0c0d8", "spoon4-3a3e82e9d3746297", "sink plug-4fe2ab06f3d3aade", "spoon5-7f4bca22b3f00984", "spoon6-5678aae24053cfe5", "pestle-16c66a31b9e2f57d", "mortar-69fddd7526c01079", "regulator of pressure cooker-769d6c5b28da8f98", "lid of pressure cooker-1c8ec329ef372c28", "plate3-d06e3a2b3c05a3bd", "serving spoon-c9c9f96a634adf87", "parchment paper-b304d09d2f756172", "white spoon-58bb40a8807be721", "weighing scales-24aa1e611d7eef8c", "stack of plastic boxes-efbb3fffad4e439d", "spoon7-8824aea87e11043e", "pen-2b6413786463fa00", "cap of pen-d1abcc7c527de322", "notebook-2204dcb867bc80b7" ], "reasoning": "1. I located the moment the user is serving the rice at 15:51 and looked on the kitchen counter to the items around the plate (plate3-d06e3a2b3c05a3bd).\n2. I noticed the yellow and orange rags on the kitchen counter diagonally to the left from the user's perspective.\n3. I kept watching to locate the purple rag (purple cloth-bc298e9dd628f207), which is visible at 16:19, to the right of the user, in front of the double burner (hot plate-bdb2f36f470baabe). I also observed at 16:21 that the pressure cooker regulator (regulator of pressure cooker-769d6c5b28da8f98) is on the counter, diagonally towards the upper left corner of the purple rag (purple cloth-bc298e9dd628f207).\n4. Therefore, if the user switches the positions of the yellow and orange rags with the purple rag (purple cloth-bc298e9dd628f207) in the segment where they are serving the rice , the pressure cooker regulator (regulator of pressure cooker-769d6c5b28da8f98) would be towards the upper left corner of the yellow and orange rags .", "referenced_object_masks": [ "hot plate-bdb2f36f470baabe", "plate3-d06e3a2b3c05a3bd", "regulator of pressure cooker-769d6c5b28da8f98", "purple cloth-bc298e9dd628f207" ] }, { "key": "P04-20240415-184003:d76ed4bb443368e29d06a53d12a502ab458cf3fe", "video_id": "P04-20240415-184003", "question": "What object do I clean with a sponge right before the pressure cooker goes off?", "answer_choice_0": "Mug", "answer_choice_1": "Plate", "answer_choice_2": "Bowl", "answer_choice_3": "Wooden spatula", "answer_choice_4": "Metallic cup", "answer_id": 4, "answer": "Metallic cup", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "mug1-bc42b8b4245cca68", "mug2-2fed1fcc04d6657d", "glass-5f0690a1864ecddc", "carton of milk-12ec887ce8367ee0", "water cooler's drip tray-d9d3de2126debcbd", "sponge-9dbc2671c7ef8878", "measuring jug-859522538fc68034", "knife-4754276585807924", "spoon1-4e4a44c7a0a65c24", "spoon2-531733cc037ca5f3", "spoon3-6413b53a66d4a2f1", "purple cloth-bc298e9dd628f207", "metallic cup-9b7b19e462bd15eb", "chopping board-fab81e2ae0a666e4", "cooling rack-53ff57d481c4b7dc", "wooden spatula-60f3ad35616358e3", "pressure cooker-0246013723857daf", "hot plate-bdb2f36f470baabe", "plate1-90441383d9ec2ef5", "plate2-0c1ebe243bbb7415", "bowl1-08bc348b54211814", "bowl2-d6ef8746af1cfbf3", "mug3-c8797ffdf6d0c0d8", "spoon4-3a3e82e9d3746297", "sink plug-4fe2ab06f3d3aade", "spoon5-7f4bca22b3f00984", "spoon6-5678aae24053cfe5", "pestle-16c66a31b9e2f57d", "mortar-69fddd7526c01079", "regulator of pressure cooker-769d6c5b28da8f98", "lid of pressure cooker-1c8ec329ef372c28", "plate3-d06e3a2b3c05a3bd", "serving spoon-c9c9f96a634adf87", "parchment paper-b304d09d2f756172", "white spoon-58bb40a8807be721", "weighing scales-24aa1e611d7eef8c", "stack of plastic boxes-efbb3fffad4e439d", "spoon7-8824aea87e11043e", "pen-2b6413786463fa00", "cap of pen-d1abcc7c527de322", "notebook-2204dcb867bc80b7" ], "reasoning": "1. I locate the pressure cooker (pressure cooker-0246013723857daf) at 00:00 on the hot plate (hot plate-bdb2f36f470baabe) to the left of the sink .\n2. I observe the pressure cooker (pressure cooker-0246013723857daf) go off at 03:49.\n3. I rewind to observe the user pick up a sponge (sponge-9dbc2671c7ef8878) at 03:06.\n4. I observe the user clean the metallic cup (metallic cup-9b7b19e462bd15eb) with the sponge (sponge-9dbc2671c7ef8878) from 03:06 to 03:18.\n5. I watch until 03:49 to determine that no other object is cleaned until the pressure cooker (pressure cooker-0246013723857daf) goes off.", "referenced_object_masks": [ "hot plate-bdb2f36f470baabe", "metallic cup-9b7b19e462bd15eb", "sponge-9dbc2671c7ef8878", "pressure cooker-0246013723857daf" ] }, { "key": "P04-20240415-184003:dd988b7bd45d6761f249f19447dbc2cf95e8c968", "video_id": "P04-20240415-184003", "question": "What is the difference between the first and last temperature readings on the hot plate?", "answer_choice_0": "400", "answer_choice_1": "300", "answer_choice_2": "500", "answer_choice_3": "600", "answer_choice_4": "200", "answer_id": 0, "answer": "400", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning", "Numerical Reasoning" ], "objects_of_interest": [ "mug1-bc42b8b4245cca68", "mug2-2fed1fcc04d6657d", "glass-5f0690a1864ecddc", "carton of milk-12ec887ce8367ee0", "water cooler's drip tray-d9d3de2126debcbd", "sponge-9dbc2671c7ef8878", "measuring jug-859522538fc68034", "knife-4754276585807924", "spoon1-4e4a44c7a0a65c24", "spoon2-531733cc037ca5f3", "spoon3-6413b53a66d4a2f1", "purple cloth-bc298e9dd628f207", "metallic cup-9b7b19e462bd15eb", "chopping board-fab81e2ae0a666e4", "cooling rack-53ff57d481c4b7dc", "wooden spatula-60f3ad35616358e3", "pressure cooker-0246013723857daf", "hot plate-bdb2f36f470baabe", "plate1-90441383d9ec2ef5", "plate2-0c1ebe243bbb7415", "bowl1-08bc348b54211814", "bowl2-d6ef8746af1cfbf3", "mug3-c8797ffdf6d0c0d8", "spoon4-3a3e82e9d3746297", "sink plug-4fe2ab06f3d3aade", "spoon5-7f4bca22b3f00984", "spoon6-5678aae24053cfe5", "pestle-16c66a31b9e2f57d", "mortar-69fddd7526c01079", "regulator of pressure cooker-769d6c5b28da8f98", "lid of pressure cooker-1c8ec329ef372c28", "plate3-d06e3a2b3c05a3bd", "serving spoon-c9c9f96a634adf87", "parchment paper-b304d09d2f756172", "white spoon-58bb40a8807be721", "weighing scales-24aa1e611d7eef8c", "stack of plastic boxes-efbb3fffad4e439d", "spoon7-8824aea87e11043e", "pen-2b6413786463fa00", "cap of pen-d1abcc7c527de322", "notebook-2204dcb867bc80b7" ], "reasoning": "1. I locate the hot plate (hot plate-bdb2f36f470baabe) at 00:00 to the left of the sink .\n2. I determine the first temperature reading on the hot plate (hot plate-bdb2f36f470baabe) is 600 at 00:43. \n3. I observe the user turn the hot plate (hot plate-bdb2f36f470baabe) off at 04:42.\n4. I try to find the last number shown on the hot plate (hot plate-bdb2f36f470baabe) and I find it at 04:41.\n5. I read it and discover that it is 200.\n5. I subtract 200 from 600 to get 400.", "referenced_object_masks": [ "hot plate-bdb2f36f470baabe" ] }, { "key": "P05-20240423-170021:0eaeeb18f97a8c5a65043b2328d4214cd1452be9", "video_id": "P05-20240423-170021", "question": "How many times did I stir the onions?", "answer_choice_0": "224 times", "answer_choice_1": "150 times", "answer_choice_2": "234 times", "answer_choice_3": "302 times", "answer_choice_4": "197 times", "answer_id": 0, "answer": "224 times", "question_type": [ "Counting", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "pan-4df8c931ca350ae0", "bottle of oil-de702e66317d3296", "chopping board-8f4981c49bb83b03", "knife-f014c71b617dbc9a", "onion1-f24307ea89e936e0", "onion2-13b99426481ecac0", "onion3-1ccd3874246098f1", "onion4-4d22ca85290b353d", "onion5-ff1bffcf90d969db", "onionhalf1-2-4763de74f6508776", "onion half2-2-b3917f702ecb51a4", "onion half3-2-e66c429fe8c7816e", "onion half4-2-b1fac77b195c0c9b", "onion half5-2-d96b2be8858ccb5d", "wooden spoon-94ecbebb0cd824ee", "recycling container-537c7b667329d769", "lid of recycling container-b1c5daac7234d378", "scales-0bba1e7cc7433c93", "packet-bf2fb3d72df534ed", "ginger-1f3a23ea98451db2", "packet of chilies-fa5b9bca92e5856e", "some chillies-09ba2847a3e6a95d", "chillies2-85ff319d5680249a", "chillies3-85d202ef71da6694", "chillies4-58f8cb4099672dc6", "chillies5-b92fa05df4f8db9a", "bag-537393362b30a33a", "frozen garlic-dc83d5622c1d29d4", "phone-ef96e687dcd5ac8c", "curry powder jar-95d828fcfeccf9b8", "chilli powder jar-9f933647dfda14b4", "packet of smoked paprika-492663d77f14e566", "pot of cinnamon-0ac6a4e87fc7b5d4", "lid of pot of cinnamon-c1ce83cf756d6bca", "cinnamon stick-e116acccb8669545", "lid of pot of cardamon-82e05945d24f5f96", "bowl-43cac69e66c32583", "measuring spoons-0b573d3225382179", "lid of curry powder-cc285f94c1abe8a1", "lid of chilli powder-b3dacd102ffc9fc2", "turmeric jar-adb4a6c789d6885f", "tandoori masala jar-ca968e9a7aaf7891", "lid of turmeric-05e71d5e79a0a283", "lid of tandoori masala jar-36a112b0779c73dc", "spice pot-86eaaeae40a4e360", "lid of spice pot-6854e93c8503014e", "pot of spice-c29290dd6f9b931f", "box of spices-4b476e75b957c453", "plastic lid of spice-3c8ba673f3c60517", "bottle of oil2-2595aa54bcbe3a49", "container of pepper-d7874f1497993af6", "container of salt-15a6da04b95b9de9", "spice packet-cbc48105f820aa94", "pot unknown-6894722cad8e5d03", "another pot unknown-873e993ade7ef26a", "two tins of plum tomatoes-cf08b80ab8c73611", "oven glove-c9ceab02eb1d30de" ], "reasoning": "1. I watched the video to see the first time the user stirred the onions and I saw this happen 23 times between 08:30-08:39.\n2. I then observed the user stirring the onions 4 times between 09:18-09:20.\n3. I then observed the user stirring the onions 17 times between 11:17-11:24.\n4. I then observed the user stirring the onions 18 times between 12:15-12:21.\n5. I then observed the user stirring the onions 23 times between 14:02-14:11.\n6. I then observed the user stirring the onions 32 times between 15:22-15:33.\n7. I then observed the user stirring the onions 42 times between 16:10-16:25.\n8. I then observed the user stirring the onions 33 times between 18:03-18:16.\n9. I then observed the user stirring the onions 28 times between 18:44-18:54, (wooden spoon-94ecbebb0cd824ee) to seemingly crush some ingredients in the pan (pan-4df8c931ca350ae0) then 4 times again between 18:58-19:00.\n10. The total amount of stirs comes to 224.", "referenced_object_masks": [ "wooden spoon-94ecbebb0cd824ee", "pan-4df8c931ca350ae0" ] }, { "key": "P05-20240423-170021:3a426ad194da7e767acb89dc5e5156ace7057f15", "video_id": "P05-20240423-170021", "question": "What was the average amount of times I chopped each onion half?", "answer_choice_0": "18.6", "answer_choice_1": "19.8", "answer_choice_2": "15.4", "answer_choice_3": "17.5", "answer_choice_4": "21.9", "answer_id": 1, "answer": "19.8", "question_type": [ "Counting", "Object Recognition", "Action Recognition", "Numerical Reasoning" ], "objects_of_interest": [ "pan-4df8c931ca350ae0", "bottle of oil-de702e66317d3296", "chopping board-8f4981c49bb83b03", "knife-f014c71b617dbc9a", "onion1-f24307ea89e936e0", "onion2-13b99426481ecac0", "onion3-1ccd3874246098f1", "onion4-4d22ca85290b353d", "onion5-ff1bffcf90d969db", "onionhalf1-2-4763de74f6508776", "onion half2-2-b3917f702ecb51a4", "onion half3-2-e66c429fe8c7816e", "onion half4-2-b1fac77b195c0c9b", "onion half5-2-d96b2be8858ccb5d", "wooden spoon-94ecbebb0cd824ee", "recycling container-537c7b667329d769", "lid of recycling container-b1c5daac7234d378", "scales-0bba1e7cc7433c93", "packet-bf2fb3d72df534ed", "ginger-1f3a23ea98451db2", "packet of chilies-fa5b9bca92e5856e", "some chillies-09ba2847a3e6a95d", "chillies2-85ff319d5680249a", "chillies3-85d202ef71da6694", "chillies4-58f8cb4099672dc6", "chillies5-b92fa05df4f8db9a", "bag-537393362b30a33a", "frozen garlic-dc83d5622c1d29d4", "phone-ef96e687dcd5ac8c", "curry powder jar-95d828fcfeccf9b8", "chilli powder jar-9f933647dfda14b4", "packet of smoked paprika-492663d77f14e566", "pot of cinnamon-0ac6a4e87fc7b5d4", "lid of pot of cinnamon-c1ce83cf756d6bca", "cinnamon stick-e116acccb8669545", "lid of pot of cardamon-82e05945d24f5f96", "bowl-43cac69e66c32583", "measuring spoons-0b573d3225382179", "lid of curry powder-cc285f94c1abe8a1", "lid of chilli powder-b3dacd102ffc9fc2", "turmeric jar-adb4a6c789d6885f", "tandoori masala jar-ca968e9a7aaf7891", "lid of turmeric-05e71d5e79a0a283", "lid of tandoori masala jar-36a112b0779c73dc", "spice pot-86eaaeae40a4e360", "lid of spice pot-6854e93c8503014e", "pot of spice-c29290dd6f9b931f", "box of spices-4b476e75b957c453", "plastic lid of spice-3c8ba673f3c60517", "bottle of oil2-2595aa54bcbe3a49", "container of pepper-d7874f1497993af6", "container of salt-15a6da04b95b9de9", "spice packet-cbc48105f820aa94", "pot unknown-6894722cad8e5d03", "another pot unknown-873e993ade7ef26a", "two tins of plum tomatoes-cf08b80ab8c73611", "oven glove-c9ceab02eb1d30de" ], "reasoning": "1. I watched the video to see when the user started chopping onions and saw them chop the first onion half 24 times between 04:19-04:38.\n2. I then observed the user chop the second onion half 17 times between 04:47-05:00.\n3. I then observed the user chop the third onion half 20 times between 05:03-05:23.\n4. I then observed the user chop the fourth onion half 19 times between 05:32-05:48.\n5. I then observed the user chop the fifth onion half 17 times between 05:50-06:05.\n6. I then observed the user chop the sixth onion half 21 times between 06:14-06:33.\n7. I then observed the user chop the seventh onion half 15 times between 06:40-06:53.\n8. I then observed the user chop the eighth onion half 22 times between 07:03-07:21.\n9. I then observed the user chop the ninth onion half 14 times between 07:32-07:42.\n10. I then observed the user chop the tenth onion half 29 times between 07:50-08:14.\n11. Adding all of these together comes to 198 and dividing 198 by 10 comes out to 19.8 chops per onion half.", "referenced_object_masks": [] }, { "key": "P05-20240423-170021:47c5223921a7a8ef6743985a771392f83849b2bb", "video_id": "P05-20240423-170021", "question": "How many different drawers did I attempt to open in the freezer?", "answer_choice_0": "1 drawer", "answer_choice_1": "3 drawers", "answer_choice_2": "4 drawers", "answer_choice_3": "9 drawers", "answer_choice_4": "5 drawers", "answer_id": 1, "answer": "3 drawers", "question_type": [ "Action Recognition", "Object Recognition", "Counting" ], "objects_of_interest": [ "pan-4df8c931ca350ae0", "bottle of oil-de702e66317d3296", "chopping board-8f4981c49bb83b03", "knife-f014c71b617dbc9a", "onion1-f24307ea89e936e0", "onion2-13b99426481ecac0", "onion3-1ccd3874246098f1", "onion4-4d22ca85290b353d", "onion5-ff1bffcf90d969db", "onionhalf1-2-4763de74f6508776", "onion half2-2-b3917f702ecb51a4", "onion half3-2-e66c429fe8c7816e", "onion half4-2-b1fac77b195c0c9b", "onion half5-2-d96b2be8858ccb5d", "wooden spoon-94ecbebb0cd824ee", "recycling container-537c7b667329d769", "lid of recycling container-b1c5daac7234d378", "scales-0bba1e7cc7433c93", "packet-bf2fb3d72df534ed", "ginger-1f3a23ea98451db2", "packet of chilies-fa5b9bca92e5856e", "some chillies-09ba2847a3e6a95d", "chillies2-85ff319d5680249a", "chillies3-85d202ef71da6694", "chillies4-58f8cb4099672dc6", "chillies5-b92fa05df4f8db9a", "bag-537393362b30a33a", "frozen garlic-dc83d5622c1d29d4", "phone-ef96e687dcd5ac8c", "curry powder jar-95d828fcfeccf9b8", "chilli powder jar-9f933647dfda14b4", "packet of smoked paprika-492663d77f14e566", "pot of cinnamon-0ac6a4e87fc7b5d4", "lid of pot of cinnamon-c1ce83cf756d6bca", "cinnamon stick-e116acccb8669545", "lid of pot of cardamon-82e05945d24f5f96", "bowl-43cac69e66c32583", "measuring spoons-0b573d3225382179", "lid of curry powder-cc285f94c1abe8a1", "lid of chilli powder-b3dacd102ffc9fc2", "turmeric jar-adb4a6c789d6885f", "tandoori masala jar-ca968e9a7aaf7891", "lid of turmeric-05e71d5e79a0a283", "lid of tandoori masala jar-36a112b0779c73dc", "spice pot-86eaaeae40a4e360", "lid of spice pot-6854e93c8503014e", "pot of spice-c29290dd6f9b931f", "box of spices-4b476e75b957c453", "plastic lid of spice-3c8ba673f3c60517", "bottle of oil2-2595aa54bcbe3a49", "container of pepper-d7874f1497993af6", "container of salt-15a6da04b95b9de9", "spice packet-cbc48105f820aa94", "pot unknown-6894722cad8e5d03", "another pot unknown-873e993ade7ef26a", "two tins of plum tomatoes-cf08b80ab8c73611", "oven glove-c9ceab02eb1d30de" ], "reasoning": "1. I watched the video to see what drawers the user attempted to open in the freezer and saw the user open the top drawer at 09:25.\n2. I then observed the user attempt to open the third drawer from the top at 10:15, but then promptly close it, before opening the second drawer from the top at 10:16.\n3. Therefore, the user attempted to open 3 drawers in the freezer .", "referenced_object_masks": [] }, { "key": "P05-20240423-170021:501b6b876b16a99c47526c47536e7e43d82e6017", "video_id": "P05-20240423-170021", "question": "What is unique about the first two spices I add to the dish compared to the other spices I use?", "answer_choice_0": "They are crushed instead of whole", "answer_choice_1": "They are ground and not whole", "answer_choice_2": "They are fresh and not dried", "answer_choice_3": "They are whole and not ground", "answer_choice_4": "They are dried and not fresh", "answer_id": 3, "answer": "They are whole and not ground", "question_type": [ "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "pan-4df8c931ca350ae0", "bottle of oil-de702e66317d3296", "chopping board-8f4981c49bb83b03", "knife-f014c71b617dbc9a", "onion1-f24307ea89e936e0", "onion2-13b99426481ecac0", "onion3-1ccd3874246098f1", "onion4-4d22ca85290b353d", "onion5-ff1bffcf90d969db", "onionhalf1-2-4763de74f6508776", "onion half2-2-b3917f702ecb51a4", "onion half3-2-e66c429fe8c7816e", "onion half4-2-b1fac77b195c0c9b", "onion half5-2-d96b2be8858ccb5d", "wooden spoon-94ecbebb0cd824ee", "recycling container-537c7b667329d769", "lid of recycling container-b1c5daac7234d378", "scales-0bba1e7cc7433c93", "packet-bf2fb3d72df534ed", "ginger-1f3a23ea98451db2", "packet of chilies-fa5b9bca92e5856e", "some chillies-09ba2847a3e6a95d", "chillies2-85ff319d5680249a", "chillies3-85d202ef71da6694", "chillies4-58f8cb4099672dc6", "chillies5-b92fa05df4f8db9a", "bag-537393362b30a33a", "frozen garlic-dc83d5622c1d29d4", "phone-ef96e687dcd5ac8c", "curry powder jar-95d828fcfeccf9b8", "chilli powder jar-9f933647dfda14b4", "packet of smoked paprika-492663d77f14e566", "pot of cinnamon-0ac6a4e87fc7b5d4", "lid of pot of cinnamon-c1ce83cf756d6bca", "cinnamon stick-e116acccb8669545", "lid of pot of cardamon-82e05945d24f5f96", "bowl-43cac69e66c32583", "measuring spoons-0b573d3225382179", "lid of curry powder-cc285f94c1abe8a1", "lid of chilli powder-b3dacd102ffc9fc2", "turmeric jar-adb4a6c789d6885f", "tandoori masala jar-ca968e9a7aaf7891", "lid of turmeric-05e71d5e79a0a283", "lid of tandoori masala jar-36a112b0779c73dc", "spice pot-86eaaeae40a4e360", "lid of spice pot-6854e93c8503014e", "pot of spice-c29290dd6f9b931f", "box of spices-4b476e75b957c453", "plastic lid of spice-3c8ba673f3c60517", "bottle of oil2-2595aa54bcbe3a49", "container of pepper-d7874f1497993af6", "container of salt-15a6da04b95b9de9", "spice packet-cbc48105f820aa94", "pot unknown-6894722cad8e5d03", "another pot unknown-873e993ade7ef26a", "two tins of plum tomatoes-cf08b80ab8c73611", "oven glove-c9ceab02eb1d30de" ], "reasoning": "1. First, I looked for the user to add the first spice to the dish and saw them add a whole cinnamon stick (cinnamon stick-e116acccb8669545) at 12:00. \n2. Next, I looked for the user to add the next spice to the dish and saw them add whole cardamom at 12:07. \n3. I continued watching and saw the user add spices to the dish at 14:42 , 15:15 , 15:52 , and 16:03 . I noticed that every other spice added was ground and not whole. \n4. I concluded that what makes the first two spices unique is that they are whole and not ground.", "referenced_object_masks": [ "cinnamon stick-e116acccb8669545" ] }, { "key": "P05-20240423-170021:5b0b3b3c9f36d050a7ae0be83e5b369a8a129c94", "video_id": "P05-20240423-170021", "question": "When I open the window, where is the drawer I retrieved the knife from in relation to me?", "answer_choice_0": "9 o'clock", "answer_choice_1": "4 o'clock", "answer_choice_2": "11 o'clock", "answer_choice_3": "2 o'clock", "answer_choice_4": "6 o'clock", "answer_id": 0, "answer": "9 o'clock", "question_type": [ "Action Recognition", "Object Recognition", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "pan-4df8c931ca350ae0", "bottle of oil-de702e66317d3296", "chopping board-8f4981c49bb83b03", "knife-f014c71b617dbc9a", "onion1-f24307ea89e936e0", "onion2-13b99426481ecac0", "onion3-1ccd3874246098f1", "onion4-4d22ca85290b353d", "onion5-ff1bffcf90d969db", "onionhalf1-2-4763de74f6508776", "onion half2-2-b3917f702ecb51a4", "onion half3-2-e66c429fe8c7816e", "onion half4-2-b1fac77b195c0c9b", "onion half5-2-d96b2be8858ccb5d", "wooden spoon-94ecbebb0cd824ee", "recycling container-537c7b667329d769", "lid of recycling container-b1c5daac7234d378", "scales-0bba1e7cc7433c93", "packet-bf2fb3d72df534ed", "ginger-1f3a23ea98451db2", "packet of chilies-fa5b9bca92e5856e", "some chillies-09ba2847a3e6a95d", "chillies2-85ff319d5680249a", "chillies3-85d202ef71da6694", "chillies4-58f8cb4099672dc6", "chillies5-b92fa05df4f8db9a", "bag-537393362b30a33a", "frozen garlic-dc83d5622c1d29d4", "phone-ef96e687dcd5ac8c", "curry powder jar-95d828fcfeccf9b8", "chilli powder jar-9f933647dfda14b4", "packet of smoked paprika-492663d77f14e566", "pot of cinnamon-0ac6a4e87fc7b5d4", "lid of pot of cinnamon-c1ce83cf756d6bca", "cinnamon stick-e116acccb8669545", "lid of pot of cardamon-82e05945d24f5f96", "bowl-43cac69e66c32583", "measuring spoons-0b573d3225382179", "lid of curry powder-cc285f94c1abe8a1", "lid of chilli powder-b3dacd102ffc9fc2", "turmeric jar-adb4a6c789d6885f", "tandoori masala jar-ca968e9a7aaf7891", "lid of turmeric-05e71d5e79a0a283", "lid of tandoori masala jar-36a112b0779c73dc", "spice pot-86eaaeae40a4e360", "lid of spice pot-6854e93c8503014e", "pot of spice-c29290dd6f9b931f", "box of spices-4b476e75b957c453", "plastic lid of spice-3c8ba673f3c60517", "bottle of oil2-2595aa54bcbe3a49", "container of pepper-d7874f1497993af6", "container of salt-15a6da04b95b9de9", "spice packet-cbc48105f820aa94", "pot unknown-6894722cad8e5d03", "another pot unknown-873e993ade7ef26a", "two tins of plum tomatoes-cf08b80ab8c73611", "oven glove-c9ceab02eb1d30de" ], "reasoning": "1. First, I looked for a moment where the user opened a window and found they do so at 07:00. \n2. I noticed a knife (knife-f014c71b617dbc9a) on the counter at 07:00 and went back to see when the user retrieved it (knife-f014c71b617dbc9a) from a drawer at 00:48. \n3. Reasoning about the scene, I determined that the drawer from which the user retrieved a knife (knife-f014c71b617dbc9a) is at the user's 9 o'clock when they open the window.", "referenced_object_masks": [ "knife-f014c71b617dbc9a" ] }, { "key": "P05-20240423-170021:a1b852bdb4b13449aa12b9aaf5c8f244ad367eb2", "video_id": "P05-20240423-170021", "question": "If I divide the number of steps I take when I walk to the counter after returning the blue box to its home by the approximate number of seconds it took me to travel, what would the answer be?", "answer_choice_0": "4", "answer_choice_1": "3", "answer_choice_2": "2", "answer_choice_3": "1", "answer_choice_4": "5", "answer_id": 3, "answer": "1", "question_type": [ "Action Recognition", "Temporal Reasoning", "Numerical Reasoning", "Counting", "Object Recognition" ], "objects_of_interest": [ "pan-4df8c931ca350ae0", "bottle of oil-de702e66317d3296", "chopping board-8f4981c49bb83b03", "knife-f014c71b617dbc9a", "onion1-f24307ea89e936e0", "onion2-13b99426481ecac0", "onion3-1ccd3874246098f1", "onion4-4d22ca85290b353d", "onion5-ff1bffcf90d969db", "onionhalf1-2-4763de74f6508776", "onion half2-2-b3917f702ecb51a4", "onion half3-2-e66c429fe8c7816e", "onion half4-2-b1fac77b195c0c9b", "onion half5-2-d96b2be8858ccb5d", "wooden spoon-94ecbebb0cd824ee", "recycling container-537c7b667329d769", "lid of recycling container-b1c5daac7234d378", "scales-0bba1e7cc7433c93", "packet-bf2fb3d72df534ed", "ginger-1f3a23ea98451db2", "packet of chilies-fa5b9bca92e5856e", "some chillies-09ba2847a3e6a95d", "chillies2-85ff319d5680249a", "chillies3-85d202ef71da6694", "chillies4-58f8cb4099672dc6", "chillies5-b92fa05df4f8db9a", "bag-537393362b30a33a", "frozen garlic-dc83d5622c1d29d4", "phone-ef96e687dcd5ac8c", "curry powder jar-95d828fcfeccf9b8", "chilli powder jar-9f933647dfda14b4", "packet of smoked paprika-492663d77f14e566", "pot of cinnamon-0ac6a4e87fc7b5d4", "lid of pot of cinnamon-c1ce83cf756d6bca", "cinnamon stick-e116acccb8669545", "lid of pot of cardamon-82e05945d24f5f96", "bowl-43cac69e66c32583", "measuring spoons-0b573d3225382179", "lid of curry powder-cc285f94c1abe8a1", "lid of chilli powder-b3dacd102ffc9fc2", "turmeric jar-adb4a6c789d6885f", "tandoori masala jar-ca968e9a7aaf7891", "lid of turmeric-05e71d5e79a0a283", "lid of tandoori masala jar-36a112b0779c73dc", "spice pot-86eaaeae40a4e360", "lid of spice pot-6854e93c8503014e", "pot of spice-c29290dd6f9b931f", "box of spices-4b476e75b957c453", "plastic lid of spice-3c8ba673f3c60517", "bottle of oil2-2595aa54bcbe3a49", "container of pepper-d7874f1497993af6", "container of salt-15a6da04b95b9de9", "spice packet-cbc48105f820aa94", "pot unknown-6894722cad8e5d03", "another pot unknown-873e993ade7ef26a", "two tins of plum tomatoes-cf08b80ab8c73611", "oven glove-c9ceab02eb1d30de" ], "reasoning": "1. First, I looked for the moment when the user returns a blue box (box of spices-4b476e75b957c453) to its home and found they place the box (box of spices-4b476e75b957c453) in a cabinet at 17:04. \n2. Next, I counted the number of steps the user took on their way from the cabinet back to the counter and saw them move a foot forward at 17:05 and 17:06. \n3. Then, I looked for the amount of time that passed fromt the moment the user left the cabinet at 17:04 to the moment they arrive at 17:06 and got 2 seconds. \n4. Finally, I divided 2/2 and got 1.", "referenced_object_masks": [ "box of spices-4b476e75b957c453" ] }, { "key": "P05-20240423-170021:d74abdd2387ad4fd672b365144d1331aa62aa37f", "video_id": "P05-20240423-170021", "question": "If I multiply the final weight of the pan after I add the oil to the number of onions I cut in half, what would the total be?", "answer_choice_0": "56", "answer_choice_1": "45", "answer_choice_2": "33", "answer_choice_3": "12", "answer_choice_4": "24", "answer_id": 2, "answer": "33", "question_type": [ "Counting", "Numerical Reasoning", "Action Recognition", "Reading", "Object Recognition" ], "objects_of_interest": [ "pan-4df8c931ca350ae0", "bottle of oil-de702e66317d3296", "chopping board-8f4981c49bb83b03", "knife-f014c71b617dbc9a", "onion1-f24307ea89e936e0", "onion2-13b99426481ecac0", "onion3-1ccd3874246098f1", "onion4-4d22ca85290b353d", "onion5-ff1bffcf90d969db", "onionhalf1-2-4763de74f6508776", "onion half2-2-b3917f702ecb51a4", "onion half3-2-e66c429fe8c7816e", "onion half4-2-b1fac77b195c0c9b", "onion half5-2-d96b2be8858ccb5d", "wooden spoon-94ecbebb0cd824ee", "recycling container-537c7b667329d769", "lid of recycling container-b1c5daac7234d378", "scales-0bba1e7cc7433c93", "packet-bf2fb3d72df534ed", "ginger-1f3a23ea98451db2", "packet of chilies-fa5b9bca92e5856e", "some chillies-09ba2847a3e6a95d", "chillies2-85ff319d5680249a", "chillies3-85d202ef71da6694", "chillies4-58f8cb4099672dc6", "chillies5-b92fa05df4f8db9a", "bag-537393362b30a33a", "frozen garlic-dc83d5622c1d29d4", "phone-ef96e687dcd5ac8c", "curry powder jar-95d828fcfeccf9b8", "chilli powder jar-9f933647dfda14b4", "packet of smoked paprika-492663d77f14e566", "pot of cinnamon-0ac6a4e87fc7b5d4", "lid of pot of cinnamon-c1ce83cf756d6bca", "cinnamon stick-e116acccb8669545", "lid of pot of cardamon-82e05945d24f5f96", "bowl-43cac69e66c32583", "measuring spoons-0b573d3225382179", "lid of curry powder-cc285f94c1abe8a1", "lid of chilli powder-b3dacd102ffc9fc2", "turmeric jar-adb4a6c789d6885f", "tandoori masala jar-ca968e9a7aaf7891", "lid of turmeric-05e71d5e79a0a283", "lid of tandoori masala jar-36a112b0779c73dc", "spice pot-86eaaeae40a4e360", "lid of spice pot-6854e93c8503014e", "pot of spice-c29290dd6f9b931f", "box of spices-4b476e75b957c453", "plastic lid of spice-3c8ba673f3c60517", "bottle of oil2-2595aa54bcbe3a49", "container of pepper-d7874f1497993af6", "container of salt-15a6da04b95b9de9", "spice packet-cbc48105f820aa94", "pot unknown-6894722cad8e5d03", "another pot unknown-873e993ade7ef26a", "two tins of plum tomatoes-cf08b80ab8c73611", "oven glove-c9ceab02eb1d30de" ], "reasoning": "1. First, I found the moment when the user added oil to the pan (pan-4df8c931ca350ae0) at 00:34. I watched the weight on the scale beneath the pan (pan-4df8c931ca350ae0) go up as the user added oil until 00:39. I saw the numbers on the scale go up to 6.6 at 00:41 and did not see the weight increase due to the addition of oil after that moment. \n2. Next, I watched for the user to cut onions in half. I saw them do this at 01:13 (onion5-ff1bffcf90d969db), 01:42 (onion1-f24307ea89e936e0), 02:17 (onion2-13b99426481ecac0), 02:55 (onion3-1ccd3874246098f1), and 03:33 (onion4-4d22ca85290b353d) for a total of 5 times. \n3. Finally, I multipled 5x6.6 and got a total of 33.", "referenced_object_masks": [ "pan-4df8c931ca350ae0", "onion3-1ccd3874246098f1", "onion4-4d22ca85290b353d", "onion5-ff1bffcf90d969db", "onion1-f24307ea89e936e0", "onion2-13b99426481ecac0" ] }, { "key": "P05-20240423-170021:e1d6bc8ed2847776081a8b1cce5ea7e27cae3dec", "video_id": "P05-20240423-170021", "question": "If I am standing facing the stove, what directions would I have to take to end up in front of the fridge?", "answer_choice_0": "Turn left, walk forward, turn left, walk forward, turn left", "answer_choice_1": "Turn around, walk forward, turn left", "answer_choice_2": "Turn around, walk forward", "answer_choice_3": "Turn right, walk forward, turn right, walk forward, turn left", "answer_choice_4": "Turn around, walk forward, turn right", "answer_id": 3, "answer": "Turn right, walk forward, turn right, walk forward, turn left", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "pan-4df8c931ca350ae0", "bottle of oil-de702e66317d3296", "chopping board-8f4981c49bb83b03", "knife-f014c71b617dbc9a", "onion1-f24307ea89e936e0", "onion2-13b99426481ecac0", "onion3-1ccd3874246098f1", "onion4-4d22ca85290b353d", "onion5-ff1bffcf90d969db", "onionhalf1-2-4763de74f6508776", "onion half2-2-b3917f702ecb51a4", "onion half3-2-e66c429fe8c7816e", "onion half4-2-b1fac77b195c0c9b", "onion half5-2-d96b2be8858ccb5d", "wooden spoon-94ecbebb0cd824ee", "recycling container-537c7b667329d769", "lid of recycling container-b1c5daac7234d378", "scales-0bba1e7cc7433c93", "packet-bf2fb3d72df534ed", "ginger-1f3a23ea98451db2", "packet of chilies-fa5b9bca92e5856e", "some chillies-09ba2847a3e6a95d", "chillies2-85ff319d5680249a", "chillies3-85d202ef71da6694", "chillies4-58f8cb4099672dc6", "chillies5-b92fa05df4f8db9a", "bag-537393362b30a33a", "frozen garlic-dc83d5622c1d29d4", "phone-ef96e687dcd5ac8c", "curry powder jar-95d828fcfeccf9b8", "chilli powder jar-9f933647dfda14b4", "packet of smoked paprika-492663d77f14e566", "pot of cinnamon-0ac6a4e87fc7b5d4", "lid of pot of cinnamon-c1ce83cf756d6bca", "cinnamon stick-e116acccb8669545", "lid of pot of cardamon-82e05945d24f5f96", "bowl-43cac69e66c32583", "measuring spoons-0b573d3225382179", "lid of curry powder-cc285f94c1abe8a1", "lid of chilli powder-b3dacd102ffc9fc2", "turmeric jar-adb4a6c789d6885f", "tandoori masala jar-ca968e9a7aaf7891", "lid of turmeric-05e71d5e79a0a283", "lid of tandoori masala jar-36a112b0779c73dc", "spice pot-86eaaeae40a4e360", "lid of spice pot-6854e93c8503014e", "pot of spice-c29290dd6f9b931f", "box of spices-4b476e75b957c453", "plastic lid of spice-3c8ba673f3c60517", "bottle of oil2-2595aa54bcbe3a49", "container of pepper-d7874f1497993af6", "container of salt-15a6da04b95b9de9", "spice packet-cbc48105f820aa94", "pot unknown-6894722cad8e5d03", "another pot unknown-873e993ade7ef26a", "two tins of plum tomatoes-cf08b80ab8c73611", "oven glove-c9ceab02eb1d30de" ], "reasoning": "1. I watched the video to see the positioning of the user when they were standing facing the stove and saw this position at 09:19.\n2. I continued watching the video and saw the user move to the fridge from the stove between 09:20-09:23.\n3. Relative to the position of the user facing the stove , the fridge is across the kitchen to their lower right hand side.\n4. Therefore, the directions the user would have to take to end up in front of the fridge , would be turn right, walk forward, turn right, walk forward, then turn left.", "referenced_object_masks": [] }, { "key": "P05-20240423-170021:e6f7ede7d93d138106f2a1ac8bc2f424b2d67f78", "video_id": "P05-20240423-170021", "question": "What is the main difference between the two vessels that the onion skins and pieces are put into?", "answer_choice_0": "One is deeper than the other", "answer_choice_1": "They are different colors", "answer_choice_2": "They have completely different uses", "answer_choice_3": "One is a bowl and one is a plate", "answer_choice_4": "One is a different shape", "answer_id": 0, "answer": "One is deeper than the other", "question_type": [ "Object Recognition", "Egocentric & Spatial Perception", "Situational Awareness" ], "objects_of_interest": [ "pan-4df8c931ca350ae0", "bottle of oil-de702e66317d3296", "chopping board-8f4981c49bb83b03", "knife-f014c71b617dbc9a", "onion1-f24307ea89e936e0", "onion2-13b99426481ecac0", "onion3-1ccd3874246098f1", "onion4-4d22ca85290b353d", "onion5-ff1bffcf90d969db", "onionhalf1-2-4763de74f6508776", "onion half2-2-b3917f702ecb51a4", "onion half3-2-e66c429fe8c7816e", "onion half4-2-b1fac77b195c0c9b", "onion half5-2-d96b2be8858ccb5d", "wooden spoon-94ecbebb0cd824ee", "recycling container-537c7b667329d769", "lid of recycling container-b1c5daac7234d378", "scales-0bba1e7cc7433c93", "packet-bf2fb3d72df534ed", "ginger-1f3a23ea98451db2", "packet of chilies-fa5b9bca92e5856e", "some chillies-09ba2847a3e6a95d", "chillies2-85ff319d5680249a", "chillies3-85d202ef71da6694", "chillies4-58f8cb4099672dc6", "chillies5-b92fa05df4f8db9a", "bag-537393362b30a33a", "frozen garlic-dc83d5622c1d29d4", "phone-ef96e687dcd5ac8c", "curry powder jar-95d828fcfeccf9b8", "chilli powder jar-9f933647dfda14b4", "packet of smoked paprika-492663d77f14e566", "pot of cinnamon-0ac6a4e87fc7b5d4", "lid of pot of cinnamon-c1ce83cf756d6bca", "cinnamon stick-e116acccb8669545", "lid of pot of cardamon-82e05945d24f5f96", "bowl-43cac69e66c32583", "measuring spoons-0b573d3225382179", "lid of curry powder-cc285f94c1abe8a1", "lid of chilli powder-b3dacd102ffc9fc2", "turmeric jar-adb4a6c789d6885f", "tandoori masala jar-ca968e9a7aaf7891", "lid of turmeric-05e71d5e79a0a283", "lid of tandoori masala jar-36a112b0779c73dc", "spice pot-86eaaeae40a4e360", "lid of spice pot-6854e93c8503014e", "pot of spice-c29290dd6f9b931f", "box of spices-4b476e75b957c453", "plastic lid of spice-3c8ba673f3c60517", "bottle of oil2-2595aa54bcbe3a49", "container of pepper-d7874f1497993af6", "container of salt-15a6da04b95b9de9", "spice packet-cbc48105f820aa94", "pot unknown-6894722cad8e5d03", "another pot unknown-873e993ade7ef26a", "two tins of plum tomatoes-cf08b80ab8c73611", "oven glove-c9ceab02eb1d30de" ], "reasoning": "1. I watched the video to see what vessel the user put the onion pieces into and saw this at 04:42 when the user scraped them off a cutting board (chopping board-8f4981c49bb83b03) and into a pan (pan-4df8c931ca350ae0).\n2. I then watched the video to see what vessel the user put the onions skins into and saw this at 08:48 when the user put them into a plastic container (recycling container-537c7b667329d769).\n3. Looking at these two vessels, the main difference between the pan (pan-4df8c931ca350ae0) and the plastic container (recycling container-537c7b667329d769) is that one is deeper than the other.", "referenced_object_masks": [ "pan-4df8c931ca350ae0", "chopping board-8f4981c49bb83b03", "recycling container-537c7b667329d769" ] }, { "key": "P05-20240423-170021:fe4d99a55b1496a81837acd390ed105d8d8e32eb", "video_id": "P05-20240423-170021", "question": "What is the second thing I do directly after retrieving the third spice jar?", "answer_choice_0": "Look at the phone", "answer_choice_1": "Stir the onions", "answer_choice_2": "Cut a pepper", "answer_choice_3": "Open the window", "answer_choice_4": "Retrieve another spice jar", "answer_id": 0, "answer": "Look at the phone", "question_type": [ "Counting", "Action Recognition", "Temporal Reasoning", "Object Recognition" ], "objects_of_interest": [ "pan-4df8c931ca350ae0", "bottle of oil-de702e66317d3296", "chopping board-8f4981c49bb83b03", "knife-f014c71b617dbc9a", "onion1-f24307ea89e936e0", "onion2-13b99426481ecac0", "onion3-1ccd3874246098f1", "onion4-4d22ca85290b353d", "onion5-ff1bffcf90d969db", "onionhalf1-2-4763de74f6508776", "onion half2-2-b3917f702ecb51a4", "onion half3-2-e66c429fe8c7816e", "onion half4-2-b1fac77b195c0c9b", "onion half5-2-d96b2be8858ccb5d", "wooden spoon-94ecbebb0cd824ee", "recycling container-537c7b667329d769", "lid of recycling container-b1c5daac7234d378", "scales-0bba1e7cc7433c93", "packet-bf2fb3d72df534ed", "ginger-1f3a23ea98451db2", "packet of chilies-fa5b9bca92e5856e", "some chillies-09ba2847a3e6a95d", "chillies2-85ff319d5680249a", "chillies3-85d202ef71da6694", "chillies4-58f8cb4099672dc6", "chillies5-b92fa05df4f8db9a", "bag-537393362b30a33a", "frozen garlic-dc83d5622c1d29d4", "phone-ef96e687dcd5ac8c", "curry powder jar-95d828fcfeccf9b8", "chilli powder jar-9f933647dfda14b4", "packet of smoked paprika-492663d77f14e566", "pot of cinnamon-0ac6a4e87fc7b5d4", "lid of pot of cinnamon-c1ce83cf756d6bca", "cinnamon stick-e116acccb8669545", "lid of pot of cardamon-82e05945d24f5f96", "bowl-43cac69e66c32583", "measuring spoons-0b573d3225382179", "lid of curry powder-cc285f94c1abe8a1", "lid of chilli powder-b3dacd102ffc9fc2", "turmeric jar-adb4a6c789d6885f", "tandoori masala jar-ca968e9a7aaf7891", "lid of turmeric-05e71d5e79a0a283", "lid of tandoori masala jar-36a112b0779c73dc", "spice pot-86eaaeae40a4e360", "lid of spice pot-6854e93c8503014e", "pot of spice-c29290dd6f9b931f", "box of spices-4b476e75b957c453", "plastic lid of spice-3c8ba673f3c60517", "bottle of oil2-2595aa54bcbe3a49", "container of pepper-d7874f1497993af6", "container of salt-15a6da04b95b9de9", "spice packet-cbc48105f820aa94", "pot unknown-6894722cad8e5d03", "another pot unknown-873e993ade7ef26a", "two tins of plum tomatoes-cf08b80ab8c73611", "oven glove-c9ceab02eb1d30de" ], "reasoning": "1. First, I looked for the user to begin retrieving spice jars and found they retrieve the first one at 10:48 (curry powder jar-95d828fcfeccf9b8), the second at 10:53 (chilli powder jar-9f933647dfda14b4), and the third at 11:11. \n2. Next, I looked for the user to perform an action and saw them stir the onions at 11:17. \n3. Finally, I looked for them to perform a second action and saw them pick up a phone (phone-ef96e687dcd5ac8c) and look at it at 11:27.", "referenced_object_masks": [ "chilli powder jar-9f933647dfda14b4", "phone-ef96e687dcd5ac8c", "curry powder jar-95d828fcfeccf9b8" ] }, { "key": "P05-20240423-172243:261a8e743d7e810e3f60afed83e4a983e5525143", "video_id": "P05-20240423-172243", "question": "What is the second to last thing I did before washing my hands at the end of the video?", "answer_choice_0": "Put the bowl in the dishwasher", "answer_choice_1": "Clean the stove", "answer_choice_2": "Put the mango chutney in the fridge", "answer_choice_3": "Put the pan in the oven", "answer_choice_4": "Clean the kitchen counter", "answer_id": 0, "answer": "Put the bowl in the dishwasher", "question_type": [ "Object Recognition", "Action Recognition", "Event Occurence" ], "objects_of_interest": [ "oven glove-75f3c147808c6871", "pan-faba13013c5fcebf", "wooden spoon-019b03326f3e107f", "bowl-6c7e11ef5294c3b3", "tin of tomatoes-5df4a9a0e1699495", "top of can-3ed24c0444fcdfff", "tin of tomatoes2-8eb03077a10f6199", "top of can2-98242a3326c1c25c", "kettle-2cacde035718b68e", "pot of mango chutney-8b3738e9a560ae30", "lid of mango chutney-18aa6765c3138e86", "measuring spoon-6a9ffef6eb2ca7e0", "lid of pan-782dcaf9c04a49ed", "phone-3e0c7e54ce563cbb" ], "reasoning": "1. I looked for the moment the user washes their hands at 05:33. \n2. I scrolled back to see what the user does before washing their hands. At 05:29, the user opens the fridge to put the mango chutney jar (pot of mango chutney-8b3738e9a560ae30) in the fridge. This is the last thing the user did before washing their hands. \n3. I scrolled back once more and watched as the user placed an empty bowl (bowl-6c7e11ef5294c3b3) in the dishwasher at 05:24. Therefore, the second to last thing the user did before washing their hands was putting the bowl (bowl-6c7e11ef5294c3b3) in the dishwasher.", "referenced_object_masks": [ "pot of mango chutney-8b3738e9a560ae30", "bowl-6c7e11ef5294c3b3" ] }, { "key": "P05-20240423-172243:5ad1d6decbc369d775e62b3095858e2b9270a23c", "video_id": "P05-20240423-172243", "question": "Approximately how much mango chutney do I need if I\u2019m only adding one can of tomatoes to the recipe?", "answer_choice_0": "2 tablespoons", "answer_choice_1": "1 tablespoon", "answer_choice_2": "1 and a half tablespoons", "answer_choice_3": "A teaspooon", "answer_choice_4": "Half a tablespoon", "answer_id": 1, "answer": "1 tablespoon", "question_type": [ "Object Recognition", "Action Recognition", "Event Occurence", "Counting", "Numerical Reasoning" ], "objects_of_interest": [ "oven glove-75f3c147808c6871", "pan-faba13013c5fcebf", "wooden spoon-019b03326f3e107f", "bowl-6c7e11ef5294c3b3", "tin of tomatoes-5df4a9a0e1699495", "top of can-3ed24c0444fcdfff", "tin of tomatoes2-8eb03077a10f6199", "top of can2-98242a3326c1c25c", "kettle-2cacde035718b68e", "pot of mango chutney-8b3738e9a560ae30", "lid of mango chutney-18aa6765c3138e86", "measuring spoon-6a9ffef6eb2ca7e0", "lid of pan-782dcaf9c04a49ed", "phone-3e0c7e54ce563cbb" ], "reasoning": "1. I watched the video to locate the moment the user adds ingredients to the pan (pan-faba13013c5fcebf), which happens from 00:51-02:52.\n2. The ingredients the user adds to the pan are: a bowl (bowl-6c7e11ef5294c3b3) of seasonings at 00:51, followed by 2 cans of crushed tomatoes (tin of tomatoes-5df4a9a0e1699495, tin of tomatoes2-8eb03077a10f6199) at 01:31 and 01:44, and then the mango chutney (pot of mango chutney-8b3738e9a560ae30) at 02:52. \n3. The user adds 1 tablespoon plus about half of another tablespoon of mango chutney (pot of mango chutney-8b3738e9a560ae30) to the dish using a measuring spoon (measuring spoon-6a9ffef6eb2ca7e0), from 02:52 to 03:00. \n4. The user then stirs the food from 03:00 to 03:33 and tastes it with a wooden spoon (wooden spoon-019b03326f3e107f) at 03:34. \n5. After the user tries the food at 03:34, they decide to add approximately another half tablespoon of mango chutney (pot of mango chutney-8b3738e9a560ae30) to the dish at 03:50. This is the last ingredient the user adds to the dish before covering the pan (pan-faba13013c5fcebf) with a lid (lid of pan-782dcaf9c04a49ed) at 04:30.\n6. Based on the information gathered from the video, the recipe calls for 1 tablespoon (measuring spoon-6a9ffef6eb2ca7e0) of mango chutney (pot of mango chutney-8b3738e9a560ae30) for each can of crushed tomatoes (tin of tomatoes-5df4a9a0e1699495, tin of tomatoes2-8eb03077a10f6199). Therefore, if the user is only using 1 can of tomatoes (tin of tomatoes-5df4a9a0e1699495) for the recipe, then it is safe to assume that it would need 1 tablespoon (measuring spoon-6a9ffef6eb2ca7e0) of mango chutney (pot of mango chutney-8b3738e9a560ae30).", "referenced_object_masks": [ "pan-faba13013c5fcebf", "wooden spoon-019b03326f3e107f", "lid of pan-782dcaf9c04a49ed", "tin of tomatoes-5df4a9a0e1699495", "tin of tomatoes2-8eb03077a10f6199", "measuring spoon-6a9ffef6eb2ca7e0", "pot of mango chutney-8b3738e9a560ae30", "bowl-6c7e11ef5294c3b3" ] }, { "key": "P05-20240423-172243:827a12cd97708a9728f030e38b2278630ddc60f0", "video_id": "P05-20240423-172243", "question": "If I remove the empty can of tomatoes from the kitchen counter, what object would be the closest to the left of the mango chutney lid?", "answer_choice_0": "Cell phone", "answer_choice_1": "Yellow glove", "answer_choice_2": "Sugar container", "answer_choice_3": "Green plant", "answer_choice_4": "Top of the can", "answer_id": 4, "answer": "Top of the can", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning", "Event Occurence" ], "objects_of_interest": [ "oven glove-75f3c147808c6871", "pan-faba13013c5fcebf", "wooden spoon-019b03326f3e107f", "bowl-6c7e11ef5294c3b3", "tin of tomatoes-5df4a9a0e1699495", "top of can-3ed24c0444fcdfff", "tin of tomatoes2-8eb03077a10f6199", "top of can2-98242a3326c1c25c", "kettle-2cacde035718b68e", "pot of mango chutney-8b3738e9a560ae30", "lid of mango chutney-18aa6765c3138e86", "measuring spoon-6a9ffef6eb2ca7e0", "lid of pan-782dcaf9c04a49ed", "phone-3e0c7e54ce563cbb" ], "reasoning": "1. I watched the video to locate the moment the user places the mango chutney lid (lid of mango chutney-18aa6765c3138e86) on the kitchen counter. This happens after opening the jar of mango chutney (pot of mango chutney-8b3738e9a560ae30) at 02:46.\n2. I noticed a few items around the lid (lid of mango chutney-18aa6765c3138e86) at 02:46. There's the empty can of tomatoes (tin of tomatoes-5df4a9a0e1699495) to the left of the lid (lid of mango chutney-18aa6765c3138e86), an empty bowl (bowl-6c7e11ef5294c3b3) almost parallel to the lid (lid of mango chutney-18aa6765c3138e86), and a glass container with sugar to the right. \n3. To the left of the empty can of tomatoes (tin of tomatoes-5df4a9a0e1699495), there's the top of the opened can (top of can2-98242a3326c1c25c). \n3. If the user removes the empty can of tomatoes (tin of tomatoes-5df4a9a0e1699495) from the kitchen counter, then the top of the opened can (top of can2-98242a3326c1c25c) would be the closest item to the left of the mango chutney lid (lid of mango chutney-18aa6765c3138e86).", "referenced_object_masks": [ "bowl-6c7e11ef5294c3b3", "lid of mango chutney-18aa6765c3138e86", "top of can2-98242a3326c1c25c", "pot of mango chutney-8b3738e9a560ae30", "tin of tomatoes-5df4a9a0e1699495" ] }, { "key": "P05-20240423-172243:c1e60709f47ba0543219379c9eb0deaa087aca3c", "video_id": "P05-20240423-172243", "question": "How many times do I shake the can of tomatoes over the dish before pouring water into it?", "answer_choice_0": "One", "answer_choice_1": "Five", "answer_choice_2": "Four", "answer_choice_3": "Three", "answer_choice_4": "Two", "answer_id": 2, "answer": "Four", "question_type": [ "Object Recognition", "Action Recognition", "Counting", "Event Occurence" ], "objects_of_interest": [ "oven glove-75f3c147808c6871", "pan-faba13013c5fcebf", "wooden spoon-019b03326f3e107f", "bowl-6c7e11ef5294c3b3", "tin of tomatoes-5df4a9a0e1699495", "top of can-3ed24c0444fcdfff", "tin of tomatoes2-8eb03077a10f6199", "top of can2-98242a3326c1c25c", "kettle-2cacde035718b68e", "pot of mango chutney-8b3738e9a560ae30", "lid of mango chutney-18aa6765c3138e86", "measuring spoon-6a9ffef6eb2ca7e0", "lid of pan-782dcaf9c04a49ed", "phone-3e0c7e54ce563cbb" ], "reasoning": "1. I located the moment where the user adds water (kettle-2cacde035718b68e) to the can of tomatoes (tin of tomatoes2-8eb03077a10f6199) at 01:53. \n2. I scrolled back to locate the moment right before the user poured water (kettle-2cacde035718b68e) into the can (tin of tomatoes2-8eb03077a10f6199) and found it at 01:44, where the user is pouring the second can of tomatoes (tin of tomatoes2-8eb03077a10f6199) into the pan (pan-faba13013c5fcebf).\n4. After pouring the tomatoes (tin of tomatoes2-8eb03077a10f6199) into the pan (pan-faba13013c5fcebf), I counted the number of times the user shook the can (tin of tomatoes2-8eb03077a10f6199) over the dish before walking to the kettle (kettle-2cacde035718b68e). The user shook the can (tin of tomatoes2-8eb03077a10f6199) over the dish four times, from 01:45 to 01:48.", "referenced_object_masks": [ "kettle-2cacde035718b68e", "pan-faba13013c5fcebf", "tin of tomatoes2-8eb03077a10f6199" ] }, { "key": "P05-20240423-172243:f1d9759daca8debeb8a43e6ada0c06cb8c023c11", "video_id": "P05-20240423-172243", "question": "After stirring the food for the first time, what is the second thing I grab with my right hand?", "answer_choice_0": "Measuring spoons", "answer_choice_1": "Bowl of seasonings", "answer_choice_2": "Yellow glove", "answer_choice_3": "Wooden spoon", "answer_choice_4": "Can of tomatoes", "answer_id": 3, "answer": "Wooden spoon", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Event Occurence" ], "objects_of_interest": [ "oven glove-75f3c147808c6871", "pan-faba13013c5fcebf", "wooden spoon-019b03326f3e107f", "bowl-6c7e11ef5294c3b3", "tin of tomatoes-5df4a9a0e1699495", "top of can-3ed24c0444fcdfff", "tin of tomatoes2-8eb03077a10f6199", "top of can2-98242a3326c1c25c", "kettle-2cacde035718b68e", "pot of mango chutney-8b3738e9a560ae30", "lid of mango chutney-18aa6765c3138e86", "measuring spoon-6a9ffef6eb2ca7e0", "lid of pan-782dcaf9c04a49ed", "phone-3e0c7e54ce563cbb" ], "reasoning": "1. I watched for the moment the user stirred the food in the pan (pan-faba13013c5fcebf) for the first time, which happened at 00:19. \n2. I continue to watch to see what items the user grabbed with the right hand after releasing the spoon (wooden spoon-019b03326f3e107f) used to stir the food. At 00:46, the user releases the spoon (wooden spoon-019b03326f3e107f) and moves towards the counter. The first item the user grabs with the right hand is the bowl with seasonings (bowl-6c7e11ef5294c3b3) at 00:48. \n3. The user then moves back to the front of the stove to pour the seasoning into the dish (pan-faba13013c5fcebf), and grabs the wooden spoon (wooden spoon-019b03326f3e107f) resting on the pan (pan-faba13013c5fcebf) at 00:51. Therefore, the second item the user grabbed with the right hand after stirring the dish for the first time is the wooden spoon (wooden spoon-019b03326f3e107f).", "referenced_object_masks": [ "pan-faba13013c5fcebf", "wooden spoon-019b03326f3e107f", "bowl-6c7e11ef5294c3b3" ] }, { "key": "P05-20240423-175745:2a975e1b5e6321d99eaf1bc32d137c6f25b61359", "video_id": "P05-20240423-175745", "question": "What is the difference between the number of times I opened an oven door with a yellow mit and the number of visible steps I took forward with my right foot between closing the kitchen door and reaching the dark granite counter?", "answer_choice_0": "5", "answer_choice_1": "7", "answer_choice_2": "0", "answer_choice_3": "2", "answer_choice_4": "4", "answer_id": 3, "answer": "2", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Temporal Reasoning", "Action Recognition", "Counting" ], "objects_of_interest": [ "oven glove-454892ee01fa96ff", "pan-c93688d2411ff889", "wooden spoon-16f503c47b8913c5", "lid of pan-d362a4798e13ec6a" ], "reasoning": "1. I watched the whole video and saw the user open the oven door with a yellow mit (oven glove-454892ee01fa96ff) 3 times at 00:10, 01:19 and 01:32.\n2. At 00:05-00:06, I saw the user close the kitchen door and take 1 visible step forward with their right foot in order to reach the dark granite counter.\n3. Finally, to find the difference, I subtracted 1 step from the 3 times the user opened the oven door with the yellow mit (oven glove-454892ee01fa96ff) and got a total of 2.", "referenced_object_masks": [ "oven glove-454892ee01fa96ff" ] }, { "key": "P05-20240423-175745:30a58d9db0cb4b2fda52fd05e494634cf00875c6", "video_id": "P05-20240423-175745", "question": "Where is the kitchen table in relation to me when I am working on my sauce?", "answer_choice_0": "To my left", "answer_choice_1": "Behind me and to my left", "answer_choice_2": "To my right", "answer_choice_3": "Behind me and to my right", "answer_choice_4": "Directly behind me", "answer_id": 3, "answer": "Behind me and to my right", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Event Occurence", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "oven glove-454892ee01fa96ff", "pan-c93688d2411ff889", "wooden spoon-16f503c47b8913c5", "lid of pan-d362a4798e13ec6a" ], "reasoning": "1. I watched the whole video and noticed when the user walks into the kitchen at 00:02, the kitchen table is in front of them.\n2. At 00:25, the user starts stirring and working on their sauce . \n3. At 01:28, the user puts the sauce back in the oven .\n4. At 01:31, the user adjusts the pan (pan-c93688d2411ff889) so it will fit in the oven.\n5. At 01:31, the back legs of one of the kitchen chairs can be seen to the right of the user.\n6. So the kitchen table is behind and to the right of the user when working on their sauce at the stove .", "referenced_object_masks": [ "pan-c93688d2411ff889" ] }, { "key": "P05-20240423-175745:8132b362851ee2aa680988f71dc7fa65f6907155", "video_id": "P05-20240423-175745", "question": "If I take the number of seconds I spent stirring the sauce at the stove, add to it the number of times I hit the wooden spoon on the side of the pan, and then divide this number by the number of black knobs on the cabinets below the counter by the window, what number would I get, rounded to the nearest whole number?", "answer_choice_0": "6", "answer_choice_1": "7", "answer_choice_2": "9", "answer_choice_3": "3", "answer_choice_4": "4", "answer_id": 0, "answer": "6", "question_type": [ "Action Recognition", "Counting", "Egocentric & Spatial Perception", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "oven glove-454892ee01fa96ff", "pan-c93688d2411ff889", "wooden spoon-16f503c47b8913c5", "lid of pan-d362a4798e13ec6a" ], "reasoning": "1. I watched the whole video and found the user stirred their sauce from 0:25 to 1:07 which is 42 seconds.\n2. I continued to watch to see the user tap the wooden spoon (wooden spoon-16f503c47b8913c5) on the side of the pan (pan-c93688d2411ff889) 8 times beginning at 01:08.\n3. I determined that 43 seconds + 7 taps = 50.\n4. At 0:05 when the user is walking into the kitchen, the cabinets under the black counter can be seen. There are 9 black knobs .\n5. 50 divided by 9 is 5.5 repeating.\n6. Rounding 5.5 repeating to the nearest whole number is 6.", "referenced_object_masks": [ "pan-c93688d2411ff889", "wooden spoon-16f503c47b8913c5" ] }, { "key": "P05-20240423-175745:8991b4c557edae80724e4f4b90b5f0639e3373de", "video_id": "P05-20240423-175745", "question": "If you added the number of times I jerked the pan lid in my hand to the number of plants on the black counter in front of the windows, and then raised that number to the 3rd power, what number would you get?", "answer_choice_0": "64", "answer_choice_1": "125", "answer_choice_2": "216", "answer_choice_3": "343", "answer_choice_4": "27", "answer_id": 3, "answer": "343", "question_type": [ "Action Recognition", "Counting", "Numerical Reasoning", "Egocentric & Spatial Perception", "Temporal Reasoning" ], "objects_of_interest": [ "oven glove-454892ee01fa96ff", "pan-c93688d2411ff889", "wooden spoon-16f503c47b8913c5", "lid of pan-d362a4798e13ec6a" ], "reasoning": "1. I watched the whole video and saw the user jerk the pan lid (lid of pan-d362a4798e13ec6a) in their hand 6 times from 00:21-00:22.\n2. At 0:05 as the user is walking to the black counter , there is 1 plant on the counter near the window .\n3. The number of times the pan lid (lid of pan-d362a4798e13ec6a) is shaken plus the number of plants on the black counter is 7.\n4. 7^3 is 343.", "referenced_object_masks": [ "lid of pan-d362a4798e13ec6a" ] }, { "key": "P05-20240423-175745:c17dbeba70966118d273973ece7876400a0a26ce", "video_id": "P05-20240423-175745", "question": "How many more kitchen chairs are at the kitchen table than the number of items on the cutting board after I put the stew back in the oven?", "answer_choice_0": "4", "answer_choice_1": "0", "answer_choice_2": "3", "answer_choice_3": "2", "answer_choice_4": "1", "answer_id": 1, "answer": "0", "question_type": [ "Counting", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "oven glove-454892ee01fa96ff", "pan-c93688d2411ff889", "wooden spoon-16f503c47b8913c5", "lid of pan-d362a4798e13ec6a" ], "reasoning": "1. I watched the whole video and immediately saw at 0:02 that there were 3 chairs at the kitchen table .\n2. Then, I looked for the moment that the user put the pan with the stew (pan-c93688d2411ff889) back into the oven. I found this moment at 01:27.\n3. Then, I tried to determine how many items were on the cutting board after 01:27. The cutting board is not immediately visible at 01:27, but it came into view at 01:35. At this time, there was a wooden spoon (wooden spoon-16f503c47b8913c5), a knife and a measuring spoon on top of the cutting board . This totals 3 items.\n4. I subtracted 3 from 3 to get a total of 0.", "referenced_object_masks": [ "wooden spoon-16f503c47b8913c5", "pan-c93688d2411ff889" ] }, { "key": "P05-20240423-180529:1fc0947ace7f60077ddb4e6b966ab2c2c0892396", "video_id": "P05-20240423-180529", "question": "After I close the kitchen door, what material is the third item I touch with my right hand made of?", "answer_choice_0": "Ceramic", "answer_choice_1": "Cloth", "answer_choice_2": "Wood", "answer_choice_3": "Metal", "answer_choice_4": "Plastic", "answer_id": 1, "answer": "Cloth", "question_type": [ "Counting", "Event Occurence", "Action Recognition" ], "objects_of_interest": [ "kettle-452a2b894cb0c2a8", "bowl of water-afbcc69f996972d5", "cloth-02f0694594fca358", "oven glove-1e7582ed39c9b3e3", "pan-7e589b2a6101695b", "pan lid-ba9b640370e6ad72", "wooden spoon-275666359a0d7d14", "tins of chickpeas-e2f636cab57a361b", "tins of chickpeas2-7e413874b297c96a", "pot-9263bdfa4ea8d66d", "pot of salt-e8a502e14698f73c", "pot lid-892b74e11c774a60", "packet of rice-1aa563779ca757c5", "mug-f02aeeda1b85ce7e", "phone-acec62c0c30bb2a1", "bag of spinach-a3c73edd5fc497cc", "spinach-8256ea2714aa9d25", "spinach2-758781ca50c1b51d", "spinach3-318b6435b072df84", "spinach4-22f30708916d0f88", "spinach5-b6856db8e92865f4", "spinach6-38cf4de8b40e4d54", "spinach7-79d229c829e6b82d", "bag of rice-2448643751874ce7", "bag of green things-73eb2e73e56fa667", "measuring spoon-dd99b96849fe7663", "knife-6271397240d2ab02", "colander-611a17072081e978", "colander2-8f9fc4463f3722dd", "lid of tin-81f797ef6a69874f", "scale-77e26cf0733c390b", "containers-fe81ebe775f705bd", "yoghurt-56f0bdd5387b25bf", "coconut milk tin-d8f35f562a62487c", "spoon-2537cd8d327de287", "lemon-23a5c76a1c89de04", "second half of lemon-7b2a0b9f2ec5188e", "measuring spoon2-06446d8dc337f3d4", "glass-681ef44efd9eb8e6", "plate-4469359f1a382485", "spoon2-35e002beea08de16", "spoon3-5b4632657a4b7d41", "chopping board-62a08cef24d2d329", "oven glove2-0324a8806743b71f", "plate2-561442093803b3c2", "brush-308a27d2f5a57ab4", "towel-bba79bcc4f0ebe85", "skewer-62ef3ed7639fb6e5", "spoon4-2891703779d733e6" ], "reasoning": "1. I first watch the video to determine when the kitchen door is closed: this occurs at 00:04. After this, I observe the actions of the right hand of the user, counting for the third thing I see them touch.\n2. The first thing I see the user touch with their right hand after closing the kitchen door is a coffee pot (kettle-452a2b894cb0c2a8) at 00:08.\n3. The second thing I see the user touch with their right hand after closing the kitchen door is a plastic bin full of water in the sink (bowl of water-afbcc69f996972d5) at 00:19.\n4. The third thing I see the user touch with their right hand after closing the kitchen door is a wet rag (cloth-02f0694594fca358) that they dumped out of the plastic bin (bowl of water-afbcc69f996972d5) and into the sink; they pick this rag up at 00:26.\n5. I identify that the wet rag (cloth-02f0694594fca358) is made of cloth, making C the correct answer to the question.", "referenced_object_masks": [ "kettle-452a2b894cb0c2a8", "bowl of water-afbcc69f996972d5", "cloth-02f0694594fca358" ] }, { "key": "P05-20240423-180529:42326335fb7144e2eccb1dbf856d3084bad77713", "video_id": "P05-20240423-180529", "question": "What steps would I need to take to move from the spot where I cut the lemon to where I threw away the empty chickpea cans?", "answer_choice_0": "Turn left, walk forward, turn left, and then turn left again", "answer_choice_1": "Turn around, walk forward, turn right, and walk forward again", "answer_choice_2": "Turn left, walk forward, turn right, and turn right again", "answer_choice_3": "Turn around, walk forward, turn right, and turn right again", "answer_choice_4": "Turn right, walk forward, turn left, and walk forward again", "answer_id": 3, "answer": "Turn around, walk forward, turn right, and turn right again", "question_type": [ "Counterfactual", "Egocentric & Spatial Perception", "Situational Awareness", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "kettle-452a2b894cb0c2a8", "bowl of water-afbcc69f996972d5", "cloth-02f0694594fca358", "oven glove-1e7582ed39c9b3e3", "pan-7e589b2a6101695b", "pan lid-ba9b640370e6ad72", "wooden spoon-275666359a0d7d14", "tins of chickpeas-e2f636cab57a361b", "tins of chickpeas2-7e413874b297c96a", "pot-9263bdfa4ea8d66d", "pot of salt-e8a502e14698f73c", "pot lid-892b74e11c774a60", "packet of rice-1aa563779ca757c5", "mug-f02aeeda1b85ce7e", "phone-acec62c0c30bb2a1", "bag of spinach-a3c73edd5fc497cc", "spinach-8256ea2714aa9d25", "spinach2-758781ca50c1b51d", "spinach3-318b6435b072df84", "spinach4-22f30708916d0f88", "spinach5-b6856db8e92865f4", "spinach6-38cf4de8b40e4d54", "spinach7-79d229c829e6b82d", "bag of rice-2448643751874ce7", "bag of green things-73eb2e73e56fa667", "measuring spoon-dd99b96849fe7663", "knife-6271397240d2ab02", "colander-611a17072081e978", "colander2-8f9fc4463f3722dd", "lid of tin-81f797ef6a69874f", "scale-77e26cf0733c390b", "containers-fe81ebe775f705bd", "yoghurt-56f0bdd5387b25bf", "coconut milk tin-d8f35f562a62487c", "spoon-2537cd8d327de287", "lemon-23a5c76a1c89de04", "second half of lemon-7b2a0b9f2ec5188e", "measuring spoon2-06446d8dc337f3d4", "glass-681ef44efd9eb8e6", "plate-4469359f1a382485", "spoon2-35e002beea08de16", "spoon3-5b4632657a4b7d41", "chopping board-62a08cef24d2d329", "oven glove2-0324a8806743b71f", "plate2-561442093803b3c2", "brush-308a27d2f5a57ab4", "towel-bba79bcc4f0ebe85", "skewer-62ef3ed7639fb6e5", "spoon4-2891703779d733e6" ], "reasoning": "1. I watched the video and looked for when the user cut a lemon. At 12:12, I noticed the user grabbed a lemon (lemon-23a5c76a1c89de04) and brought it over to the counter.\n2. At 12:26, I saw the user use the knife (knife-6271397240d2ab02) to cut the lemon (lemon-23a5c76a1c89de04).\n3. From here, I tried to determine what steps the user would need to take to find the spot where they threw away chickpea cans (tins of chickpeas-e2f636cab57a361b, tins of chickpeas2-7e413874b297c96a). Moving back in the video, I found that the user threw these cans (tins of chickpeas-e2f636cab57a361b, tins of chickpeas2-7e413874b297c96a) away at 09:13.\n4. Just after the user threw the cans (tins of chickpeas-e2f636cab57a361b, tins of chickpeas2-7e413874b297c96a) away at 09:13, I noticed the user bent upward from their location at 09:14, which allowed them to see much of the rest of the kitchen.\n5. From this shot at 09:14, the counter where the user will eventually cut the lemon (lemon-23a5c76a1c89de04) is visible in the back, which subsequently shows what actions the user would need to take to get from there to the spot where they threw away the cans (tins of chickpeas-e2f636cab57a361b, tins of chickpeas2-7e413874b297c96a). From the counter, they would need to turn around, walk forward toward the sink, turn right, and then turn right again.", "referenced_object_masks": [ "lemon-23a5c76a1c89de04", "knife-6271397240d2ab02", "tins of chickpeas-e2f636cab57a361b", "tins of chickpeas2-7e413874b297c96a" ] }, { "key": "P05-20240423-180529:47cdc79f77589a49d31d91ad911a12988855b572", "video_id": "P05-20240423-180529", "question": "Which of the following is not one of the actions I performed in the interim between the first and second times I wore the yellow oven mitt?", "answer_choice_0": "Drained chickpeas", "answer_choice_1": ", Took pot off shelf", "answer_choice_2": "Cleaned counter space", "answer_choice_3": "Refilled coffee pot", "answer_choice_4": "Looked at phone", "answer_id": 3, "answer": "Refilled coffee pot", "question_type": [ "Egocentric & Spatial Perception", "Action Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "kettle-452a2b894cb0c2a8", "bowl of water-afbcc69f996972d5", "cloth-02f0694594fca358", "oven glove-1e7582ed39c9b3e3", "pan-7e589b2a6101695b", "pan lid-ba9b640370e6ad72", "wooden spoon-275666359a0d7d14", "tins of chickpeas-e2f636cab57a361b", "tins of chickpeas2-7e413874b297c96a", "pot-9263bdfa4ea8d66d", "pot of salt-e8a502e14698f73c", "pot lid-892b74e11c774a60", "packet of rice-1aa563779ca757c5", "mug-f02aeeda1b85ce7e", "phone-acec62c0c30bb2a1", "bag of spinach-a3c73edd5fc497cc", "spinach-8256ea2714aa9d25", "spinach2-758781ca50c1b51d", "spinach3-318b6435b072df84", "spinach4-22f30708916d0f88", "spinach5-b6856db8e92865f4", "spinach6-38cf4de8b40e4d54", "spinach7-79d229c829e6b82d", "bag of rice-2448643751874ce7", "bag of green things-73eb2e73e56fa667", "measuring spoon-dd99b96849fe7663", "knife-6271397240d2ab02", "colander-611a17072081e978", "colander2-8f9fc4463f3722dd", "lid of tin-81f797ef6a69874f", "scale-77e26cf0733c390b", "containers-fe81ebe775f705bd", "yoghurt-56f0bdd5387b25bf", "coconut milk tin-d8f35f562a62487c", "spoon-2537cd8d327de287", "lemon-23a5c76a1c89de04", "second half of lemon-7b2a0b9f2ec5188e", "measuring spoon2-06446d8dc337f3d4", "glass-681ef44efd9eb8e6", "plate-4469359f1a382485", "spoon2-35e002beea08de16", "spoon3-5b4632657a4b7d41", "chopping board-62a08cef24d2d329", "oven glove2-0324a8806743b71f", "plate2-561442093803b3c2", "brush-308a27d2f5a57ab4", "towel-bba79bcc4f0ebe85", "skewer-62ef3ed7639fb6e5", "spoon4-2891703779d733e6" ], "reasoning": "1. To determine the span of time I must observe actions in, I first identify the first and second timespans that the user wears the yellow oven mitt (oven glove-1e7582ed39c9b3e3): I observe that the first time the user places this mitt on is at 00:38, and wears it until 01:25, when they take it off and place it on the countertop.\n2. I continue watching the video to identify when the user next puts on the yellow oven mitt (oven glove-1e7582ed39c9b3e3): this occurs at 04:34. I have now identified the timespan that I need to look at as between 01:25-04:34.\n3. I watch the video during the timespan of 01:25-04:34, comparing the actions I see onscreen with my answer choices to eliminate actions performed during this time: the first answer I eliminate is D, as I see the user take a large metal pot (pot-9263bdfa4ea8d66d) off the shelf at 01:49.\n4. I continue watching the allotted span and eliminate E as an answer choice when I observe the user wiping down counter space with a wet rag from 02:04-02:10.\n5. I continue watching and eliminate A as an answer choice when I observe the user look at their cell phone (phone-acec62c0c30bb2a1) from 03:53-03:56.\n6. I continue watching and eliminate B as an answer choice when I see the user drain a can of chickpeas into the sink at 04:09.\n7. With the answer choice narrowed down to C, I watch the remainder of the span to ensure that a coffee pot refill does not occur before 04:34. It does not, and I choose C as my answer.", "referenced_object_masks": [ "phone-acec62c0c30bb2a1", "oven glove-1e7582ed39c9b3e3", "pot-9263bdfa4ea8d66d" ] }, { "key": "P05-20240423-180529:68cb22f89c8186846b56ac0ccada528255d954d7", "video_id": "P05-20240423-180529", "question": "If the order in which I added the lemon juice, chickpeas, yogurt, sprouts, and chopped veggie leaves to the stew were reversed, which of the items listed would be the second one added to the stew?", "answer_choice_0": "Sprouts", "answer_choice_1": "Chickpeas", "answer_choice_2": "Yogurt", "answer_choice_3": "Chopped veggie leaves", "answer_choice_4": "Lemon juice", "answer_id": 4, "answer": "Lemon juice", "question_type": [ "Action Recognition", "Temporal Reasoning", "Object Recognition", "Counting" ], "objects_of_interest": [ "kettle-452a2b894cb0c2a8", "bowl of water-afbcc69f996972d5", "cloth-02f0694594fca358", "oven glove-1e7582ed39c9b3e3", "pan-7e589b2a6101695b", "pan lid-ba9b640370e6ad72", "wooden spoon-275666359a0d7d14", "tins of chickpeas-e2f636cab57a361b", "tins of chickpeas2-7e413874b297c96a", "pot-9263bdfa4ea8d66d", "pot of salt-e8a502e14698f73c", "pot lid-892b74e11c774a60", "packet of rice-1aa563779ca757c5", "mug-f02aeeda1b85ce7e", "phone-acec62c0c30bb2a1", "bag of spinach-a3c73edd5fc497cc", "spinach-8256ea2714aa9d25", "spinach2-758781ca50c1b51d", "spinach3-318b6435b072df84", "spinach4-22f30708916d0f88", "spinach5-b6856db8e92865f4", "spinach6-38cf4de8b40e4d54", "spinach7-79d229c829e6b82d", "bag of rice-2448643751874ce7", "bag of green things-73eb2e73e56fa667", "measuring spoon-dd99b96849fe7663", "knife-6271397240d2ab02", "colander-611a17072081e978", "colander2-8f9fc4463f3722dd", "lid of tin-81f797ef6a69874f", "scale-77e26cf0733c390b", "containers-fe81ebe775f705bd", "yoghurt-56f0bdd5387b25bf", "coconut milk tin-d8f35f562a62487c", "spoon-2537cd8d327de287", "lemon-23a5c76a1c89de04", "second half of lemon-7b2a0b9f2ec5188e", "measuring spoon2-06446d8dc337f3d4", "glass-681ef44efd9eb8e6", "plate-4469359f1a382485", "spoon2-35e002beea08de16", "spoon3-5b4632657a4b7d41", "chopping board-62a08cef24d2d329", "oven glove2-0324a8806743b71f", "plate2-561442093803b3c2", "brush-308a27d2f5a57ab4", "towel-bba79bcc4f0ebe85", "skewer-62ef3ed7639fb6e5", "spoon4-2891703779d733e6" ], "reasoning": "1. I first watch the video to determine the order in which the question's listed items are added to the stew .\n2. I observe that the first item added to the stew (pan-7e589b2a6101695b) is the can of chickpeas (tins of chickpeas-e2f636cab57a361b) at 04:43 by the user.\n3. I observe that the second item added to the stew (pan-7e589b2a6101695b) is the sprouts (spinach-8256ea2714aa9d25) from the sprouts bag (bag of spinach-a3c73edd5fc497cc) by the user at 06:11.\n4. I observe that the third item added to the stew (pan-7e589b2a6101695b) by the user is the yogurt (yoghurt-56f0bdd5387b25bf) at 10:47.\n5. I observe that the fourth item added to the stew (pan-7e589b2a6101695b) by the user is the lemon juice from a small lemon wedge (second half of lemon-7b2a0b9f2ec5188e) at 12:31.\n6. I observe that the fifth thing added to the stew (pan-7e589b2a6101695b) by the user is the chopped leaves from the cutting board (chopping board-62a08cef24d2d329) at 15:41.\n6. Counterfactually, I then assume that the order I added these objects in was reversed, and realize that the second item added now would be the fourth item truly added, which was the lemon juice from the small wedge (second half of lemon-7b2a0b9f2ec5188e), making A the correct answer.", "referenced_object_masks": [ "second half of lemon-7b2a0b9f2ec5188e", "bag of spinach-a3c73edd5fc497cc", "chopping board-62a08cef24d2d329", "yoghurt-56f0bdd5387b25bf", "pan-7e589b2a6101695b", "tins of chickpeas-e2f636cab57a361b", "spinach-8256ea2714aa9d25" ] }, { "key": "P05-20240423-180529:727624e5fa2cb7d9c089981ebaf7a0141b767981", "video_id": "P05-20240423-180529", "question": "How many times did I move the brush upward while cleaning off the knife?", "answer_choice_0": "11 times", "answer_choice_1": "14 times", "answer_choice_2": "12 times", "answer_choice_3": "13 times", "answer_choice_4": "10 times", "answer_id": 3, "answer": "13 times", "question_type": [ "Action Recognition", "Counting", "Object Recognition", "Event Occurence" ], "objects_of_interest": [ "kettle-452a2b894cb0c2a8", "bowl of water-afbcc69f996972d5", "cloth-02f0694594fca358", "oven glove-1e7582ed39c9b3e3", "pan-7e589b2a6101695b", "pan lid-ba9b640370e6ad72", "wooden spoon-275666359a0d7d14", "tins of chickpeas-e2f636cab57a361b", "tins of chickpeas2-7e413874b297c96a", "pot-9263bdfa4ea8d66d", "pot of salt-e8a502e14698f73c", "pot lid-892b74e11c774a60", "packet of rice-1aa563779ca757c5", "mug-f02aeeda1b85ce7e", "phone-acec62c0c30bb2a1", "bag of spinach-a3c73edd5fc497cc", "spinach-8256ea2714aa9d25", "spinach2-758781ca50c1b51d", "spinach3-318b6435b072df84", "spinach4-22f30708916d0f88", "spinach5-b6856db8e92865f4", "spinach6-38cf4de8b40e4d54", "spinach7-79d229c829e6b82d", "bag of rice-2448643751874ce7", "bag of green things-73eb2e73e56fa667", "measuring spoon-dd99b96849fe7663", "knife-6271397240d2ab02", "colander-611a17072081e978", "colander2-8f9fc4463f3722dd", "lid of tin-81f797ef6a69874f", "scale-77e26cf0733c390b", "containers-fe81ebe775f705bd", "yoghurt-56f0bdd5387b25bf", "coconut milk tin-d8f35f562a62487c", "spoon-2537cd8d327de287", "lemon-23a5c76a1c89de04", "second half of lemon-7b2a0b9f2ec5188e", "measuring spoon2-06446d8dc337f3d4", "glass-681ef44efd9eb8e6", "plate-4469359f1a382485", "spoon2-35e002beea08de16", "spoon3-5b4632657a4b7d41", "chopping board-62a08cef24d2d329", "oven glove2-0324a8806743b71f", "plate2-561442093803b3c2", "brush-308a27d2f5a57ab4", "towel-bba79bcc4f0ebe85", "skewer-62ef3ed7639fb6e5", "spoon4-2891703779d733e6" ], "reasoning": "1. I watched the video and looked for the time when the user used a brush to clean a knife. I found this moment at 16:49, when the user grabbed a brush (brush-308a27d2f5a57ab4) in one hand, and held a knife (knife-6271397240d2ab02) in the other hand.\n2. At 16:50, the user brushed up once on the knife (knife-6271397240d2ab02).\n3. Then, the user brought the knife (knife-6271397240d2ab02) over to the soap dispenser at 16:53 in order to add more soap .\n4. Then, the user started to brush the knife (knife-6271397240d2ab02) up and down several times in succession using the brush (brush-308a27d2f5a57ab4). They did this until 16:57, pausing about halfway through to turn the knife (knife-6271397240d2ab02) over to its other side - during this scrubbing, I observed 12 upward brush movements.\n5. In total, between 16:50-16:57, the user moved the brush (brush-308a27d2f5a57ab4) upward 13 times, making that the final answer.", "referenced_object_masks": [ "brush-308a27d2f5a57ab4", "knife-6271397240d2ab02" ] }, { "key": "P05-20240423-180529:7dfe21071bd9bb9ba8a715fa36cdc77040b06e19", "video_id": "P05-20240423-180529", "question": "How many times do I touch a spoon with either hand in the final minute of the video?", "answer_choice_0": "2", "answer_choice_1": "3", "answer_choice_2": "4", "answer_choice_3": "5", "answer_choice_4": "6", "answer_id": 4, "answer": "6", "question_type": [ "Action Recognition", "Counting", "Temporal Reasoning" ], "objects_of_interest": [ "kettle-452a2b894cb0c2a8", "bowl of water-afbcc69f996972d5", "cloth-02f0694594fca358", "oven glove-1e7582ed39c9b3e3", "pan-7e589b2a6101695b", "pan lid-ba9b640370e6ad72", "wooden spoon-275666359a0d7d14", "tins of chickpeas-e2f636cab57a361b", "tins of chickpeas2-7e413874b297c96a", "pot-9263bdfa4ea8d66d", "pot of salt-e8a502e14698f73c", "pot lid-892b74e11c774a60", "packet of rice-1aa563779ca757c5", "mug-f02aeeda1b85ce7e", "phone-acec62c0c30bb2a1", "bag of spinach-a3c73edd5fc497cc", "spinach-8256ea2714aa9d25", "spinach2-758781ca50c1b51d", "spinach3-318b6435b072df84", "spinach4-22f30708916d0f88", "spinach5-b6856db8e92865f4", "spinach6-38cf4de8b40e4d54", "spinach7-79d229c829e6b82d", "bag of rice-2448643751874ce7", "bag of green things-73eb2e73e56fa667", "measuring spoon-dd99b96849fe7663", "knife-6271397240d2ab02", "colander-611a17072081e978", "colander2-8f9fc4463f3722dd", "lid of tin-81f797ef6a69874f", "scale-77e26cf0733c390b", "containers-fe81ebe775f705bd", "yoghurt-56f0bdd5387b25bf", "coconut milk tin-d8f35f562a62487c", "spoon-2537cd8d327de287", "lemon-23a5c76a1c89de04", "second half of lemon-7b2a0b9f2ec5188e", "measuring spoon2-06446d8dc337f3d4", "glass-681ef44efd9eb8e6", "plate-4469359f1a382485", "spoon2-35e002beea08de16", "spoon3-5b4632657a4b7d41", "chopping board-62a08cef24d2d329", "oven glove2-0324a8806743b71f", "plate2-561442093803b3c2", "brush-308a27d2f5a57ab4", "towel-bba79bcc4f0ebe85", "skewer-62ef3ed7639fb6e5", "spoon4-2891703779d733e6" ], "reasoning": "1. To determine this, I watch the final minute of the video - since it is 19:32 long, I watched from 18:32-19:32, paying attention to when the user touched a spoon - not necessarily a unique one - and counting that spoon toward the tally.\n2. At 18:40, I watch the user pick up two spoons (spoon2-35e002beea08de16, spoon3-5b4632657a4b7d41) with one hand, and counted these as touch #1 and touch #2 in the tally.\n3. At 18:42, the user transfers one spoon (spoon2-35e002beea08de16) to their left hand. This is touch #3.\n3. I observe that the user places spoon #1 (spoon2-35e002beea08de16) in the stew (pan-7e589b2a6101695b) at 18:42, and holds onto spoon #2 (spoon3-5b4632657a4b7d41), using it to ladle rice from the rice pot (pot-9263bdfa4ea8d66d) until placing it into the pot 18:54.\n4. At 18:56, I observe the user picking up spoon #1 (spoon2-35e002beea08de16) and using it to scoop stew from the stew pot (pan-7e589b2a6101695b). Since the user put this spoon down and is now picking it up again, I count it as touch #4 in the tally.\n5. At 19:13, I observe the user put spoon #1 (spoon2-35e002beea08de16) back in the stew (pan-7e589b2a6101695b), and turn briefly away, before thinking better of it and picking up the spoon again at 19:14 to ladle more food - I count this at touch #5.\n6. At 19:24, I observe the user open a kitchen drawer and take out a third spoon (spoon4-2891703779d733e6) making this touch #6.", "referenced_object_masks": [ "spoon4-2891703779d733e6", "pot-9263bdfa4ea8d66d", "pan-7e589b2a6101695b", "spoon2-35e002beea08de16", "spoon3-5b4632657a4b7d41" ] }, { "key": "P05-20240423-180529:ab797e0aa55ba77e952bdb0d7cf6d14caf3ae925", "video_id": "P05-20240423-180529", "question": "What is the product of the number of unique containers I poured water into using the kettle and the number of total times I poured water from the kettle?", "answer_choice_0": "9", "answer_choice_1": "6", "answer_choice_2": "12", "answer_choice_3": "8", "answer_choice_4": "10", "answer_id": 3, "answer": "8", "question_type": [ "Numerical Reasoning", "Action Recognition", "Object Recognition", "Counting" ], "objects_of_interest": [ "kettle-452a2b894cb0c2a8", "bowl of water-afbcc69f996972d5", "cloth-02f0694594fca358", "oven glove-1e7582ed39c9b3e3", "pan-7e589b2a6101695b", "pan lid-ba9b640370e6ad72", "wooden spoon-275666359a0d7d14", "tins of chickpeas-e2f636cab57a361b", "tins of chickpeas2-7e413874b297c96a", "pot-9263bdfa4ea8d66d", "pot of salt-e8a502e14698f73c", "pot lid-892b74e11c774a60", "packet of rice-1aa563779ca757c5", "mug-f02aeeda1b85ce7e", "phone-acec62c0c30bb2a1", "bag of spinach-a3c73edd5fc497cc", "spinach-8256ea2714aa9d25", "spinach2-758781ca50c1b51d", "spinach3-318b6435b072df84", "spinach4-22f30708916d0f88", "spinach5-b6856db8e92865f4", "spinach6-38cf4de8b40e4d54", "spinach7-79d229c829e6b82d", "bag of rice-2448643751874ce7", "bag of green things-73eb2e73e56fa667", "measuring spoon-dd99b96849fe7663", "knife-6271397240d2ab02", "colander-611a17072081e978", "colander2-8f9fc4463f3722dd", "lid of tin-81f797ef6a69874f", "scale-77e26cf0733c390b", "containers-fe81ebe775f705bd", "yoghurt-56f0bdd5387b25bf", "coconut milk tin-d8f35f562a62487c", "spoon-2537cd8d327de287", "lemon-23a5c76a1c89de04", "second half of lemon-7b2a0b9f2ec5188e", "measuring spoon2-06446d8dc337f3d4", "glass-681ef44efd9eb8e6", "plate-4469359f1a382485", "spoon2-35e002beea08de16", "spoon3-5b4632657a4b7d41", "chopping board-62a08cef24d2d329", "oven glove2-0324a8806743b71f", "plate2-561442093803b3c2", "brush-308a27d2f5a57ab4", "towel-bba79bcc4f0ebe85", "skewer-62ef3ed7639fb6e5", "spoon4-2891703779d733e6" ], "reasoning": "1. I watched the video and looked for the times when the user used the kettle (kettle-452a2b894cb0c2a8) to pour water into a container. At 02:54, I noticed the user picked up the kettle (kettle-452a2b894cb0c2a8) and brought it over to the other side of the kitchen.\n2. From 02:58-03:09, I noticed the user poured water from the kettle (kettle-452a2b894cb0c2a8) into a pot (pot-9263bdfa4ea8d66d). I counted this as one pour from the kettle (kettle-452a2b894cb0c2a8) and so far, one container.\n3. At 09:48, I noticed the user once again holding the kettle (kettle-452a2b894cb0c2a8). At this moment, they added a little bit of water to the pan (pan-7e589b2a6101695b). I counted this as the 2nd pour from the kettle (kettle-452a2b894cb0c2a8) and the 2nd unique container.\n4. From 17:43-17:47, I noticed the user once again holding the kettle (kettle-452a2b894cb0c2a8), and at this time, they poured water from it into the same pot (pot-9263bdfa4ea8d66d) from earlier. I counted this as the 3rd pour from the kettle (kettle-452a2b894cb0c2a8) but kept the unique container number at 2.\n5. From 17:57-18:03, I noticed the user once again holding the kettle (kettle-452a2b894cb0c2a8), and at this time, they poured water from it into the same pot (pot-9263bdfa4ea8d66d) from before. I counted this as the 4th pour from the kettle (kettle-452a2b894cb0c2a8) but kept the unique container number at 2.\n6. From this point on, I did not see any other pours of water from the kettle (kettle-452a2b894cb0c2a8), leaving the final count at 4 total pours and 2 unique containers.\n7. 4x2=8, making the product of 8 the final answer.", "referenced_object_masks": [ "pan-7e589b2a6101695b", "kettle-452a2b894cb0c2a8", "pot-9263bdfa4ea8d66d" ] }, { "key": "P05-20240423-180529:b63ac3755f4106d00af03596d3e7699d3979e12c", "video_id": "P05-20240423-180529", "question": "If I multiplied the number of brussels sprouts I placed in the stew, by the number of total spoonfuls of food I place on my finished dinner plate, what would be the product of this equation?", "answer_choice_0": "49", "answer_choice_1": "20", "answer_choice_2": "21", "answer_choice_3": "42", "answer_choice_4": "10", "answer_id": 0, "answer": "49", "question_type": [ "Action Recognition", "Counting", "Object Recognition", "Numerical Reasoning" ], "objects_of_interest": [ "kettle-452a2b894cb0c2a8", "bowl of water-afbcc69f996972d5", "cloth-02f0694594fca358", "oven glove-1e7582ed39c9b3e3", "pan-7e589b2a6101695b", "pan lid-ba9b640370e6ad72", "wooden spoon-275666359a0d7d14", "tins of chickpeas-e2f636cab57a361b", "tins of chickpeas2-7e413874b297c96a", "pot-9263bdfa4ea8d66d", "pot of salt-e8a502e14698f73c", "pot lid-892b74e11c774a60", "packet of rice-1aa563779ca757c5", "mug-f02aeeda1b85ce7e", "phone-acec62c0c30bb2a1", "bag of spinach-a3c73edd5fc497cc", "spinach-8256ea2714aa9d25", "spinach2-758781ca50c1b51d", "spinach3-318b6435b072df84", "spinach4-22f30708916d0f88", "spinach5-b6856db8e92865f4", "spinach6-38cf4de8b40e4d54", "spinach7-79d229c829e6b82d", "bag of rice-2448643751874ce7", "bag of green things-73eb2e73e56fa667", "measuring spoon-dd99b96849fe7663", "knife-6271397240d2ab02", "colander-611a17072081e978", "colander2-8f9fc4463f3722dd", "lid of tin-81f797ef6a69874f", "scale-77e26cf0733c390b", "containers-fe81ebe775f705bd", "yoghurt-56f0bdd5387b25bf", "coconut milk tin-d8f35f562a62487c", "spoon-2537cd8d327de287", "lemon-23a5c76a1c89de04", "second half of lemon-7b2a0b9f2ec5188e", "measuring spoon2-06446d8dc337f3d4", "glass-681ef44efd9eb8e6", "plate-4469359f1a382485", "spoon2-35e002beea08de16", "spoon3-5b4632657a4b7d41", "chopping board-62a08cef24d2d329", "oven glove2-0324a8806743b71f", "plate2-561442093803b3c2", "brush-308a27d2f5a57ab4", "towel-bba79bcc4f0ebe85", "skewer-62ef3ed7639fb6e5", "spoon4-2891703779d733e6" ], "reasoning": "1. To find this answer, I first identify when the user places brussels sprouts in the stew, and count the number of sprouts. I observe the user retrieve this sprouts bag (bag of spinach-a3c73edd5fc497cc) from the fridge at 06:00, and then walk over to the stew - from 06:10-06:29, I count as the user places 7 sprouts, one by one (spinach-8256ea2714aa9d25, spinach2-758781ca50c1b51d, spinach3-318b6435b072df84, spinach4-22f30708916d0f88, spinach5-b6856db8e92865f4, spinach6-38cf4de8b40e4d54, spinach7-79d229c829e6b82d) into the stew .\n2. I then determine when the user places spoonfuls of their finished dinner onto a plate (plate-4469359f1a382485) - this occurs starting at 18:44, when the user picks up a plate (plate-4469359f1a382485) and ladles spoonfuls of rice from the rice bowl (pot-9263bdfa4ea8d66d) onto it - from 18:45-18:56, I watch the user ladle 3 spoonfuls of rice onto their plate.\n3. I then watch from 18:59-19:19 as the user ladles 4 spoonfuls of finished stew (pan-7e589b2a6101695b) on top of the rice that is on the dinner plate (plate-4469359f1a382485). I add these 4 spoonfuls to the 3 spoonfuls counted in the previous step, to discover a total of 7 spoonfuls of finished dinner heaped onto the plate.\n4. I multiply the 7 sprouts placed in the stew from the sprout bag by the 7 spoonfuls of food placed on the dinner plate (plate-4469359f1a382485) to obtain a product of 49, making E the correct answer.", "referenced_object_masks": [ "bag of spinach-a3c73edd5fc497cc", "pot-9263bdfa4ea8d66d", "pan-7e589b2a6101695b", "spinach-8256ea2714aa9d25", "spinach2-758781ca50c1b51d", "spinach3-318b6435b072df84", "spinach4-22f30708916d0f88", "spinach5-b6856db8e92865f4", "spinach6-38cf4de8b40e4d54", "spinach7-79d229c829e6b82d", "plate-4469359f1a382485" ] }, { "key": "P05-20240423-180529:dd7c142020337defb5ccdf929c511a8cbcdd440d", "video_id": "P05-20240423-180529", "question": "How many complete clockwise circles did I make during the first 2 times that I stirred the stew?", "answer_choice_0": "12 circles", "answer_choice_1": "10 circles", "answer_choice_2": "16 circles", "answer_choice_3": "14 circles", "answer_choice_4": "8 circles", "answer_id": 1, "answer": "10 circles", "question_type": [ "Action Recognition", "Counting", "Object Recognition", "Event Occurence", "Numerical Reasoning" ], "objects_of_interest": [ "kettle-452a2b894cb0c2a8", "bowl of water-afbcc69f996972d5", "cloth-02f0694594fca358", "oven glove-1e7582ed39c9b3e3", "pan-7e589b2a6101695b", "pan lid-ba9b640370e6ad72", "wooden spoon-275666359a0d7d14", "tins of chickpeas-e2f636cab57a361b", "tins of chickpeas2-7e413874b297c96a", "pot-9263bdfa4ea8d66d", "pot of salt-e8a502e14698f73c", "pot lid-892b74e11c774a60", "packet of rice-1aa563779ca757c5", "mug-f02aeeda1b85ce7e", "phone-acec62c0c30bb2a1", "bag of spinach-a3c73edd5fc497cc", "spinach-8256ea2714aa9d25", "spinach2-758781ca50c1b51d", "spinach3-318b6435b072df84", "spinach4-22f30708916d0f88", "spinach5-b6856db8e92865f4", "spinach6-38cf4de8b40e4d54", "spinach7-79d229c829e6b82d", "bag of rice-2448643751874ce7", "bag of green things-73eb2e73e56fa667", "measuring spoon-dd99b96849fe7663", "knife-6271397240d2ab02", "colander-611a17072081e978", "colander2-8f9fc4463f3722dd", "lid of tin-81f797ef6a69874f", "scale-77e26cf0733c390b", "containers-fe81ebe775f705bd", "yoghurt-56f0bdd5387b25bf", "coconut milk tin-d8f35f562a62487c", "spoon-2537cd8d327de287", "lemon-23a5c76a1c89de04", "second half of lemon-7b2a0b9f2ec5188e", "measuring spoon2-06446d8dc337f3d4", "glass-681ef44efd9eb8e6", "plate-4469359f1a382485", "spoon2-35e002beea08de16", "spoon3-5b4632657a4b7d41", "chopping board-62a08cef24d2d329", "oven glove2-0324a8806743b71f", "plate2-561442093803b3c2", "brush-308a27d2f5a57ab4", "towel-bba79bcc4f0ebe85", "skewer-62ef3ed7639fb6e5", "spoon4-2891703779d733e6" ], "reasoning": "1. I watched the video and looked for the first 2 times that the user stirred the stew. I found the first segment of this from 00:55-01:22, when the user used the wooden spoon (wooden spoon-275666359a0d7d14) to stir the stew inside the pan (pan-7e589b2a6101695b).\n2. From 00:55-01:22, I noticed the user completed full clockwise circles with the spoon (wooden spoon-275666359a0d7d14) at 01:06, 01:07, 01:09, 01:16, and 01:17. This equates to 5 total circles.\n3. I continued watching, looking for the 2nd time that the user completed clockwise stirs of the stew in the pan (pan-7e589b2a6101695b). I found this segment from 04:59-05:24.\n4. From 04:59-05:24, I found 4 stirs in quick succession from 05:05-05:07. Then, there was an additional full clockwise stir at 05:16.\n5. Since there were 5 stirs the first time and 5 stirs the second time, I added them together to get a final total of 10 circles.", "referenced_object_masks": [ "wooden spoon-275666359a0d7d14", "pan-7e589b2a6101695b" ] }, { "key": "P05-20240423-180529:e76f2d9637332d8d5246e8f7a641873437de6b9c", "video_id": "P05-20240423-180529", "question": "What is the only item that I touched the 5th time that I walked over to the area of the kitchen with the sink?", "answer_choice_0": "The cloth", "answer_choice_1": "The mug", "answer_choice_2": "The brush", "answer_choice_3": "The colander", "answer_choice_4": "The kettle", "answer_id": 4, "answer": "The kettle", "question_type": [ "Action Recognition", "Counting", "Egocentric & Spatial Perception", "Object Recognition" ], "objects_of_interest": [ "kettle-452a2b894cb0c2a8", "bowl of water-afbcc69f996972d5", "cloth-02f0694594fca358", "oven glove-1e7582ed39c9b3e3", "pan-7e589b2a6101695b", "pan lid-ba9b640370e6ad72", "wooden spoon-275666359a0d7d14", "tins of chickpeas-e2f636cab57a361b", "tins of chickpeas2-7e413874b297c96a", "pot-9263bdfa4ea8d66d", "pot of salt-e8a502e14698f73c", "pot lid-892b74e11c774a60", "packet of rice-1aa563779ca757c5", "mug-f02aeeda1b85ce7e", "phone-acec62c0c30bb2a1", "bag of spinach-a3c73edd5fc497cc", "spinach-8256ea2714aa9d25", "spinach2-758781ca50c1b51d", "spinach3-318b6435b072df84", "spinach4-22f30708916d0f88", "spinach5-b6856db8e92865f4", "spinach6-38cf4de8b40e4d54", "spinach7-79d229c829e6b82d", "bag of rice-2448643751874ce7", "bag of green things-73eb2e73e56fa667", "measuring spoon-dd99b96849fe7663", "knife-6271397240d2ab02", "colander-611a17072081e978", "colander2-8f9fc4463f3722dd", "lid of tin-81f797ef6a69874f", "scale-77e26cf0733c390b", "containers-fe81ebe775f705bd", "yoghurt-56f0bdd5387b25bf", "coconut milk tin-d8f35f562a62487c", "spoon-2537cd8d327de287", "lemon-23a5c76a1c89de04", "second half of lemon-7b2a0b9f2ec5188e", "measuring spoon2-06446d8dc337f3d4", "glass-681ef44efd9eb8e6", "plate-4469359f1a382485", "spoon2-35e002beea08de16", "spoon3-5b4632657a4b7d41", "chopping board-62a08cef24d2d329", "oven glove2-0324a8806743b71f", "plate2-561442093803b3c2", "brush-308a27d2f5a57ab4", "towel-bba79bcc4f0ebe85", "skewer-62ef3ed7639fb6e5", "spoon4-2891703779d733e6" ], "reasoning": "1. I watched the video and looked for the times when the user entered the area of the kitchen which contained the sink . I noticed this occurred for the first time from 00:07-00:34.\n2. Then, I counted subsequent visits to the area of the kitchen which contained the sink . From 01:42-01:45, the user visited the area of the kitchen with the sink for the 2nd time.\n3. From 01:56-02:01, the user visited the area of the kitchen with the sink for the 3rd time.\n4. From 02:38-02:39, the user visited the area of the kitchen with the sink for the 4th time.\n5. From 02:52-02:55, the user visited the area of the kitchen with the sink for the 5th time.\n6. Since this was the target visit number for the kitchen, I looked to see what the user touched during the 02:52-02:55 segment. At 02:53, I noticed the user picked up the kettle (kettle-452a2b894cb0c2a8). That makes this object the final answer.", "referenced_object_masks": [ "kettle-452a2b894cb0c2a8" ] }, { "key": "P05-20240423-183358:533891030c7048ae73a8590d5d0a67b124498ea5", "video_id": "P05-20240423-183358", "question": "How many more times did I hit the wooden spoon on the edge of the frying pan than the number of times I opened the dishwasher?", "answer_choice_0": "9", "answer_choice_1": "8", "answer_choice_2": "7", "answer_choice_3": "10", "answer_choice_4": "6", "answer_id": 1, "answer": "8", "question_type": [ "Numerical Reasoning", "Temporal Reasoning" ], "objects_of_interest": [ "plate-e481cddf1243a29a", "spoon-6c59a84e0328eee6", "spoon2-c7f331cf10e4df11", "pot-f477bcb8a16394b7", "lid of rice-ba8230d7f111822d", "pan-83e3bf526f61dfaf", "wooden spoon-57e2df13db4f2d7d", "other cutlery-e219e3f1485ba7d5", "right glove-fa4932703b3243c5", "left glove-9c1b7485df9e6430", "lid2-3adfd241b7c418dd" ], "reasoning": "1. At 0:25, the user hits the wooden spoon (wooden spoon-57e2df13db4f2d7d) on the edge of the frying pan (pan-83e3bf526f61dfaf) 10 times.\n2. At 0:07, the user opens the dishwasher to put in a plate (plate-e481cddf1243a29a) and a spoon (spoon-6c59a84e0328eee6).\n3. At 0:33, the user opens the dishwasher to put in the wooden spoon (wooden spoon-57e2df13db4f2d7d).\n4. So the number of times the dishwasher is opened is 2.\n5. The number of times the user hits the wooden spoon (wooden spoon-57e2df13db4f2d7d) on the edge of the pan (pan-83e3bf526f61dfaf), 10, minus the number of times the dishwasher is opened, 2, is 8.", "referenced_object_masks": [ "pan-83e3bf526f61dfaf", "wooden spoon-57e2df13db4f2d7d", "plate-e481cddf1243a29a", "spoon-6c59a84e0328eee6" ] }, { "key": "P05-20240423-183358:5bbe48fe253232d860260968461ef09e926f3776", "video_id": "P05-20240423-183358", "question": "Find the number of sinks in the kitchen, the number of cooking vessels on the stove at 0:17 and the number of pan lids on the black counter at 0:17. If you multiply these 3 numbers together, what do you get?", "answer_choice_0": "4", "answer_choice_1": "8", "answer_choice_2": "18", "answer_choice_3": "12", "answer_choice_4": "6", "answer_id": 1, "answer": "8", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness", "Temporal Reasoning", "Counting", "Numerical Reasoning" ], "objects_of_interest": [ "plate-e481cddf1243a29a", "spoon-6c59a84e0328eee6", "spoon2-c7f331cf10e4df11", "pot-f477bcb8a16394b7", "lid of rice-ba8230d7f111822d", "pan-83e3bf526f61dfaf", "wooden spoon-57e2df13db4f2d7d", "other cutlery-e219e3f1485ba7d5", "right glove-fa4932703b3243c5", "left glove-9c1b7485df9e6430", "lid2-3adfd241b7c418dd" ], "reasoning": "1. I watched the whole video and saw the user go to the dishwasher next to the sink at 0:05. There are 2 sinks .\n2. At 0:17, the user turns left and walks to the stove where there are 2 cooking vessels (pot-f477bcb8a16394b7, pan-83e3bf526f61dfaf) on the stove.\n3. At 0:17, there are 2 pan lids (lid of rice-ba8230d7f111822d, lid2-3adfd241b7c418dd) on the black counter .\n4. The number of sinks times the number of cooking vessels (pot-f477bcb8a16394b7, pan-83e3bf526f61dfaf) on the stove times the number of pan lids (lid of rice-ba8230d7f111822d, lid2-3adfd241b7c418dd) on the counter is 2*2*2 which is 8.", "referenced_object_masks": [ "lid of rice-ba8230d7f111822d", "lid2-3adfd241b7c418dd", "pot-f477bcb8a16394b7", "pan-83e3bf526f61dfaf" ] }, { "key": "P05-20240423-183358:75552781c85ac76f29b49ac109c06aa1c2d1be3a", "video_id": "P05-20240423-183358", "question": "Where are the 2 dog bowls in relation to me when I am at the marble counter?", "answer_choice_0": "To my left", "answer_choice_1": "In back of me to my left", "answer_choice_2": "To my right", "answer_choice_3": "In front of me", "answer_choice_4": "In back of me to my right", "answer_id": 4, "answer": "In back of me to my right", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness", "Temporal Reasoning" ], "objects_of_interest": [ "plate-e481cddf1243a29a", "spoon-6c59a84e0328eee6", "spoon2-c7f331cf10e4df11", "pot-f477bcb8a16394b7", "lid of rice-ba8230d7f111822d", "pan-83e3bf526f61dfaf", "wooden spoon-57e2df13db4f2d7d", "other cutlery-e219e3f1485ba7d5", "right glove-fa4932703b3243c5", "left glove-9c1b7485df9e6430", "lid2-3adfd241b7c418dd" ], "reasoning": "1. I watched the whole video and noticed right at 00:00 that the 2 dog bowls are inside the door to the right in the kitchen.\n2. At 00:16, the user walks past the 2 dog bowls .\n3. At 00:18, the user takes a pot of rice (pot-f477bcb8a16394b7) off the stove and places it on the black, marble counter .\n4. The 2 dog bowls are in the back of the user to the right.", "referenced_object_masks": [ "pot-f477bcb8a16394b7" ] }, { "key": "P05-20240423-183358:794319f296cedc71c008f66b8cbd9cb1bd5eb55f", "video_id": "P05-20240423-183358", "question": "Where are the oven mitts in relation to the dishwasher?", "answer_choice_0": "Hanging on hook to the right of the sink", "answer_choice_1": "When facing the dishwasher, move left, walk directly down the kitchen and turn right to be in front of the stove where the oven mitts are hanging on the wall to the left", "answer_choice_2": "Hanging on a hook to the left of the sink", "answer_choice_3": "When facing the dishwasher, move left and walk directly down the kitchen to the black counter", "answer_choice_4": "To the right of the dishwasher", "answer_id": 3, "answer": "When facing the dishwasher, move left and walk directly down the kitchen to the black counter", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness", "Temporal Reasoning" ], "objects_of_interest": [ "plate-e481cddf1243a29a", "spoon-6c59a84e0328eee6", "spoon2-c7f331cf10e4df11", "pot-f477bcb8a16394b7", "lid of rice-ba8230d7f111822d", "pan-83e3bf526f61dfaf", "wooden spoon-57e2df13db4f2d7d", "other cutlery-e219e3f1485ba7d5", "right glove-fa4932703b3243c5", "left glove-9c1b7485df9e6430", "lid2-3adfd241b7c418dd" ], "reasoning": "1. I watched the whole video and saw the gloves (right glove-fa4932703b3243c5, left glove-9c1b7485df9e6430) on the black counter to the right as the user walked into the kitchen.\n2. Then, I looked for the dishwasher. I found it as the user enters the kitchen, turns left and walks down the kitchen to the dishwasher at 0:07.\n3. The user closes the dishwasher , turns left and walks down the kitchen to the black counter that the oven mitts (right glove-fa4932703b3243c5, left glove-9c1b7485df9e6430) are on. Since this led the user from the dishwasher to the oven mitts (right glove-fa4932703b3243c5, left glove-9c1b7485df9e6430), I determined this was the answer.", "referenced_object_masks": [ "right glove-fa4932703b3243c5", "left glove-9c1b7485df9e6430" ] }, { "key": "P05-20240423-183358:f0b42c3659aa6b6604466dc5309241961e534275", "video_id": "P05-20240423-183358", "question": "If you were to add together the number of items I put in the dishwasher, the number of pans I take off the stove, and the number of times I tap the wooden spoon on the pan, what would the total be?", "answer_choice_0": "10", "answer_choice_1": "12", "answer_choice_2": "15", "answer_choice_3": "17", "answer_choice_4": "13", "answer_id": 2, "answer": "15", "question_type": [ "Action Recognition", "Counting", "Event Occurence", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "plate-e481cddf1243a29a", "spoon-6c59a84e0328eee6", "spoon2-c7f331cf10e4df11", "pot-f477bcb8a16394b7", "lid of rice-ba8230d7f111822d", "pan-83e3bf526f61dfaf", "wooden spoon-57e2df13db4f2d7d", "other cutlery-e219e3f1485ba7d5", "right glove-fa4932703b3243c5", "left glove-9c1b7485df9e6430", "lid2-3adfd241b7c418dd" ], "reasoning": "1. At 00:09, the user puts 2 items, a spoon (spoon-6c59a84e0328eee6) and a plate (plate-e481cddf1243a29a), into the dishwasher .\n2. At 00:34, the user puts a wooden spoon (wooden spoon-57e2df13db4f2d7d) in the dishwasher .\n3. No other items are put in the dishwasher , meaning the total is 2 plus 1 which is 3.\n4. At 00:17, the user takes a pot of rice (pot-f477bcb8a16394b7) off the stove .\n5. At 00:23, the user takes a frying pan of beans and meat (pan-83e3bf526f61dfaf) off the stove .\n6. The number of pans taken off the stove is 1 plus 1 which is 2.\n7. At 00:25, the user picks up the wooden spoon (wooden spoon-57e2df13db4f2d7d) from the pan (pan-83e3bf526f61dfaf) and then taps the side of the pan (pan-83e3bf526f61dfaf) 10 times from 00:25-00:27.\n8. The number of items put into the dishwasher plus the number of pans taken off the stove is 3+2+10, which is a total of 15.", "referenced_object_masks": [ "pot-f477bcb8a16394b7", "wooden spoon-57e2df13db4f2d7d", "pan-83e3bf526f61dfaf", "spoon-6c59a84e0328eee6", "plate-e481cddf1243a29a" ] }, { "key": "P05-20240424-085204:0230cae6367feb4dea653b62ceec257b7c2ff6c9", "video_id": "P05-20240424-085204", "question": "What is the order in which I add ingredients to the first object that I put on the scales?", "answer_choice_0": "Porridge, water, milk", "answer_choice_1": "Water, porridge, sugar", "answer_choice_2": "Porridge, milk, water", "answer_choice_3": "Milk, sugar, porridge", "answer_choice_4": "Milk, porridge, water", "answer_id": 2, "answer": "Porridge, milk, water", "question_type": [ "Action Recognition", "Object Recognition", "Counting", "Temporal Reasoning" ], "objects_of_interest": [ "scales-96309edcd40d7f91", "bowl-20a43f99c89bf5c3", "porridge bag-e66efbacf4319f34", "milk bottle-54c16a16d66d533b", "pint glass-c0d00eae4c894fa5", "lid-c93364fbd585603e", "syrup jar-062cba3b1f9776fb", "honey jar-6bbdd500183e81c5", "dish cloth-cdf2f45e6b34cad2", "pint glass2-53ec5e52d0bbc16a", "plate-f11cbfd3aa5f073a", "banana-cfe0a9fa967f9bb1", "knife-6932a520e69bf940", "spoon-f050990f105e4e1a" ], "reasoning": "1. I watched for the user to get out the scales (scales-96309edcd40d7f91), which they do at 00:14.\n2. I then watched for the first object that the user placed on the scales (scales-96309edcd40d7f91), which was a bowl (bowl-20a43f99c89bf5c3) at 00:23. \n3. I waited to see what ingredients the user put into the bowl (bowl-20a43f99c89bf5c3), and those were porridge from the porridge bag (porridge bag-e66efbacf4319f34) at 00:35, milk from the milk bottle (milk bottle-54c16a16d66d533b) at 01:03, and water from the pint glass (pint glass-c0d00eae4c894fa5) at 01:23, making the order of ingredients porridge, milk, and water.", "referenced_object_masks": [ "bowl-20a43f99c89bf5c3", "porridge bag-e66efbacf4319f34", "scales-96309edcd40d7f91", "pint glass-c0d00eae4c894fa5", "milk bottle-54c16a16d66d533b" ] }, { "key": "P05-20240424-085204:25d3b466b1a2d67da52bb7a9441bd5e401a04b33", "video_id": "P05-20240424-085204", "question": "What is the sum of the number of times I slice the banana multiplied by the number of times I turn the scales on?", "answer_choice_0": "40", "answer_choice_1": "12", "answer_choice_2": "30", "answer_choice_3": "26", "answer_choice_4": "22", "answer_id": 3, "answer": "26", "question_type": [ "Action Recognition", "Counting", "Egocentric & Spatial Perception", "Numerical Reasoning", "Object Recognition" ], "objects_of_interest": [ "scales-96309edcd40d7f91", "bowl-20a43f99c89bf5c3", "porridge bag-e66efbacf4319f34", "milk bottle-54c16a16d66d533b", "pint glass-c0d00eae4c894fa5", "lid-c93364fbd585603e", "syrup jar-062cba3b1f9776fb", "honey jar-6bbdd500183e81c5", "dish cloth-cdf2f45e6b34cad2", "pint glass2-53ec5e52d0bbc16a", "plate-f11cbfd3aa5f073a", "banana-cfe0a9fa967f9bb1", "knife-6932a520e69bf940", "spoon-f050990f105e4e1a" ], "reasoning": "1. I watched for the user to turn on the scales (scales-96309edcd40d7f91), which occurs at 00:25 and 03:30 for a total of 2 times.\n2. I then watched for the user to slice the banana (banana-cfe0a9fa967f9bb1), which occurs from 03:58-04:04.\n3. The user slices the banana (banana-cfe0a9fa967f9bb1) 13 times from 03:58-04:04.\n4. 2 x 13 = 26.", "referenced_object_masks": [ "banana-cfe0a9fa967f9bb1", "scales-96309edcd40d7f91" ] }, { "key": "P05-20240424-085204:6b52e6a37aa612e42db10324e63e749a15e0eccc", "video_id": "P05-20240424-085204", "question": "How many objects are touching the table when I am slicing the banana with the knife?", "answer_choice_0": "5", "answer_choice_1": "3", "answer_choice_2": "0", "answer_choice_3": "4", "answer_choice_4": "2", "answer_id": 0, "answer": "5", "question_type": [ "Action Recognition", "Object Recognition", "Counting" ], "objects_of_interest": [ "scales-96309edcd40d7f91", "bowl-20a43f99c89bf5c3", "porridge bag-e66efbacf4319f34", "milk bottle-54c16a16d66d533b", "pint glass-c0d00eae4c894fa5", "lid-c93364fbd585603e", "syrup jar-062cba3b1f9776fb", "honey jar-6bbdd500183e81c5", "dish cloth-cdf2f45e6b34cad2", "pint glass2-53ec5e52d0bbc16a", "plate-f11cbfd3aa5f073a", "banana-cfe0a9fa967f9bb1", "knife-6932a520e69bf940", "spoon-f050990f105e4e1a" ], "reasoning": "1. I watched for the user to pick up the banana (banana-cfe0a9fa967f9bb1), which occurs at 03:33.\n2. I then waited for the user to get a knife (knife-6932a520e69bf940), which occurs at 03:42.\n3.I looked for the user to slice the banana (banana-cfe0a9fa967f9bb1) with the knife (knife-6932a520e69bf940), which occurs at 03:58.\n4. At 03:58, the objects touching the table are the scales (scales-96309edcd40d7f91), the plate (plate-f11cbfd3aa5f073a), a mixing bowl , a saucepan (OBJ2), and a pint glass (pint glass-c0d00eae4c894fa5), making the answer 5 objects.", "referenced_object_masks": [ "knife-6932a520e69bf940", "plate-f11cbfd3aa5f073a", "banana-cfe0a9fa967f9bb1", "scales-96309edcd40d7f91", "pint glass-c0d00eae4c894fa5" ] }, { "key": "P05-20240424-085204:a5ed8e1e90b0721065aca2e190288872f8abd0d2", "video_id": "P05-20240424-085204", "question": "How many surfaces do I wipe down with the dishcloth?", "answer_choice_0": "4", "answer_choice_1": "1", "answer_choice_2": "5", "answer_choice_3": "3", "answer_choice_4": "2", "answer_id": 4, "answer": "2", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Counting", "Object Recognition" ], "objects_of_interest": [ "scales-96309edcd40d7f91", "bowl-20a43f99c89bf5c3", "porridge bag-e66efbacf4319f34", "milk bottle-54c16a16d66d533b", "pint glass-c0d00eae4c894fa5", "lid-c93364fbd585603e", "syrup jar-062cba3b1f9776fb", "honey jar-6bbdd500183e81c5", "dish cloth-cdf2f45e6b34cad2", "pint glass2-53ec5e52d0bbc16a", "plate-f11cbfd3aa5f073a", "banana-cfe0a9fa967f9bb1", "knife-6932a520e69bf940", "spoon-f050990f105e4e1a" ], "reasoning": "1. I watched for the user to pick up the dishcloth (dish cloth-cdf2f45e6b34cad2), which occurs at 02:14.\n2. The user then walks over to the marble countertop and wipes it down at 02:25, marking the first surface wiped down with the dishcloth (dish cloth-cdf2f45e6b34cad2).\n3. Then the user walks across the room and wipes down the small wooden table underneath the microwave at 02:42, making it the second surface wiped down with the dishcloth (dish cloth-cdf2f45e6b34cad2).\n4. The user does not wipe down any further surfaces with the dishcloth (dish cloth-cdf2f45e6b34cad2), making the answer 2.", "referenced_object_masks": [ "dish cloth-cdf2f45e6b34cad2" ] }, { "key": "P05-20240424-085204:e8699cf157d8ee9cce18101fa696c00601484d3f", "video_id": "P05-20240424-085204", "question": "Where is the compost bin in relation to the honey and syrup jars before I move them?", "answer_choice_0": "In front of them", "answer_choice_1": "To the right", "answer_choice_2": "Below", "answer_choice_3": "Above", "answer_choice_4": "To the left", "answer_id": 4, "answer": "To the left", "question_type": [ "Action Recognition", "Temporal Reasoning", "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "scales-96309edcd40d7f91", "bowl-20a43f99c89bf5c3", "porridge bag-e66efbacf4319f34", "milk bottle-54c16a16d66d533b", "pint glass-c0d00eae4c894fa5", "lid-c93364fbd585603e", "syrup jar-062cba3b1f9776fb", "honey jar-6bbdd500183e81c5", "dish cloth-cdf2f45e6b34cad2", "pint glass2-53ec5e52d0bbc16a", "plate-f11cbfd3aa5f073a", "banana-cfe0a9fa967f9bb1", "knife-6932a520e69bf940", "spoon-f050990f105e4e1a" ], "reasoning": "1. I looked for the compost bin , which is only identifiable as such at 03:52 when the user puts the banana (banana-cfe0a9fa967f9bb1) peel inside.\n2. The compost bin is on the far left of the marble countertop.\n3. I rewatched the video to find out where the honey jar (honey jar-6bbdd500183e81c5) and the syrup jar (syrup jar-062cba3b1f9776fb) are before the user moves them.\n4. At 01:56, the user moves the honey jar (honey jar-6bbdd500183e81c5) and the syrup jar (syrup jar-062cba3b1f9776fb) to a cabinet from the middle of the marble countertop.\n5. In relation to where the honey jar (honey jar-6bbdd500183e81c5) and the syrup jar (syrup jar-062cba3b1f9776fb) are initially, the compost bin is to their left.", "referenced_object_masks": [ "banana-cfe0a9fa967f9bb1", "honey jar-6bbdd500183e81c5", "syrup jar-062cba3b1f9776fb" ] }, { "key": "P05-20240424-090812:02a7ac8ddf83849e9ce15e0377956fcd76b9a534", "video_id": "P05-20240424-090812", "question": "How many times do I turn my head after I place the spoon back onto the plate but before I grab the bags from above the fridge?", "answer_choice_0": "2", "answer_choice_1": "3", "answer_choice_2": "1", "answer_choice_3": "5", "answer_choice_4": "4", "answer_id": 4, "answer": "4", "question_type": [ "Counting", "Temporal Reasoning", "Situational Awareness", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "tablespoon-fa0440d2fddbf9fb", "bowl-d03cc995995d5392", "bag of porridge-5cd01d253cb021cc", "other bag-7e9777f96bdab1e7", "big packet of pecan nuts-a8ba5029a8db1ff6", "packet of raisins-cda6f8f032180e16", "knife-03514e7fd630aa6b", "plate-db9f86cb51769045", "scales-fe9eeffcf5e0ccad" ], "reasoning": "1. I see the user take a spoon (tablespoon-fa0440d2fddbf9fb) off a plate (plate-db9f86cb51769045) at 00:07.\n1. Then, I see the user place this same spoon (tablespoon-fa0440d2fddbf9fb) back onto the plate (plate-db9f86cb51769045) after taking it off at 00:24. I know this is the moment for me to start looking for changes in the user's gaze.\n2. After this, the user shoots a glance toward the window to their right at 00:25.\n3. Then, the user turns around and shoots a glance to their left at the microwave at 00:26.\n4. Then, the user quickly moves their head to look at the sink at 00:26.\n4. Then the user moves their head to look at the fridge at 00:28.\n5. From 00:29-00:30, the user approaches the fridge and begins to grab the bag of oatmeal (bag of porridge-5cd01d253cb021cc) from it, so I stop counting glances. I also determined there were no head moves as the user walked forward.\n6. I add these head turns and I get 4.", "referenced_object_masks": [ "plate-db9f86cb51769045", "bag of porridge-5cd01d253cb021cc", "tablespoon-fa0440d2fddbf9fb" ] }, { "key": "P05-20240424-090812:3e92bd0572db43ee72c229a26189542f08153fd7", "video_id": "P05-20240424-090812", "question": "What is the last fruit the user places in the bowl?", "answer_choice_0": "Blueberries", "answer_choice_1": "Strawberries", "answer_choice_2": "Bananas", "answer_choice_3": "Raisins", "answer_choice_4": "Blackberries", "answer_id": 2, "answer": "Bananas", "question_type": [ "Action Recognition", "Object Recognition", "Counting" ], "objects_of_interest": [ "tablespoon-fa0440d2fddbf9fb", "bowl-d03cc995995d5392", "bag of porridge-5cd01d253cb021cc", "other bag-7e9777f96bdab1e7", "big packet of pecan nuts-a8ba5029a8db1ff6", "packet of raisins-cda6f8f032180e16", "knife-03514e7fd630aa6b", "plate-db9f86cb51769045", "scales-fe9eeffcf5e0ccad" ], "reasoning": "1. I noticed the user add ingredients to the bowl from 01:42-02:22.\n2. I noticed from 01:46-01:49 that the user pours a bag of raisins (packet of raisins-cda6f8f032180e16) into the bowl (bowl-d03cc995995d5392).\n3. I noticed from 02:00-02:08 that the user pours pecans (big packet of pecan nuts-a8ba5029a8db1ff6) into the bowl (bowl-d03cc995995d5392).\n4. Finally, I noticed from 02:17- 02:19 that the user scrapes bananas from a plate (plate-db9f86cb51769045) into the bowl (bowl-d03cc995995d5392).\n5. After this, the user adds no more ingredients.\n6. The first fruit added to the bowl (bowl-d03cc995995d5392) were raisins, and the second, and last, fruit added were bananas.\n7. Therefore, bananas is the answer.", "referenced_object_masks": [ "plate-db9f86cb51769045", "bowl-d03cc995995d5392", "packet of raisins-cda6f8f032180e16", "big packet of pecan nuts-a8ba5029a8db1ff6" ] }, { "key": "P05-20240424-090812:a909fbf5daf3604649b325b5ba94eea409f52a81", "video_id": "P05-20240424-090812", "question": "Relative to myself, from which section or sections of the island countertop do I pull to add ingredients to the porridge?", "answer_choice_0": "Upper Left and lower right", "answer_choice_1": "Upper right and lower left", "answer_choice_2": "Upper Left", "answer_choice_3": "Upper right", "answer_choice_4": "Lower Right", "answer_id": 0, "answer": "Upper Left and lower right", "question_type": [ "Situational Awareness", "Object Recognition", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "tablespoon-fa0440d2fddbf9fb", "bowl-d03cc995995d5392", "bag of porridge-5cd01d253cb021cc", "other bag-7e9777f96bdab1e7", "big packet of pecan nuts-a8ba5029a8db1ff6", "packet of raisins-cda6f8f032180e16", "knife-03514e7fd630aa6b", "plate-db9f86cb51769045", "scales-fe9eeffcf5e0ccad" ], "reasoning": "1. I looked for the time the user adds ingredients into the porridge from the table. I find this from 01:42-02:22.\n2. I noticed from 01:46-01:49 that the user pours a bag of raisins (packet of raisins-cda6f8f032180e16) into the bowl (bowl-d03cc995995d5392). The user grabs this ingredient from the upper left quadrant of the table. \n3. I noticed from 02:00-02:08 that the user pours pecans (big packet of pecan nuts-a8ba5029a8db1ff6) into the bowl (bowl-d03cc995995d5392). The user grabs this ingredient from the upper left quadrant of the table.\n4. Finally, I noticed from 02:17- 02:19 that the user scrapes bananas into the bowl (bowl-d03cc995995d5392). The user gets this ingredient from the lower right quadrant of the table.\n5.Therefore, the answer is upper left and lower right.", "referenced_object_masks": [ "bowl-d03cc995995d5392", "packet of raisins-cda6f8f032180e16", "big packet of pecan nuts-a8ba5029a8db1ff6" ] }, { "key": "P05-20240424-090812:c8ece3b75d6b3bd9bf8668d75d2d92bc05eb7e76", "video_id": "P05-20240424-090812", "question": "How many doors are closed before a door is opened (not counting doors already left open)?", "answer_choice_0": "1", "answer_choice_1": "2", "answer_choice_2": "0", "answer_choice_3": "4", "answer_choice_4": "3", "answer_id": 1, "answer": "2", "question_type": [ "Counting", "Audio Reasoning", "Action Recognition", "Situational Awareness", "Object Recognition", "State Changes" ], "objects_of_interest": [ "tablespoon-fa0440d2fddbf9fb", "bowl-d03cc995995d5392", "bag of porridge-5cd01d253cb021cc", "other bag-7e9777f96bdab1e7", "big packet of pecan nuts-a8ba5029a8db1ff6", "packet of raisins-cda6f8f032180e16", "knife-03514e7fd630aa6b", "plate-db9f86cb51769045", "scales-fe9eeffcf5e0ccad" ], "reasoning": "1. I looked for the first time a door is opened, and I found it at 01:22, when the user opens the microwave door .\n2. I watched from 00:00 to 01:22 to examine if there were any doors closed before the opening of the microwave door .\n3. I noticed from 00:01-00:06, the user pushes through an open door and closes it behind them. Though I do not see the door closing, I hear it.\n4. From 00:19-00:20, I noticed the user close a microwave door.\n5. From 00:20-01:22, the user doesn't close any more doors.\n6. Therefore, not counting doors already open, the user closes 2 doors before opening one.", "referenced_object_masks": [] }, { "key": "P05-20240424-090812:cb60e6ec51a69d5598e38bc2eb96a606c3911964", "video_id": "P05-20240424-090812", "question": "Before placing the bowl in the microwave, what is the ratio of clockwise stirs I make in the bowl to the number of times I audibly tap the bowl with a spoon?", "answer_choice_0": "10:1", "answer_choice_1": "5:2", "answer_choice_2": "15:7", "answer_choice_3": "3:5", "answer_choice_4": "2:9", "answer_id": 0, "answer": "10:1", "question_type": [ "Counting", "Object Recognition", "Audio Reasoning" ], "objects_of_interest": [ "tablespoon-fa0440d2fddbf9fb", "bowl-d03cc995995d5392", "bag of porridge-5cd01d253cb021cc", "other bag-7e9777f96bdab1e7", "big packet of pecan nuts-a8ba5029a8db1ff6", "packet of raisins-cda6f8f032180e16", "knife-03514e7fd630aa6b", "plate-db9f86cb51769045", "scales-fe9eeffcf5e0ccad" ], "reasoning": "1. I observe the user stir the food in the bowl (bowl-d03cc995995d5392) with the spoon (tablespoon-fa0440d2fddbf9fb) between 00:09 and 00:15.\n2. I count the user make 10 clockwise stirs between 00:09-00:15.\n3. I observe the user place the bowl (bowl-d03cc995995d5392) into the microwave at 00:18.\n4. Before 00:18, I can hear the user tap the spoon (tablespoon-fa0440d2fddbf9fb) on the side of the bowl (bowl-d03cc995995d5392) 1 time.\n5. Finally, I determine the ratio of clockwise stirs to the number of audible taps to be 10:1.", "referenced_object_masks": [ "bowl-d03cc995995d5392", "tablespoon-fa0440d2fddbf9fb" ] }, { "key": "P05-20240424-130119:09c582442430a3d5d71f99d13b586e38d8584531", "video_id": "P05-20240424-130119", "question": "How many more coffee mugs are in the dishwasher than cutting boards?", "answer_choice_0": "1", "answer_choice_1": "5", "answer_choice_2": "9", "answer_choice_3": "4", "answer_choice_4": "3", "answer_id": 1, "answer": "5", "question_type": [ "Counting", "Numerical Reasoning", "Object Recognition" ], "objects_of_interest": [ "plate-22f9b754e62d52c2", "tablespoon-7189ab977ddb7396", "colander of rice-114a0a8693299a50", "pan-2f1658705e3e07dc", "pan lid-b282d27e24b49810", "ladle-84f40862cd0f3616", "glass-37ddddf1f44b884f", "chopping board-355f8d36adfeadb6", "chopping board2-58ac7da77c8a3cfe", "mug-b6b0b2ca76eb88a8", "mug2-788889f0822832f3", "bowl of water-0a463ae8fc639235", "basket-a7f25f212129f83a", "knife-74fd5d33260a9183", "wooden chopping board-1f97802507a1b3ec", "dish cloth-097c5b40ae63d0b4", "jar of aubergine pickle-55ca436ea31414d7", "lid of aubergine pickle-cb862f8ad0925506", "empty pint glass-5393735af650f8bd" ], "reasoning": "1. I watched the video to find when the dishwasher was opened, noting that the user first opened it at 01:10.\n2. After seeing the user open the dishwasher , I counted how many mugs were in it . Specifically, I used the shot at 01:13 to count 7 mugs in the top rack of the dishwasher .\n3. I continued to watch the user load items in the dishwasher and saw them grab a cutting board (chopping board-355f8d36adfeadb6) at 01:36. \n4. After grabbing the cutting board (chopping board-355f8d36adfeadb6), the user opens the bottom rack of the dishwasher and loads the cutting board (chopping board-355f8d36adfeadb6) from 01:44-01:46.\n5. At this point, I counted 1 cutting board (chopping board2-58ac7da77c8a3cfe) already in the dishwasher, so this one, plus the one the user loads, totals 2 cutting boards (chopping board-355f8d36adfeadb6, chopping board2-58ac7da77c8a3cfe).\n6. I then took the total number of mugs which is 7 and subtracted the total number of cutting boards (chopping board-355f8d36adfeadb6, chopping board2-58ac7da77c8a3cfe) which is 2 and got a total of 5.", "referenced_object_masks": [ "chopping board2-58ac7da77c8a3cfe", "chopping board-355f8d36adfeadb6" ] }, { "key": "P05-20240424-130119:41d2cf9e2fecb552cf3c40fad27c3fc34c4d411b", "video_id": "P05-20240424-130119", "question": "After rinsing my spoon at the kitchen sink, what is the second item I touch with my right hand?", "answer_choice_0": "The jar of pickles", "answer_choice_1": "The microwave door", "answer_choice_2": "The spoon", "answer_choice_3": "The pan", "answer_choice_4": "The bowl of rice", "answer_id": 0, "answer": "The jar of pickles", "question_type": [ "Action Recognition", "Object Recognition", "Temporal Reasoning", "Counting" ], "objects_of_interest": [ "plate-22f9b754e62d52c2", "tablespoon-7189ab977ddb7396", "colander of rice-114a0a8693299a50", "pan-2f1658705e3e07dc", "pan lid-b282d27e24b49810", "ladle-84f40862cd0f3616", "glass-37ddddf1f44b884f", "chopping board-355f8d36adfeadb6", "chopping board2-58ac7da77c8a3cfe", "mug-b6b0b2ca76eb88a8", "mug2-788889f0822832f3", "bowl of water-0a463ae8fc639235", "basket-a7f25f212129f83a", "knife-74fd5d33260a9183", "wooden chopping board-1f97802507a1b3ec", "dish cloth-097c5b40ae63d0b4", "jar of aubergine pickle-55ca436ea31414d7", "lid of aubergine pickle-cb862f8ad0925506", "empty pint glass-5393735af650f8bd" ], "reasoning": "1. I watched the video until 03:31-03:36, when the user rinses the spoon (tablespoon-7189ab977ddb7396) at the kitchen sink .\n2. From this point, I continued to watch the video, observing the user open the refrigerator door at 03:38 with their right hand. This makes the fridge door the first item touched with the right hand after rinsing the spoon (tablespoon-7189ab977ddb7396).\n3. I then continued to watch and saw the user stand and look into the refrigerator from 03:40-03:45 before bending down and reaching to grab the jar of pickles (jar of aubergine pickle-55ca436ea31414d7) with their right hand at 03:45-03:47.\n4. This makes the jar of pickles (jar of aubergine pickle-55ca436ea31414d7) the second item the user grabbed with their right hand after rinsing the spoon (tablespoon-7189ab977ddb7396).", "referenced_object_masks": [ "jar of aubergine pickle-55ca436ea31414d7", "tablespoon-7189ab977ddb7396" ] }, { "key": "P05-20240424-130119:6cd830913f85b888fbecf3f5160d506a965b542b", "video_id": "P05-20240424-130119", "question": "When I put the knife back in the drawer, where is the pan in relation to me?", "answer_choice_0": "12 o'clock", "answer_choice_1": "8 o'clock", "answer_choice_2": "4 o'clock", "answer_choice_3": "6 o'clock", "answer_choice_4": "5 o'clock", "answer_id": 1, "answer": "8 o'clock", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Object Recognition" ], "objects_of_interest": [ "plate-22f9b754e62d52c2", "tablespoon-7189ab977ddb7396", "colander of rice-114a0a8693299a50", "pan-2f1658705e3e07dc", "pan lid-b282d27e24b49810", "ladle-84f40862cd0f3616", "glass-37ddddf1f44b884f", "chopping board-355f8d36adfeadb6", "chopping board2-58ac7da77c8a3cfe", "mug-b6b0b2ca76eb88a8", "mug2-788889f0822832f3", "bowl of water-0a463ae8fc639235", "basket-a7f25f212129f83a", "knife-74fd5d33260a9183", "wooden chopping board-1f97802507a1b3ec", "dish cloth-097c5b40ae63d0b4", "jar of aubergine pickle-55ca436ea31414d7", "lid of aubergine pickle-cb862f8ad0925506", "empty pint glass-5393735af650f8bd" ], "reasoning": "1. I watched the video to see when the user places the knife (knife-74fd5d33260a9183) in the drawer which happened from 02:28-02:32. I noticed that the user is angled towards the corner of the counter at the moment and standing to the right of the drawer. \n2. Next, I went back to identify where the pan (pan-2f1658705e3e07dc) was at that moment and observed it (pan-2f1658705e3e07dc) on the counter directly behind the knife drawer at 02:00. I realized the user never moves the pan (pan-2f1658705e3e07dc) between that moment and 02:28-02:32. \n3. Reasoning about the scene, I realized that because of the direction the user is facing when they put the knife (knife-74fd5d33260a9183) away, the pan (pan-2f1658705e3e07dc) is at their relative 8 o'clock between 02:28-02:32.", "referenced_object_masks": [ "pan-2f1658705e3e07dc", "knife-74fd5d33260a9183" ] }, { "key": "P05-20240424-130119:833f095d6f9feeccb53b9d5c13d4ce6c2c1b14d2", "video_id": "P05-20240424-130119", "question": "How many more items are in my hands when I close the refrigerator door for the first time than when I initially opened the refrigerator door?", "answer_choice_0": "1", "answer_choice_1": "3", "answer_choice_2": "4", "answer_choice_3": "2", "answer_choice_4": "0", "answer_id": 0, "answer": "1", "question_type": [ "Action Recognition", "Counting", "Object Recognition" ], "objects_of_interest": [ "plate-22f9b754e62d52c2", "tablespoon-7189ab977ddb7396", "colander of rice-114a0a8693299a50", "pan-2f1658705e3e07dc", "pan lid-b282d27e24b49810", "ladle-84f40862cd0f3616", "glass-37ddddf1f44b884f", "chopping board-355f8d36adfeadb6", "chopping board2-58ac7da77c8a3cfe", "mug-b6b0b2ca76eb88a8", "mug2-788889f0822832f3", "bowl of water-0a463ae8fc639235", "basket-a7f25f212129f83a", "knife-74fd5d33260a9183", "wooden chopping board-1f97802507a1b3ec", "dish cloth-097c5b40ae63d0b4", "jar of aubergine pickle-55ca436ea31414d7", "lid of aubergine pickle-cb862f8ad0925506", "empty pint glass-5393735af650f8bd" ], "reasoning": "1. I watched the video and observed that from 00:15-00:18, the user grabs a spoon (tablespoon-7189ab977ddb7396) with their right hand and opens the refrigerator door while holding the spoon (tablespoon-7189ab977ddb7396). Thus, I realized they only have 1 object in their hand the first time they open the refrigerator door .\n2. I observed the user take a bowl of rice (colander of rice-114a0a8693299a50) out of the refrigerator at 00:22 with their left hand.\n3. I continued watching until the user closed the refrigerator door at 00:24. I realized at that moment they had 2 objects in their hands. \n4. Therefore, the user has one additional item in their hands when they close the refrigerator door than they do when they open the door for the first time.", "referenced_object_masks": [ "colander of rice-114a0a8693299a50", "tablespoon-7189ab977ddb7396" ] }, { "key": "P05-20240424-130119:b5a059249f4b9be601beb49aea4275a8982363f7", "video_id": "P05-20240424-130119", "question": "What is the sum of the number of scoops of aubergine pickles I put on the plate and the number times I pump soap into my hand after scooping the pickles?", "answer_choice_0": "5", "answer_choice_1": "4", "answer_choice_2": "3", "answer_choice_3": "6", "answer_choice_4": "2", "answer_id": 4, "answer": "2", "question_type": [ "Counting", "Numerical Reasoning", "Action Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "plate-22f9b754e62d52c2", "tablespoon-7189ab977ddb7396", "colander of rice-114a0a8693299a50", "pan-2f1658705e3e07dc", "pan lid-b282d27e24b49810", "ladle-84f40862cd0f3616", "glass-37ddddf1f44b884f", "chopping board-355f8d36adfeadb6", "chopping board2-58ac7da77c8a3cfe", "mug-b6b0b2ca76eb88a8", "mug2-788889f0822832f3", "bowl of water-0a463ae8fc639235", "basket-a7f25f212129f83a", "knife-74fd5d33260a9183", "wooden chopping board-1f97802507a1b3ec", "dish cloth-097c5b40ae63d0b4", "jar of aubergine pickle-55ca436ea31414d7", "lid of aubergine pickle-cb862f8ad0925506", "empty pint glass-5393735af650f8bd" ], "reasoning": "1. I started the video and watched to find when the user scoops the pickles onto the plate (plate-22f9b754e62d52c2). I observed the user grab the jar of pickles (jar of aubergine pickle-55ca436ea31414d7) from 03:45-03:47.\n2. I then saw the user scoop one scoop of pickles from the jar (jar of aubergine pickle-55ca436ea31414d7) and put them onto the plate (plate-22f9b754e62d52c2) from 04:06-04:10.\n3. After doing one scoop, the user puts the lid (lid of aubergine pickle-cb862f8ad0925506) back on the jar (jar of aubergine pickle-55ca436ea31414d7) before returning it back to the fridge from 04:15-04:17.\n4. Once the jar of pickles (jar of aubergine pickle-55ca436ea31414d7) is back in the refrigerator , I observed the user walk to the sink , at which they wash their hands from 04:20-04:32.\n5. During this timespan, I observed the user get one pump of soap at 04:21-04:22, indicating that only one pump of soap was used.\n6. I then did the basic math of 1 + 1 = 2.", "referenced_object_masks": [ "jar of aubergine pickle-55ca436ea31414d7", "lid of aubergine pickle-cb862f8ad0925506", "plate-22f9b754e62d52c2" ] }, { "key": "P05-20240424-131231:378dba5fa6555ce57c2873295f694a78918aa9d3", "video_id": "P05-20240424-131231", "question": "As I walk in the door to the kitchen with a dirty dish and spoon, where is the kitchen sink in relation to where I am?", "answer_choice_0": "To the right of me", "answer_choice_1": "Across the kitchen and to the left of me", "answer_choice_2": "To the left of me", "answer_choice_3": "In front of me", "answer_choice_4": "Across the kitchen and to the right of me", "answer_id": 2, "answer": "To the left of me", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "plate-8f7b6ec255a48d21", "tablespoon-b985cc1b65a3f2cd", "empty pint glass-623b532bf587c623" ], "reasoning": "1. I watched the whole video and saw the user close the kitchen door at 0:04.\n2. They then turned left to go to the dishwasher to put the dirty plate (plate-8f7b6ec255a48d21) and dirty spoon (tablespoon-b985cc1b65a3f2cd) in the dishwasher .\n3. The kitchen sink is to the right of the dishwasher.", "referenced_object_masks": [ "plate-8f7b6ec255a48d21", "tablespoon-b985cc1b65a3f2cd" ] }, { "key": "P05-20240424-131231:71ee85b57cf5d15c84903e5cd47a62d2d8eb34fc", "video_id": "P05-20240424-131231", "question": "What is the difference between the number of seconds I hold the glass of water to my face and the number of chairs at the kitchen table?", "answer_choice_0": "0", "answer_choice_1": "4", "answer_choice_2": "1", "answer_choice_3": "2", "answer_choice_4": "3", "answer_id": 3, "answer": "2", "question_type": [ "Counting", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "plate-8f7b6ec255a48d21", "tablespoon-b985cc1b65a3f2cd", "empty pint glass-623b532bf587c623" ], "reasoning": "1. I watched the whole video and saw at 00:05 3 chairs at the kitchen table .\n2. From 00:25 to 00:30, the user held the glass of water (empty pint glass-623b532bf587c623) up to their face for 5 seconds.\n3. The number of seconds the user holds the glass of water (empty pint glass-623b532bf587c623) to their face, 5, minus the number of chairs at the kitchen table , 3, is 2.", "referenced_object_masks": [ "empty pint glass-623b532bf587c623" ] }, { "key": "P05-20240424-131231:8cdcb43404c7d6f126841cf21764306de2e90a46", "video_id": "P05-20240424-131231", "question": "What is the ratio of the number of white mugs on the shelf over the sink to the number of lights on the wall above the sink? Reduce the ratio if possible.", "answer_choice_0": "1:3", "answer_choice_1": "1:1", "answer_choice_2": "1:2", "answer_choice_3": "3:2", "answer_choice_4": "2:3", "answer_id": 1, "answer": "1:1", "question_type": [ "Egocentric & Spatial Perception", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "plate-8f7b6ec255a48d21", "tablespoon-b985cc1b65a3f2cd", "empty pint glass-623b532bf587c623" ], "reasoning": "1. I watched the whole video and saw 3 white mugs on the shelf above the sink at 00:22 when the user picked up the empty pint glass (empty pint glass-623b532bf587c623) to fill with water.\n2. At 00:27, the user was drinking from the pint glass (empty pint glass-623b532bf587c623) and moved their head up to drink. There were 3 white lights on the wall above the sink .\n3. The ratio of the number of white mugs on the shelf above the sink , 3 to the number of lights on the wall above the sink , 3 is 3:3 which reduces to 1:1.", "referenced_object_masks": [ "empty pint glass-623b532bf587c623" ] }, { "key": "P05-20240424-131231:a5cc1411818ac057e2d380a5779892346aeb7c65", "video_id": "P05-20240424-131231", "question": "What steps would I need to take to go from the spot where I drink water to be standing in front of the wooden pentagonal corner unit?", "answer_choice_0": "Turn left, walk forward, and turn right", "answer_choice_1": "Turn completely around, walk forward, and turn right", "answer_choice_2": "Turn left, walk forward, and turn left", "answer_choice_3": "Turn right, walk forward, and turn left", "answer_choice_4": "Turn completely around, walk forward, and turn left", "answer_id": 0, "answer": "Turn left, walk forward, and turn right", "question_type": [ "Egocentric & Spatial Perception", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "plate-8f7b6ec255a48d21", "tablespoon-b985cc1b65a3f2cd", "empty pint glass-623b532bf587c623" ], "reasoning": "1. I watched the video to determine the locations of the wooden pentagonal corner unit and the spot where the user drinks water. The first one of these I notice is the corner unit , which the user walks past at 00:04 as they enter the kitchen. \n2. At 00:04, the corner unit at this point is below the user and on their right.\n3. Then, I looked for the spot where the user drank water. I found this moment from 00:26-00:30, when the user drinks water from a glass (empty pint glass-623b532bf587c623) which they picked up at 00:21. They are drinking water in front of the sink .\n4. Then, I determined what steps the user would need to take to go from the sink to the corner unit . At 00:06, the user turned to the left from the front door of the kitchen to move toward the sink which was a few steps away and on the left.\n5. Taking these directions backward, the user would need to turn left, walk forward, and turn right to arrive at the corner unit from the spot where they drank water.", "referenced_object_masks": [ "empty pint glass-623b532bf587c623" ] }, { "key": "P05-20240424-131231:c56e4a1dc1cc62d1577fc72b474ee3537fa455c9", "video_id": "P05-20240424-131231", "question": "If I take the total number of visible items in the dishwasher after I place my items I enter the kitchen with in the dishwasher and multiply this number by the number of shelves of glassware in the cabinet above the dishwasher, what number would I get?", "answer_choice_0": "30", "answer_choice_1": "12", "answer_choice_2": "15", "answer_choice_3": "18", "answer_choice_4": "24", "answer_id": 0, "answer": "30", "question_type": [ "Counting", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "plate-8f7b6ec255a48d21", "tablespoon-b985cc1b65a3f2cd", "empty pint glass-623b532bf587c623" ], "reasoning": "1. I watched the whole video and saw the user at 00:08 open the dishwasher , pull out the top shelf and put a spoon (tablespoon-b985cc1b65a3f2cd) in it. \n2. There is now a total of 5 items on the top shelf .\n3. At 00:13, the user pulls the bottom shelf of the dishwasher out to put a dirty plate (plate-8f7b6ec255a48d21) in it.\n4. There is now a total of 10 items on the bottom shelf .\n5. The number of items on the top shelf , 5, plus the number of items on the bottom shelf , 10, totals 15.\n6. At 00:21, the user can see that there are 2 shelves of glassware in the cabinet above the dishwasher .\n7. The total number of items in the dishwasher multiplied by the number of shelves of glassware in the cabinet over the dishwasher, 2, is 30.", "referenced_object_masks": [ "plate-8f7b6ec255a48d21", "tablespoon-b985cc1b65a3f2cd" ] }, { "key": "P05-20240424-171038:107756a40a9beeeda21706ae576587e4fbce02ba", "video_id": "P05-20240424-171038", "question": "After opening the dishwasher for the first time, what is the second object I place down with my right hand?", "answer_choice_0": "Spoon", "answer_choice_1": "Bag of carrots", "answer_choice_2": "Plate of cheese", "answer_choice_3": "White bowl", "answer_choice_4": "Plastic bag", "answer_id": 2, "answer": "Plate of cheese", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "pan-5bce93115165cf89", "spoon-150e26d4702a2065", "plate of cheese-16f8b2b3dfd8102c", "bag of carrots-6af16f6d61949d59", "plastic bag-5121db53379a5d00", "blocks of cheese-d2124a48e88f55e7", "blocks of cheese2-ac828305242ade1b", "kettle-3927a5ae0ed84f7d", "pot-1629a9c12b0e36fe", "pot lid-04efdf9cea4a36f5", "scales-6e619e343b9401a0", "packet of pasta-6b251fa48b473313", "smaller pestle-1eec1296c0dd34e4", "mortar-aa7286f2a0975017", "large pestle-7b551929f8b7c0eb", "pesto plant-0cdd77a37831c64d", "chopping board-54607b0cb7c37e14", "teaspoon-5be89dc891be0606", "cashew nuts bag-1421f96ac9e12542", "bag of pine nuts-2ab49a79525989a5", "lid of pan-af9ff43e4a614834", "metal bowl-c042e5990a357166", "salt contailer-271f5013e03e46a8", "wooden spoon-405673e4db17da56", "phone-698fa3fcc552f35a", "cheese grater-6acb57e28e3818e6", "cheese grater cover-4e2ec97c28478338", "block of cheese2 packaging-e487dcb278269e1d", "bottle of olive oil-e9fc590b25859c9a", "bag of frozen peas-56772f4f4b4b1ea5", "bag of lettuce-192e28336ea8deac", "one lettuce-4f14c3a890a458c6", "packet of tomatoes-39033ba43299deda", "glass bowl-dd2812de4d1f7e95", "lid of olive oil bottle-45c77bddac5ede03", "jar of pepper-30bb524882ba4c02", "knife-91bd5964637b8a0e", "bottle of vinegar-2acc3eac73a6421c", "colander-dcae79cb168181c7", "spoon2-1af6fcb85986d83d", "spoon3-71bff07794a2090c", "oven glove1-6fce4e0a8295c9d2", "oven glove2-7a8019a484c76b42" ], "reasoning": "1. I locate the dishwasher at 00:08 to the left of the user as they walk into the kitchen.\n2. I observe the user open the dishwasher at 00:11.\n3. I observe the user place down the first object at 00:22 when they place the spoon (spoon-150e26d4702a2065) down into the white bowl to the right of the sink .\n4. I observe the user place down the second object at 00:32 when they place the plate of cheese (plate of cheese-16f8b2b3dfd8102c) down on the top shelf of the fridge .", "referenced_object_masks": [ "spoon-150e26d4702a2065", "plate of cheese-16f8b2b3dfd8102c" ] }, { "key": "P05-20240424-171038:250a2098f9983e49d9b4c44e84cea3d6fc28f53c", "video_id": "P05-20240424-171038", "question": "When I place the electric kettle down, and taking into account only the mugs to the right side of the sink faucet, what is the product of the number of hanging mugs pointing leftward and the number pointing rightward?", "answer_choice_0": "9", "answer_choice_1": "36", "answer_choice_2": "30", "answer_choice_3": "8", "answer_choice_4": "6", "answer_id": 0, "answer": "9", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counting", "Numerical Reasoning" ], "objects_of_interest": [ "pan-5bce93115165cf89", "spoon-150e26d4702a2065", "plate of cheese-16f8b2b3dfd8102c", "bag of carrots-6af16f6d61949d59", "plastic bag-5121db53379a5d00", "blocks of cheese-d2124a48e88f55e7", "blocks of cheese2-ac828305242ade1b", "kettle-3927a5ae0ed84f7d", "pot-1629a9c12b0e36fe", "pot lid-04efdf9cea4a36f5", "scales-6e619e343b9401a0", "packet of pasta-6b251fa48b473313", "smaller pestle-1eec1296c0dd34e4", "mortar-aa7286f2a0975017", "large pestle-7b551929f8b7c0eb", "pesto plant-0cdd77a37831c64d", "chopping board-54607b0cb7c37e14", "teaspoon-5be89dc891be0606", "cashew nuts bag-1421f96ac9e12542", "bag of pine nuts-2ab49a79525989a5", "lid of pan-af9ff43e4a614834", "metal bowl-c042e5990a357166", "salt contailer-271f5013e03e46a8", "wooden spoon-405673e4db17da56", "phone-698fa3fcc552f35a", "cheese grater-6acb57e28e3818e6", "cheese grater cover-4e2ec97c28478338", "block of cheese2 packaging-e487dcb278269e1d", "bottle of olive oil-e9fc590b25859c9a", "bag of frozen peas-56772f4f4b4b1ea5", "bag of lettuce-192e28336ea8deac", "one lettuce-4f14c3a890a458c6", "packet of tomatoes-39033ba43299deda", "glass bowl-dd2812de4d1f7e95", "lid of olive oil bottle-45c77bddac5ede03", "jar of pepper-30bb524882ba4c02", "knife-91bd5964637b8a0e", "bottle of vinegar-2acc3eac73a6421c", "colander-dcae79cb168181c7", "spoon2-1af6fcb85986d83d", "spoon3-71bff07794a2090c", "oven glove1-6fce4e0a8295c9d2", "oven glove2-7a8019a484c76b42" ], "reasoning": "1. I locate the kettle (kettle-3927a5ae0ed84f7d) at 00:07 on the table directly ahead of the user in the kitchen.\n2. I observe the user pick up the kettle (kettle-3927a5ae0ed84f7d) with their right hand at 01:05. \n3. I observe the user place the kettle (kettle-3927a5ae0ed84f7d) back down at 01:16 after having filled it with water. \n4. At 01:16, I count 3 hanging mugs pointing leftward and 3 hanging mugs pointing rightward.\n5. I multiply 3 by 3 to get 9.", "referenced_object_masks": [ "kettle-3927a5ae0ed84f7d" ] }, { "key": "P05-20240424-171038:37ac0592d4e8425539fa56374f66cdf99a341783", "video_id": "P05-20240424-171038", "question": "As I am removing leaves from the pesto plant, how many mammal legs appear in total?", "answer_choice_0": "2", "answer_choice_1": "12", "answer_choice_2": "8", "answer_choice_3": "10", "answer_choice_4": "4", "answer_id": 1, "answer": "12", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counting", "Numerical Reasoning" ], "objects_of_interest": [ "pan-5bce93115165cf89", "spoon-150e26d4702a2065", "plate of cheese-16f8b2b3dfd8102c", "bag of carrots-6af16f6d61949d59", "plastic bag-5121db53379a5d00", "blocks of cheese-d2124a48e88f55e7", "blocks of cheese2-ac828305242ade1b", "kettle-3927a5ae0ed84f7d", "pot-1629a9c12b0e36fe", "pot lid-04efdf9cea4a36f5", "scales-6e619e343b9401a0", "packet of pasta-6b251fa48b473313", "smaller pestle-1eec1296c0dd34e4", "mortar-aa7286f2a0975017", "large pestle-7b551929f8b7c0eb", "pesto plant-0cdd77a37831c64d", "chopping board-54607b0cb7c37e14", "teaspoon-5be89dc891be0606", "cashew nuts bag-1421f96ac9e12542", "bag of pine nuts-2ab49a79525989a5", "lid of pan-af9ff43e4a614834", "metal bowl-c042e5990a357166", "salt contailer-271f5013e03e46a8", "wooden spoon-405673e4db17da56", "phone-698fa3fcc552f35a", "cheese grater-6acb57e28e3818e6", "cheese grater cover-4e2ec97c28478338", "block of cheese2 packaging-e487dcb278269e1d", "bottle of olive oil-e9fc590b25859c9a", "bag of frozen peas-56772f4f4b4b1ea5", "bag of lettuce-192e28336ea8deac", "one lettuce-4f14c3a890a458c6", "packet of tomatoes-39033ba43299deda", "glass bowl-dd2812de4d1f7e95", "lid of olive oil bottle-45c77bddac5ede03", "jar of pepper-30bb524882ba4c02", "knife-91bd5964637b8a0e", "bottle of vinegar-2acc3eac73a6421c", "colander-dcae79cb168181c7", "spoon2-1af6fcb85986d83d", "spoon3-71bff07794a2090c", "oven glove1-6fce4e0a8295c9d2", "oven glove2-7a8019a484c76b42" ], "reasoning": "1. I locate the pesto plant (pesto plant-0cdd77a37831c64d) at 00:04 by the windows to the right of the user. \n2. I observe the user remove leaves from the pesto plant (pesto plant-0cdd77a37831c64d) from 02:54 to 04:10.\n3. Between 02:54 and 04:10, I count the user's own 2 legs throughout, a neighbor's 2 legs outside the window at 03:01, and 8 legs between 2 dogs outside the window at 03:04.\n4. I add 2 and 2 and 8 to get 12.", "referenced_object_masks": [ "pesto plant-0cdd77a37831c64d" ] }, { "key": "P05-20240424-171038:67261028df52b321002047c8a4847a56f570b811", "video_id": "P05-20240424-171038", "question": "What direction would I have turned to look at the boiling pot on the stove if it were in the opposite direction while I was at the counter by the window?", "answer_choice_0": "90 degrees to the right", "answer_choice_1": "135 degrees to the left", "answer_choice_2": "90 degrees to the left", "answer_choice_3": "180 degrees behind", "answer_choice_4": "135 degrees to the right", "answer_id": 2, "answer": "90 degrees to the left", "question_type": [ "Situational Awareness", "Action Recognition", "Egocentric & Spatial Perception", "Temporal Reasoning", "Counterfactual", "Object Recognition" ], "objects_of_interest": [ "pan-5bce93115165cf89", "spoon-150e26d4702a2065", "plate of cheese-16f8b2b3dfd8102c", "bag of carrots-6af16f6d61949d59", "plastic bag-5121db53379a5d00", "blocks of cheese-d2124a48e88f55e7", "blocks of cheese2-ac828305242ade1b", "kettle-3927a5ae0ed84f7d", "pot-1629a9c12b0e36fe", "pot lid-04efdf9cea4a36f5", "scales-6e619e343b9401a0", "packet of pasta-6b251fa48b473313", "smaller pestle-1eec1296c0dd34e4", "mortar-aa7286f2a0975017", "large pestle-7b551929f8b7c0eb", "pesto plant-0cdd77a37831c64d", "chopping board-54607b0cb7c37e14", "teaspoon-5be89dc891be0606", "cashew nuts bag-1421f96ac9e12542", "bag of pine nuts-2ab49a79525989a5", "lid of pan-af9ff43e4a614834", "metal bowl-c042e5990a357166", "salt contailer-271f5013e03e46a8", "wooden spoon-405673e4db17da56", "phone-698fa3fcc552f35a", "cheese grater-6acb57e28e3818e6", "cheese grater cover-4e2ec97c28478338", "block of cheese2 packaging-e487dcb278269e1d", "bottle of olive oil-e9fc590b25859c9a", "bag of frozen peas-56772f4f4b4b1ea5", "bag of lettuce-192e28336ea8deac", "one lettuce-4f14c3a890a458c6", "packet of tomatoes-39033ba43299deda", "glass bowl-dd2812de4d1f7e95", "lid of olive oil bottle-45c77bddac5ede03", "jar of pepper-30bb524882ba4c02", "knife-91bd5964637b8a0e", "bottle of vinegar-2acc3eac73a6421c", "colander-dcae79cb168181c7", "spoon2-1af6fcb85986d83d", "spoon3-71bff07794a2090c", "oven glove1-6fce4e0a8295c9d2", "oven glove2-7a8019a484c76b42" ], "reasoning": "1. I watched the video and found the counter by the window at 00:59.\n2. I found the pot (pot-1629a9c12b0e36fe) at 01:24.\n3. At 08:57, I saw the boiling pot (pot-1629a9c12b0e36fe) on the stove when the user turned 90 degrees to the right to look from the counter by the window and towards the boiling pot (pot-1629a9c12b0e36fe).\n4. Since the user turned 90 degrees to the right to view the boiling pot (pot-1629a9c12b0e36fe) on the stove from the counter by the window, the boiling pot (pot-1629a9c12b0e36fe) was consequently to the right of the user while they were at the counter by the window.\n5. Since the opposite of \"to the right\" is \"to the left,\" the opposite of \"90 degrees to the right\" is \"90 degrees to the left.\"\n6. Therefore, the user would have turned 90 degrees to the left to look at the boiling pot (pot-1629a9c12b0e36fe) on the stove if it were in the opposite direction while they were at the counter by the window.", "referenced_object_masks": [ "pot-1629a9c12b0e36fe" ] }, { "key": "P05-20240424-171038:705e5249b8d3bed06a13f7289721c1d66b64eb33", "video_id": "P05-20240424-171038", "question": "Out of all the objects I touch during my first time at the fridge, which one stays in the same location?", "answer_choice_0": "Bag of carrots", "answer_choice_1": "Block of cheese", "answer_choice_2": "Bag of greens", "answer_choice_3": "Plastic bag", "answer_choice_4": "Plate of cheese", "answer_id": 2, "answer": "Bag of greens", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning", "State Changes" ], "objects_of_interest": [ "pan-5bce93115165cf89", "spoon-150e26d4702a2065", "plate of cheese-16f8b2b3dfd8102c", "bag of carrots-6af16f6d61949d59", "plastic bag-5121db53379a5d00", "blocks of cheese-d2124a48e88f55e7", "blocks of cheese2-ac828305242ade1b", "kettle-3927a5ae0ed84f7d", "pot-1629a9c12b0e36fe", "pot lid-04efdf9cea4a36f5", "scales-6e619e343b9401a0", "packet of pasta-6b251fa48b473313", "smaller pestle-1eec1296c0dd34e4", "mortar-aa7286f2a0975017", "large pestle-7b551929f8b7c0eb", "pesto plant-0cdd77a37831c64d", "chopping board-54607b0cb7c37e14", "teaspoon-5be89dc891be0606", "cashew nuts bag-1421f96ac9e12542", "bag of pine nuts-2ab49a79525989a5", "lid of pan-af9ff43e4a614834", "metal bowl-c042e5990a357166", "salt contailer-271f5013e03e46a8", "wooden spoon-405673e4db17da56", "phone-698fa3fcc552f35a", "cheese grater-6acb57e28e3818e6", "cheese grater cover-4e2ec97c28478338", "block of cheese2 packaging-e487dcb278269e1d", "bottle of olive oil-e9fc590b25859c9a", "bag of frozen peas-56772f4f4b4b1ea5", "bag of lettuce-192e28336ea8deac", "one lettuce-4f14c3a890a458c6", "packet of tomatoes-39033ba43299deda", "glass bowl-dd2812de4d1f7e95", "lid of olive oil bottle-45c77bddac5ede03", "jar of pepper-30bb524882ba4c02", "knife-91bd5964637b8a0e", "bottle of vinegar-2acc3eac73a6421c", "colander-dcae79cb168181c7", "spoon2-1af6fcb85986d83d", "spoon3-71bff07794a2090c", "oven glove1-6fce4e0a8295c9d2", "oven glove2-7a8019a484c76b42" ], "reasoning": "1. I locate the fridge at 01:28 in the corner of the room to the left of the user.\n2. I observe the user open the fridge at 00:24 and close it at 00:55.\n3. From 00:24 to 00:55, I observe the user touch the bag of greens at 00:29, the plate of cheese (plate of cheese-16f8b2b3dfd8102c) at 00:31, the bag of carrots (bag of carrots-6af16f6d61949d59) at 00:33, the plastic bag (plastic bag-5121db53379a5d00) at 00:37, the first block of cheese (blocks of cheese-d2124a48e88f55e7) at 00:42, and the second block of cheese (blocks of cheese2-ac828305242ade1b) at 00:46.\n4. I determine that the user moves every object they touch between 00:24 to 00:55 except for the bag of greens .", "referenced_object_masks": [ "blocks of cheese2-ac828305242ade1b", "plastic bag-5121db53379a5d00", "blocks of cheese-d2124a48e88f55e7", "plate of cheese-16f8b2b3dfd8102c", "bag of carrots-6af16f6d61949d59" ] }, { "key": "P05-20240424-171038:ac606d322e8357730f847cfba20d9d8e95d6d609", "video_id": "P05-20240424-171038", "question": "What prevented me from opening the jammed drawer?", "answer_choice_0": "The scale", "answer_choice_1": "The drawer was broken", "answer_choice_2": "Silverware", "answer_choice_3": "A pot lid", "answer_choice_4": "The pasta pack", "answer_id": 0, "answer": "The scale", "question_type": [ "Situational Awareness", "Action Recognition", "State Changes", "Object Recognition", "Cause and Effect" ], "objects_of_interest": [ "pan-5bce93115165cf89", "spoon-150e26d4702a2065", "plate of cheese-16f8b2b3dfd8102c", "bag of carrots-6af16f6d61949d59", "plastic bag-5121db53379a5d00", "blocks of cheese-d2124a48e88f55e7", "blocks of cheese2-ac828305242ade1b", "kettle-3927a5ae0ed84f7d", "pot-1629a9c12b0e36fe", "pot lid-04efdf9cea4a36f5", "scales-6e619e343b9401a0", "packet of pasta-6b251fa48b473313", "smaller pestle-1eec1296c0dd34e4", "mortar-aa7286f2a0975017", "large pestle-7b551929f8b7c0eb", "pesto plant-0cdd77a37831c64d", "chopping board-54607b0cb7c37e14", "teaspoon-5be89dc891be0606", "cashew nuts bag-1421f96ac9e12542", "bag of pine nuts-2ab49a79525989a5", "lid of pan-af9ff43e4a614834", "metal bowl-c042e5990a357166", "salt contailer-271f5013e03e46a8", "wooden spoon-405673e4db17da56", "phone-698fa3fcc552f35a", "cheese grater-6acb57e28e3818e6", "cheese grater cover-4e2ec97c28478338", "block of cheese2 packaging-e487dcb278269e1d", "bottle of olive oil-e9fc590b25859c9a", "bag of frozen peas-56772f4f4b4b1ea5", "bag of lettuce-192e28336ea8deac", "one lettuce-4f14c3a890a458c6", "packet of tomatoes-39033ba43299deda", "glass bowl-dd2812de4d1f7e95", "lid of olive oil bottle-45c77bddac5ede03", "jar of pepper-30bb524882ba4c02", "knife-91bd5964637b8a0e", "bottle of vinegar-2acc3eac73a6421c", "colander-dcae79cb168181c7", "spoon2-1af6fcb85986d83d", "spoon3-71bff07794a2090c", "oven glove1-6fce4e0a8295c9d2", "oven glove2-7a8019a484c76b42" ], "reasoning": "1. I watched the video and found the jammed drawer at 01:32.\n2. I saw the user was prevented from opening the jammed drawer at 01:32-02:00.\n3. I saw the user unjam, open, and close the drawer by removing a scale (scales-6e619e343b9401a0) from it at 02:00.\n4. Therefore, the scale (scales-6e619e343b9401a0) prevented the user from opening the jammed drawer.", "referenced_object_masks": [ "scales-6e619e343b9401a0" ] }, { "key": "P05-20240424-171038:c1e67db107937ea3fb1120163aab0ea078b9d411", "video_id": "P05-20240424-171038", "question": "After I pour water from the kettle to the pot for the first time and then place the kettle down, what is the second object I touch with my right hand?", "answer_choice_0": "Pot", "answer_choice_1": "Pestle", "answer_choice_2": "Spoon", "answer_choice_3": "Kettle", "answer_choice_4": "Mortar", "answer_id": 1, "answer": "Pestle", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counting" ], "objects_of_interest": [ "pan-5bce93115165cf89", "spoon-150e26d4702a2065", "plate of cheese-16f8b2b3dfd8102c", "bag of carrots-6af16f6d61949d59", "plastic bag-5121db53379a5d00", "blocks of cheese-d2124a48e88f55e7", "blocks of cheese2-ac828305242ade1b", "kettle-3927a5ae0ed84f7d", "pot-1629a9c12b0e36fe", "pot lid-04efdf9cea4a36f5", "scales-6e619e343b9401a0", "packet of pasta-6b251fa48b473313", "smaller pestle-1eec1296c0dd34e4", "mortar-aa7286f2a0975017", "large pestle-7b551929f8b7c0eb", "pesto plant-0cdd77a37831c64d", "chopping board-54607b0cb7c37e14", "teaspoon-5be89dc891be0606", "cashew nuts bag-1421f96ac9e12542", "bag of pine nuts-2ab49a79525989a5", "lid of pan-af9ff43e4a614834", "metal bowl-c042e5990a357166", "salt contailer-271f5013e03e46a8", "wooden spoon-405673e4db17da56", "phone-698fa3fcc552f35a", "cheese grater-6acb57e28e3818e6", "cheese grater cover-4e2ec97c28478338", "block of cheese2 packaging-e487dcb278269e1d", "bottle of olive oil-e9fc590b25859c9a", "bag of frozen peas-56772f4f4b4b1ea5", "bag of lettuce-192e28336ea8deac", "one lettuce-4f14c3a890a458c6", "packet of tomatoes-39033ba43299deda", "glass bowl-dd2812de4d1f7e95", "lid of olive oil bottle-45c77bddac5ede03", "jar of pepper-30bb524882ba4c02", "knife-91bd5964637b8a0e", "bottle of vinegar-2acc3eac73a6421c", "colander-dcae79cb168181c7", "spoon2-1af6fcb85986d83d", "spoon3-71bff07794a2090c", "oven glove1-6fce4e0a8295c9d2", "oven glove2-7a8019a484c76b42" ], "reasoning": "1. I locate the kettle (kettle-3927a5ae0ed84f7d) at 00:07 on the table directly ahead of the user in the kitchen.\n2. I locate the pot (pot-1629a9c12b0e36fe) at 01:23 in the dishwasher in front of the user.\n3. 07:48 to 08:00, I observe the user pouring water from the kettle (kettle-3927a5ae0ed84f7d) into the pot (pot-1629a9c12b0e36fe) with their right hand.\n4. I observe the user place the kettle (kettle-3927a5ae0ed84f7d) down on the counter to their left at 08:01.\n5. At 08:04, I observe the user pick up the spoon from the salt container (salt contailer-271f5013e03e46a8) with their right hand.\n6. At 08:08, I observe the user grab the pestle (large pestle-7b551929f8b7c0eb) on the counter in front of them with their right hand. \n7. I determine that the pestle (large pestle-7b551929f8b7c0eb) is the second object the user touches with their right hand.", "referenced_object_masks": [ "pot-1629a9c12b0e36fe", "large pestle-7b551929f8b7c0eb", "salt contailer-271f5013e03e46a8", "kettle-3927a5ae0ed84f7d" ] }, { "key": "P05-20240424-171038:db5c4c4101469286eee6a4d665b93110f348ad55", "video_id": "P05-20240424-171038", "question": "Where did I get the second item I put on the counter by the window from?", "answer_choice_0": "The stove", "answer_choice_1": "The refrigerator", "answer_choice_2": "A table", "answer_choice_3": "A cabinet", "answer_choice_4": "Another counter", "answer_id": 1, "answer": "The refrigerator", "question_type": [ "Situational Awareness", "Counting", "Action Recognition", "State Changes", "Object Recognition", "Temporal Reasoning", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "pan-5bce93115165cf89", "spoon-150e26d4702a2065", "plate of cheese-16f8b2b3dfd8102c", "bag of carrots-6af16f6d61949d59", "plastic bag-5121db53379a5d00", "blocks of cheese-d2124a48e88f55e7", "blocks of cheese2-ac828305242ade1b", "kettle-3927a5ae0ed84f7d", "pot-1629a9c12b0e36fe", "pot lid-04efdf9cea4a36f5", "scales-6e619e343b9401a0", "packet of pasta-6b251fa48b473313", "smaller pestle-1eec1296c0dd34e4", "mortar-aa7286f2a0975017", "large pestle-7b551929f8b7c0eb", "pesto plant-0cdd77a37831c64d", "chopping board-54607b0cb7c37e14", "teaspoon-5be89dc891be0606", "cashew nuts bag-1421f96ac9e12542", "bag of pine nuts-2ab49a79525989a5", "lid of pan-af9ff43e4a614834", "metal bowl-c042e5990a357166", "salt contailer-271f5013e03e46a8", "wooden spoon-405673e4db17da56", "phone-698fa3fcc552f35a", "cheese grater-6acb57e28e3818e6", "cheese grater cover-4e2ec97c28478338", "block of cheese2 packaging-e487dcb278269e1d", "bottle of olive oil-e9fc590b25859c9a", "bag of frozen peas-56772f4f4b4b1ea5", "bag of lettuce-192e28336ea8deac", "one lettuce-4f14c3a890a458c6", "packet of tomatoes-39033ba43299deda", "glass bowl-dd2812de4d1f7e95", "lid of olive oil bottle-45c77bddac5ede03", "jar of pepper-30bb524882ba4c02", "knife-91bd5964637b8a0e", "bottle of vinegar-2acc3eac73a6421c", "colander-dcae79cb168181c7", "spoon2-1af6fcb85986d83d", "spoon3-71bff07794a2090c", "oven glove1-6fce4e0a8295c9d2", "oven glove2-7a8019a484c76b42" ], "reasoning": "1. I watched the video and found the counter by the window at 00:59.\n2. I counted each item the user put on the counter by the window.\n3. At 01:00, I saw the user put 2 items on the counter by the window, which were blocks of cheese (blocks of cheese-d2124a48e88f55e7, blocks of cheese2-ac828305242ade1b).\n4. Before 01:00, I saw the user get the first block of cheese (blocks of cheese-d2124a48e88f55e7) from the refrigerator at 00:43.\n5. After 00:43 and before 01:00, I saw the user get the second block of cheese (blocks of cheese2-ac828305242ade1b) from the refrigerator at 00:50.\n6. Therefore, the user got the second item they put on the counter by the window from the refrigerator.", "referenced_object_masks": [ "blocks of cheese-d2124a48e88f55e7", "blocks of cheese2-ac828305242ade1b" ] }, { "key": "P05-20240424-171038:e8642f08b7a5decf737eb1ff2b09d5198d243560", "video_id": "P05-20240424-171038", "question": "What did I add to the boiling pot of pasta just before stirring for the third time?", "answer_choice_0": "Pine nuts", "answer_choice_1": "Salt and pepper", "answer_choice_2": "Boiling water", "answer_choice_3": "Frozen peas", "answer_choice_4": "Shredded cheese", "answer_id": 3, "answer": "Frozen peas", "question_type": [ "Object Recognition", "Action Recognition", "State Changes", "Temporal Reasoning", "Counting" ], "objects_of_interest": [ "pan-5bce93115165cf89", "spoon-150e26d4702a2065", "plate of cheese-16f8b2b3dfd8102c", "bag of carrots-6af16f6d61949d59", "plastic bag-5121db53379a5d00", "blocks of cheese-d2124a48e88f55e7", "blocks of cheese2-ac828305242ade1b", "kettle-3927a5ae0ed84f7d", "pot-1629a9c12b0e36fe", "pot lid-04efdf9cea4a36f5", "scales-6e619e343b9401a0", "packet of pasta-6b251fa48b473313", "smaller pestle-1eec1296c0dd34e4", "mortar-aa7286f2a0975017", "large pestle-7b551929f8b7c0eb", "pesto plant-0cdd77a37831c64d", "chopping board-54607b0cb7c37e14", "teaspoon-5be89dc891be0606", "cashew nuts bag-1421f96ac9e12542", "bag of pine nuts-2ab49a79525989a5", "lid of pan-af9ff43e4a614834", "metal bowl-c042e5990a357166", "salt contailer-271f5013e03e46a8", "wooden spoon-405673e4db17da56", "phone-698fa3fcc552f35a", "cheese grater-6acb57e28e3818e6", "cheese grater cover-4e2ec97c28478338", "block of cheese2 packaging-e487dcb278269e1d", "bottle of olive oil-e9fc590b25859c9a", "bag of frozen peas-56772f4f4b4b1ea5", "bag of lettuce-192e28336ea8deac", "one lettuce-4f14c3a890a458c6", "packet of tomatoes-39033ba43299deda", "glass bowl-dd2812de4d1f7e95", "lid of olive oil bottle-45c77bddac5ede03", "jar of pepper-30bb524882ba4c02", "knife-91bd5964637b8a0e", "bottle of vinegar-2acc3eac73a6421c", "colander-dcae79cb168181c7", "spoon2-1af6fcb85986d83d", "spoon3-71bff07794a2090c", "oven glove1-6fce4e0a8295c9d2", "oven glove2-7a8019a484c76b42" ], "reasoning": "1. I watched the video and found the pot (pot-1629a9c12b0e36fe) at 01:24.\n2. At 07:40, I saw the user put the pot (pot-1629a9c12b0e36fe) on the stove.\n3. At 07:48, I saw the user pour a kettle (kettle-3927a5ae0ed84f7d) of water into the pot (pot-1629a9c12b0e36fe) on the stove.\n4. At 09:02, I saw the user pour a pasta pack (packet of pasta-6b251fa48b473313) into the stove's pot (pot-1629a9c12b0e36fe) of boiling water.\n5. I counted each time the user stirred the stove's boiling pot (pot-1629a9c12b0e36fe) of pasta.\n6. At 09:25, I saw the user stir the stove's boiling pot (pot-1629a9c12b0e36fe) of pasta for the first time.\n7. At 14:15, I saw the user stir the stove's boiling pot (pot-1629a9c12b0e36fe) of pasta for the second time.\n8. At 15:51, I saw the user stir the stove's boiling pot (pot-1629a9c12b0e36fe) of pasta for the third time.\n9. Before 15:51, I saw the user add frozen peas from a bag (bag of frozen peas-56772f4f4b4b1ea5) to the boiling pot (pot-1629a9c12b0e36fe) of pasta at 15:37.\n10. Therefore, the user added frozen peas (bag of frozen peas-56772f4f4b4b1ea5) to the boiling pot (pot-1629a9c12b0e36fe) of pasta just before stirring for the third time.", "referenced_object_masks": [ "pot-1629a9c12b0e36fe", "packet of pasta-6b251fa48b473313", "kettle-3927a5ae0ed84f7d", "bag of frozen peas-56772f4f4b4b1ea5" ] }, { "key": "P05-20240424-171038:f38297087e55db477b22a0755b1c413ca34575ba", "video_id": "P05-20240424-171038", "question": "Do I lift my pestle for a greater number of times than the time I spend (in seconds) mashing the leaves on my first bout of mashing pesto leaves?", "answer_choice_0": ", Yes, I lift my pestle by more than 5 times", "answer_choice_1": "Yes, I lift my pestle by more than 10 times", "answer_choice_2": "No, I mash by more than 10 seconds", "answer_choice_3": "No, I mash by more than 5 seconds", "answer_choice_4": "No, the numbers are equal", "answer_id": 2, "answer": "No, I mash by more than 10 seconds", "question_type": [ "Action Recognition", "Object Recognition", "Temporal Reasoning", "Counting", "Numerical Reasoning", "State Changes" ], "objects_of_interest": [ "pan-5bce93115165cf89", "spoon-150e26d4702a2065", "plate of cheese-16f8b2b3dfd8102c", "bag of carrots-6af16f6d61949d59", "plastic bag-5121db53379a5d00", "blocks of cheese-d2124a48e88f55e7", "blocks of cheese2-ac828305242ade1b", "kettle-3927a5ae0ed84f7d", "pot-1629a9c12b0e36fe", "pot lid-04efdf9cea4a36f5", "scales-6e619e343b9401a0", "packet of pasta-6b251fa48b473313", "smaller pestle-1eec1296c0dd34e4", "mortar-aa7286f2a0975017", "large pestle-7b551929f8b7c0eb", "pesto plant-0cdd77a37831c64d", "chopping board-54607b0cb7c37e14", "teaspoon-5be89dc891be0606", "cashew nuts bag-1421f96ac9e12542", "bag of pine nuts-2ab49a79525989a5", "lid of pan-af9ff43e4a614834", "metal bowl-c042e5990a357166", "salt contailer-271f5013e03e46a8", "wooden spoon-405673e4db17da56", "phone-698fa3fcc552f35a", "cheese grater-6acb57e28e3818e6", "cheese grater cover-4e2ec97c28478338", "block of cheese2 packaging-e487dcb278269e1d", "bottle of olive oil-e9fc590b25859c9a", "bag of frozen peas-56772f4f4b4b1ea5", "bag of lettuce-192e28336ea8deac", "one lettuce-4f14c3a890a458c6", "packet of tomatoes-39033ba43299deda", "glass bowl-dd2812de4d1f7e95", "lid of olive oil bottle-45c77bddac5ede03", "jar of pepper-30bb524882ba4c02", "knife-91bd5964637b8a0e", "bottle of vinegar-2acc3eac73a6421c", "colander-dcae79cb168181c7", "spoon2-1af6fcb85986d83d", "spoon3-71bff07794a2090c", "oven glove1-6fce4e0a8295c9d2", "oven glove2-7a8019a484c76b42" ], "reasoning": "1. I look for the first time the user begins to mash pesto leaves . I find it between 04:23-04:54.\n2. I realize that this mashing time lasts a total of 31 seconds, but since the user takes 1 second to put a leaf that had escaped the bowl back into it, from 04:40-04:41, I discount 1 second, getting a total of 30 seconds. \n3. During this time, I count that user lifted the handle of their pestle (large pestle-7b551929f8b7c0eb) a total of 16 times.\n4. I realize that the user did not lift the handle of their pestle (large pestle-7b551929f8b7c0eb) for longer than the time the user mashed the pesto leaves .\n5. I also realize that the difference in time and the number of times the pestle was lifted (large pestle-7b551929f8b7c0eb) equals 14.\n6. Therefore, the answer is no, the user mashes by 10 seconds more than the number of times user lifts the pestle handle (large pestle-7b551929f8b7c0eb) during the first bout of pesto leaf mashing.", "referenced_object_masks": [ "large pestle-7b551929f8b7c0eb" ] }, { "key": "P05-20240424-175101:68f06cbef4406363810219cf29aa6bc45e3143a9", "video_id": "P05-20240424-175101", "question": "How many times did I walk to the opposite side of the kitchen up to the moment when I stored the cutting boards?", "answer_choice_0": "4", "answer_choice_1": "7", "answer_choice_2": "6", "answer_choice_3": "3", "answer_choice_4": "5", "answer_id": 2, "answer": "6", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Situational Awareness", "Object Recognition" ], "objects_of_interest": [ "plate-b185c804f5490c57", "packet of tomatoes-b76b90e5d8e463d9", "cheese-55adacf77b4c5034", "plate of food-7dc9c8078a1b8075", "knife-408def669796ca2a", "colander-57033ac868fbe46e", "weighing scales-6f02c2f3b394c0f0", "packet of pasta-a5a29111cb60d9db", "cap of olive oil-ce6d053d8fb2c926", "bottle of olive oil-20ade57d6da59cb7", "chopping board1-3c23dbb1ce2e9edb", "chopping board2-421ae46a07508bb7", "chopping board3-07c8c4534343ea5a", "plastic bowl-1943375a96b885b6", "mug1-4057b0a505a65862", "mug2-aae217eb2a0db381", "mug3-3804bf5c545ee085", "plate2-db1bff4f31f97301", "plate3-4f36be01c285e9a5", "cheese grater-38d750442c5e22d2", "glass1-2ade766dd494cd40", "glass2-0aed6420b1721760", "glass3-769719917bdc90c1", "glass4-69b602579037839f", "mug4-826b865304765c6c", "mug5-577a27e3f5a1ea6a", "mug6-8e0237a15b3b2b5b", "mug7-c44ce680050023de", "mug8-67e92785d22e2c1f", "mug9-c8bbe48963042ff7", "mug10-77a8870488855874", "mug11-0dfb2b1fcce0e591", "blue bowl-32b7ad8ce5416bfd", "plastic bowl2-9e38c222d33b3b84", "metal bowl-314227e72911dc05", "glass5-6096fe1d44aecf19", "butter knives-1b1d5b9b0786ea9b", "more butter knives-d6b8b338f624d1d6", "more butter knives2-9b8bf2f2c80a49c2", "knife2-19f0b561bb79b337", "knife3-0af6456a13563029", "rubber spatula-2a94d3858a10d698", "small cheese grater-2573ed65891f7bfc", "chopping board4-2a4df70b217944d6", "spoon-9fdf80158f9f6654", "spoon2-ca04b911be25be90", "spoon3-316cc424aa54d0d5", "spoon4-6f16fec1b88b289f", "spoon5-dc65dc2eec8bf24d", "fork-2b3557a38ae9dc53", "fork2-65569e6414879013", "fork3-a151f980cb5acf64", "spoon6-32096ac2be5ddb7a", "plate4-320798bb7dea27f2", "plate6-105714fb0797e72f", "glass bowl-ef6ab5b9da748ae6", "spoon7-3a6aabae37bb0801", "cheese grater cover-77bd83ce625c7481", "metal pot-eac27c4004ab736d", "pestle-a7c471037df7f5ff", "washing up brush-f5bb0f8f0e6bf1b9", "mortar-6d125fc9cc4d0f54", "white bowl-29965a6a90d3fa27", "metal pot2-383564e61b3c2b5e", "spoon8-cabe1ce1f4cb8e9b", "ramekin-3f34e6aa3b90e816", "washing up bowl-c0512ba7066a356c", "serving spoon-e952f08bed0607f4", "plastic bucket-8aee120a2089c8a2", "lid of pot-0ece1de9e4f60eb4", "packet of nuts-4503ecd6e3dc0ec1", "plastic bag-c7e8ca21d1873fa6", "large white bowl-c4ba1aea4ad2e092", "stack of bowls-89714f4048bca79a", "cloth-51ee19ee59dd2e01", "right oven glove-df779de710404110", "left oven glove-2fcc425158d925b9", "wooden spoon-152d947ce6103cd8", "butter knife-dd2f5084e6195d1d", "plate5-6110dbab96ec0352", "marmalade-8d32e0822e94b5dd" ], "reasoning": "1. I found that the user stored the three chopping boards (chopping board1-3c23dbb1ce2e9edb), (chopping board2-421ae46a07508bb7), and (chopping board3-07c8c4534343ea5a) at 01:53.\n2. From the beginning of the video to 01:53, I counted each time that the user walks in the opposite direction to reach the other side of the kitchen. This happens at 00:22, 00:34, 00:40, 01:04, 01:35, and 01:51 for a total of 6 times.", "referenced_object_masks": [ "chopping board1-3c23dbb1ce2e9edb", "chopping board3-07c8c4534343ea5a", "chopping board2-421ae46a07508bb7" ] }, { "key": "P05-20240424-175101:7dae67b9708c90dc3da5e1bf89e5e400d825e75d", "video_id": "P05-20240424-175101", "question": "What is the difference between the time that passed before I took my third bite of food and the amount of time that passed before I closed the dishwasher?", "answer_choice_0": "1 minute and 37 seconds", "answer_choice_1": "1 minute and 35 seconds", "answer_choice_2": "1 minute and 31 seconds", "answer_choice_3": "1 minute and 33 seconds", "answer_choice_4": "1 minute and 39 seconds", "answer_id": 4, "answer": "1 minute and 39 seconds", "question_type": [ "Action Recognition", "Numerical Reasoning", "State Changes", "Temporal Reasoning" ], "objects_of_interest": [ "plate-b185c804f5490c57", "packet of tomatoes-b76b90e5d8e463d9", "cheese-55adacf77b4c5034", "plate of food-7dc9c8078a1b8075", "knife-408def669796ca2a", "colander-57033ac868fbe46e", "weighing scales-6f02c2f3b394c0f0", "packet of pasta-a5a29111cb60d9db", "cap of olive oil-ce6d053d8fb2c926", "bottle of olive oil-20ade57d6da59cb7", "chopping board1-3c23dbb1ce2e9edb", "chopping board2-421ae46a07508bb7", "chopping board3-07c8c4534343ea5a", "plastic bowl-1943375a96b885b6", "mug1-4057b0a505a65862", "mug2-aae217eb2a0db381", "mug3-3804bf5c545ee085", "plate2-db1bff4f31f97301", "plate3-4f36be01c285e9a5", "cheese grater-38d750442c5e22d2", "glass1-2ade766dd494cd40", "glass2-0aed6420b1721760", "glass3-769719917bdc90c1", "glass4-69b602579037839f", "mug4-826b865304765c6c", "mug5-577a27e3f5a1ea6a", "mug6-8e0237a15b3b2b5b", "mug7-c44ce680050023de", "mug8-67e92785d22e2c1f", "mug9-c8bbe48963042ff7", "mug10-77a8870488855874", "mug11-0dfb2b1fcce0e591", "blue bowl-32b7ad8ce5416bfd", "plastic bowl2-9e38c222d33b3b84", "metal bowl-314227e72911dc05", "glass5-6096fe1d44aecf19", "butter knives-1b1d5b9b0786ea9b", "more butter knives-d6b8b338f624d1d6", "more butter knives2-9b8bf2f2c80a49c2", "knife2-19f0b561bb79b337", "knife3-0af6456a13563029", "rubber spatula-2a94d3858a10d698", "small cheese grater-2573ed65891f7bfc", "chopping board4-2a4df70b217944d6", "spoon-9fdf80158f9f6654", "spoon2-ca04b911be25be90", "spoon3-316cc424aa54d0d5", "spoon4-6f16fec1b88b289f", "spoon5-dc65dc2eec8bf24d", "fork-2b3557a38ae9dc53", "fork2-65569e6414879013", "fork3-a151f980cb5acf64", "spoon6-32096ac2be5ddb7a", "plate4-320798bb7dea27f2", "plate6-105714fb0797e72f", "glass bowl-ef6ab5b9da748ae6", "spoon7-3a6aabae37bb0801", "cheese grater cover-77bd83ce625c7481", "metal pot-eac27c4004ab736d", "pestle-a7c471037df7f5ff", "washing up brush-f5bb0f8f0e6bf1b9", "mortar-6d125fc9cc4d0f54", "white bowl-29965a6a90d3fa27", "metal pot2-383564e61b3c2b5e", "spoon8-cabe1ce1f4cb8e9b", "ramekin-3f34e6aa3b90e816", "washing up bowl-c0512ba7066a356c", "serving spoon-e952f08bed0607f4", "plastic bucket-8aee120a2089c8a2", "lid of pot-0ece1de9e4f60eb4", "packet of nuts-4503ecd6e3dc0ec1", "plastic bag-c7e8ca21d1873fa6", "large white bowl-c4ba1aea4ad2e092", "stack of bowls-89714f4048bca79a", "cloth-51ee19ee59dd2e01", "right oven glove-df779de710404110", "left oven glove-2fcc425158d925b9", "wooden spoon-152d947ce6103cd8", "butter knife-dd2f5084e6195d1d", "plate5-6110dbab96ec0352", "marmalade-8d32e0822e94b5dd" ], "reasoning": "1. I observed that the user took their first bite of food from the glass bowl (glass bowl-ef6ab5b9da748ae6) at 04:10 and the second time at 04:12. For the third time, the user took a bite from the white bowl (white bowl-29965a6a90d3fa27) at 05:59.\n2. The door of the dishwasher was opened from the beginning of the video until the user closed the door at 07:38.\n3. This means that the door of the dishwasher was open for 1 minute and 39 seconds longer than the time that passed before the user took a third bite of food.", "referenced_object_masks": [ "glass bowl-ef6ab5b9da748ae6", "white bowl-29965a6a90d3fa27" ] }, { "key": "P05-20240424-175101:ce1ee3e940a7ce4ff0ad6478dc30be8f3376494c", "video_id": "P05-20240424-175101", "question": "What is the sum of the number of red onions on the lowest basket above the drawer where I stored the knife and the number of mugs stored above the sink after I store the second mug from the dishwasher?", "answer_choice_0": "15", "answer_choice_1": "19", "answer_choice_2": "14", "answer_choice_3": "17", "answer_choice_4": "16", "answer_id": 1, "answer": "19", "question_type": [ "Object Recognition", "Situational Awareness", "Counting", "Action Recognition", "Temporal Reasoning", "Numerical Reasoning" ], "objects_of_interest": [ "plate-b185c804f5490c57", "packet of tomatoes-b76b90e5d8e463d9", "cheese-55adacf77b4c5034", "plate of food-7dc9c8078a1b8075", "knife-408def669796ca2a", "colander-57033ac868fbe46e", "weighing scales-6f02c2f3b394c0f0", "packet of pasta-a5a29111cb60d9db", "cap of olive oil-ce6d053d8fb2c926", "bottle of olive oil-20ade57d6da59cb7", "chopping board1-3c23dbb1ce2e9edb", "chopping board2-421ae46a07508bb7", "chopping board3-07c8c4534343ea5a", "plastic bowl-1943375a96b885b6", "mug1-4057b0a505a65862", "mug2-aae217eb2a0db381", "mug3-3804bf5c545ee085", "plate2-db1bff4f31f97301", "plate3-4f36be01c285e9a5", "cheese grater-38d750442c5e22d2", "glass1-2ade766dd494cd40", "glass2-0aed6420b1721760", "glass3-769719917bdc90c1", "glass4-69b602579037839f", "mug4-826b865304765c6c", "mug5-577a27e3f5a1ea6a", "mug6-8e0237a15b3b2b5b", "mug7-c44ce680050023de", "mug8-67e92785d22e2c1f", "mug9-c8bbe48963042ff7", "mug10-77a8870488855874", "mug11-0dfb2b1fcce0e591", "blue bowl-32b7ad8ce5416bfd", "plastic bowl2-9e38c222d33b3b84", "metal bowl-314227e72911dc05", "glass5-6096fe1d44aecf19", "butter knives-1b1d5b9b0786ea9b", "more butter knives-d6b8b338f624d1d6", "more butter knives2-9b8bf2f2c80a49c2", "knife2-19f0b561bb79b337", "knife3-0af6456a13563029", "rubber spatula-2a94d3858a10d698", "small cheese grater-2573ed65891f7bfc", "chopping board4-2a4df70b217944d6", "spoon-9fdf80158f9f6654", "spoon2-ca04b911be25be90", "spoon3-316cc424aa54d0d5", "spoon4-6f16fec1b88b289f", "spoon5-dc65dc2eec8bf24d", "fork-2b3557a38ae9dc53", "fork2-65569e6414879013", "fork3-a151f980cb5acf64", "spoon6-32096ac2be5ddb7a", "plate4-320798bb7dea27f2", "plate6-105714fb0797e72f", "glass bowl-ef6ab5b9da748ae6", "spoon7-3a6aabae37bb0801", "cheese grater cover-77bd83ce625c7481", "metal pot-eac27c4004ab736d", "pestle-a7c471037df7f5ff", "washing up brush-f5bb0f8f0e6bf1b9", "mortar-6d125fc9cc4d0f54", "white bowl-29965a6a90d3fa27", "metal pot2-383564e61b3c2b5e", "spoon8-cabe1ce1f4cb8e9b", "ramekin-3f34e6aa3b90e816", "washing up bowl-c0512ba7066a356c", "serving spoon-e952f08bed0607f4", "plastic bucket-8aee120a2089c8a2", "lid of pot-0ece1de9e4f60eb4", "packet of nuts-4503ecd6e3dc0ec1", "plastic bag-c7e8ca21d1873fa6", "large white bowl-c4ba1aea4ad2e092", "stack of bowls-89714f4048bca79a", "cloth-51ee19ee59dd2e01", "right oven glove-df779de710404110", "left oven glove-2fcc425158d925b9", "wooden spoon-152d947ce6103cd8", "butter knife-dd2f5084e6195d1d", "plate5-6110dbab96ec0352", "marmalade-8d32e0822e94b5dd" ], "reasoning": "1. After washing the knife (knife-408def669796ca2a), the user placed it into a drawer at 01:08. At this moment, the lowest basket of onions is hanging above the drawer. There are 6 red onions .\n2. The user placed the first mug (mug1-4057b0a505a65862) from the dishwasher onto the ledge above the sink at 02:02. After this moment, there are 12 mugs stored above the sink, 5 on the top shelf and 7 mugs hanging below. \n3. The user places the second mug onto the ledge at 02:08, making the total of mugs 13, 6 on the shelf and 7 hanging below. \n4. The sum of 6 onions and 13 mugs is 19.", "referenced_object_masks": [ "knife-408def669796ca2a", "mug1-4057b0a505a65862" ] }, { "key": "P05-20240424-175101:d51abac126e1fdc853252694af605b53a3caeae4", "video_id": "P05-20240424-175101", "question": "If I wanted to take the cheese from where I stored it and bring it to the weighing scales, which set of directions would lead me there?", "answer_choice_0": "Walk to the left", "answer_choice_1": "Walk ahead and turn right after passing the dining table", "answer_choice_2": "Walk to the right", "answer_choice_3": "Walk straight ahead and turn left after passing the dining table", "answer_choice_4": "Walk straight ahead to the counter under the window", "answer_id": 4, "answer": "Walk straight ahead to the counter under the window", "question_type": [ "Egocentric & Spatial Perception", "Counterfactual", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "plate-b185c804f5490c57", "packet of tomatoes-b76b90e5d8e463d9", "cheese-55adacf77b4c5034", "plate of food-7dc9c8078a1b8075", "knife-408def669796ca2a", "colander-57033ac868fbe46e", "weighing scales-6f02c2f3b394c0f0", "packet of pasta-a5a29111cb60d9db", "cap of olive oil-ce6d053d8fb2c926", "bottle of olive oil-20ade57d6da59cb7", "chopping board1-3c23dbb1ce2e9edb", "chopping board2-421ae46a07508bb7", "chopping board3-07c8c4534343ea5a", "plastic bowl-1943375a96b885b6", "mug1-4057b0a505a65862", "mug2-aae217eb2a0db381", "mug3-3804bf5c545ee085", "plate2-db1bff4f31f97301", "plate3-4f36be01c285e9a5", "cheese grater-38d750442c5e22d2", "glass1-2ade766dd494cd40", "glass2-0aed6420b1721760", "glass3-769719917bdc90c1", "glass4-69b602579037839f", "mug4-826b865304765c6c", "mug5-577a27e3f5a1ea6a", "mug6-8e0237a15b3b2b5b", "mug7-c44ce680050023de", "mug8-67e92785d22e2c1f", "mug9-c8bbe48963042ff7", "mug10-77a8870488855874", "mug11-0dfb2b1fcce0e591", "blue bowl-32b7ad8ce5416bfd", "plastic bowl2-9e38c222d33b3b84", "metal bowl-314227e72911dc05", "glass5-6096fe1d44aecf19", "butter knives-1b1d5b9b0786ea9b", "more butter knives-d6b8b338f624d1d6", "more butter knives2-9b8bf2f2c80a49c2", "knife2-19f0b561bb79b337", "knife3-0af6456a13563029", "rubber spatula-2a94d3858a10d698", "small cheese grater-2573ed65891f7bfc", "chopping board4-2a4df70b217944d6", "spoon-9fdf80158f9f6654", "spoon2-ca04b911be25be90", "spoon3-316cc424aa54d0d5", "spoon4-6f16fec1b88b289f", "spoon5-dc65dc2eec8bf24d", "fork-2b3557a38ae9dc53", "fork2-65569e6414879013", "fork3-a151f980cb5acf64", "spoon6-32096ac2be5ddb7a", "plate4-320798bb7dea27f2", "plate6-105714fb0797e72f", "glass bowl-ef6ab5b9da748ae6", "spoon7-3a6aabae37bb0801", "cheese grater cover-77bd83ce625c7481", "metal pot-eac27c4004ab736d", "pestle-a7c471037df7f5ff", "washing up brush-f5bb0f8f0e6bf1b9", "mortar-6d125fc9cc4d0f54", "white bowl-29965a6a90d3fa27", "metal pot2-383564e61b3c2b5e", "spoon8-cabe1ce1f4cb8e9b", "ramekin-3f34e6aa3b90e816", "washing up bowl-c0512ba7066a356c", "serving spoon-e952f08bed0607f4", "plastic bucket-8aee120a2089c8a2", "lid of pot-0ece1de9e4f60eb4", "packet of nuts-4503ecd6e3dc0ec1", "plastic bag-c7e8ca21d1873fa6", "large white bowl-c4ba1aea4ad2e092", "stack of bowls-89714f4048bca79a", "cloth-51ee19ee59dd2e01", "right oven glove-df779de710404110", "left oven glove-2fcc425158d925b9", "wooden spoon-152d947ce6103cd8", "butter knife-dd2f5084e6195d1d", "plate5-6110dbab96ec0352", "marmalade-8d32e0822e94b5dd" ], "reasoning": "1. I observed that the user stores the cheese (cheese-55adacf77b4c5034) into the refrigerator at 00:28.\n2. I identified the weighing scale (weighing scales-6f02c2f3b394c0f0) at 00:19 and its position on the kitchen counter.\n3. When viewing the setting from the refrigerator at 00:33, to get from the refrigerator to the weighing scales (weighing scales-6f02c2f3b394c0f0), the user would have to walk straight ahead to the counter under the window without turning left or right.", "referenced_object_masks": [ "cheese-55adacf77b4c5034", "weighing scales-6f02c2f3b394c0f0" ] }, { "key": "P05-20240424-175101:f16ef29124682209ad4f5ad0a8afa197970af550", "video_id": "P05-20240424-175101", "question": "How many additional magnets on the refrigerator would be needed to equal the amount of drinking glasses that I placed on the middle shelf of the cabinet above the sink?", "answer_choice_0": "5", "answer_choice_1": "1", "answer_choice_2": "3", "answer_choice_3": "4", "answer_choice_4": "2", "answer_id": 1, "answer": "1", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Counting", "Situational Awareness", "Object Recognition" ], "objects_of_interest": [ "plate-b185c804f5490c57", "packet of tomatoes-b76b90e5d8e463d9", "cheese-55adacf77b4c5034", "plate of food-7dc9c8078a1b8075", "knife-408def669796ca2a", "colander-57033ac868fbe46e", "weighing scales-6f02c2f3b394c0f0", "packet of pasta-a5a29111cb60d9db", "cap of olive oil-ce6d053d8fb2c926", "bottle of olive oil-20ade57d6da59cb7", "chopping board1-3c23dbb1ce2e9edb", "chopping board2-421ae46a07508bb7", "chopping board3-07c8c4534343ea5a", "plastic bowl-1943375a96b885b6", "mug1-4057b0a505a65862", "mug2-aae217eb2a0db381", "mug3-3804bf5c545ee085", "plate2-db1bff4f31f97301", "plate3-4f36be01c285e9a5", "cheese grater-38d750442c5e22d2", "glass1-2ade766dd494cd40", "glass2-0aed6420b1721760", "glass3-769719917bdc90c1", "glass4-69b602579037839f", "mug4-826b865304765c6c", "mug5-577a27e3f5a1ea6a", "mug6-8e0237a15b3b2b5b", "mug7-c44ce680050023de", "mug8-67e92785d22e2c1f", "mug9-c8bbe48963042ff7", "mug10-77a8870488855874", "mug11-0dfb2b1fcce0e591", "blue bowl-32b7ad8ce5416bfd", "plastic bowl2-9e38c222d33b3b84", "metal bowl-314227e72911dc05", "glass5-6096fe1d44aecf19", "butter knives-1b1d5b9b0786ea9b", "more butter knives-d6b8b338f624d1d6", "more butter knives2-9b8bf2f2c80a49c2", "knife2-19f0b561bb79b337", "knife3-0af6456a13563029", "rubber spatula-2a94d3858a10d698", "small cheese grater-2573ed65891f7bfc", "chopping board4-2a4df70b217944d6", "spoon-9fdf80158f9f6654", "spoon2-ca04b911be25be90", "spoon3-316cc424aa54d0d5", "spoon4-6f16fec1b88b289f", "spoon5-dc65dc2eec8bf24d", "fork-2b3557a38ae9dc53", "fork2-65569e6414879013", "fork3-a151f980cb5acf64", "spoon6-32096ac2be5ddb7a", "plate4-320798bb7dea27f2", "plate6-105714fb0797e72f", "glass bowl-ef6ab5b9da748ae6", "spoon7-3a6aabae37bb0801", "cheese grater cover-77bd83ce625c7481", "metal pot-eac27c4004ab736d", "pestle-a7c471037df7f5ff", "washing up brush-f5bb0f8f0e6bf1b9", "mortar-6d125fc9cc4d0f54", "white bowl-29965a6a90d3fa27", "metal pot2-383564e61b3c2b5e", "spoon8-cabe1ce1f4cb8e9b", "ramekin-3f34e6aa3b90e816", "washing up bowl-c0512ba7066a356c", "serving spoon-e952f08bed0607f4", "plastic bucket-8aee120a2089c8a2", "lid of pot-0ece1de9e4f60eb4", "packet of nuts-4503ecd6e3dc0ec1", "plastic bag-c7e8ca21d1873fa6", "large white bowl-c4ba1aea4ad2e092", "stack of bowls-89714f4048bca79a", "cloth-51ee19ee59dd2e01", "right oven glove-df779de710404110", "left oven glove-2fcc425158d925b9", "wooden spoon-152d947ce6103cd8", "butter knife-dd2f5084e6195d1d", "plate5-6110dbab96ec0352", "marmalade-8d32e0822e94b5dd" ], "reasoning": "1. I identified the refrigerator at 00:23. There is a single magnet on the refrigerator.\n2. The user placed the first glass (glass1-2ade766dd494cd40) on the top shelf of the cabinet at 02:18 and the second glass (glass2-0aed6420b1721760) at 02:21. The user doesn't place any more glasses on the second shelf of this cabinet.\n3. To equal the number of glasses that the user placed on the second shelf, there would need to be 1 additional magnet on the refrigerator.", "referenced_object_masks": [ "glass2-0aed6420b1721760", "glass1-2ade766dd494cd40" ] }, { "key": "P05-20240425-082420:32581734a90b0a7d4d0df2db0a36433e4850246a", "video_id": "P05-20240425-082420", "question": "What would be the total weight of dry oatmeal oats if I wanted to make a second serving of oatmeal for a friend?", "answer_choice_0": "57 units", "answer_choice_1": "230 units", "answer_choice_2": "114 units", "answer_choice_3": "140 units", "answer_choice_4": "94 units", "answer_id": 2, "answer": "114 units", "question_type": [ "Numerical Reasoning", "Counterfactual", "Object Recognition", "Reading" ], "objects_of_interest": [ "mug1-82b61723316c5e6e", "bowl1-5e52db782672d148", "weighing scales-ebcd6fd652a7fb66", "bag of porridge-556cd1a7d40312e7", "bottle of milk-a94cd94cfa9d3c68", "glass-9a5506fc2fe802f8", "packet of raisins-2616586065c06836", "packet2-e23cd545de7d23c5", "plate1-41b35a6213bf3e64", "plate2-22ebe10a4fcd5b0a", "banana-45c13c9b28174df1", "knife-c2b01829bf0f18c8", "plastic bottle-70e1f0a130cb67a5", "cap of bottle-24e69164df2c9ae6", "mug2-12010544c2ea472f", "glass2-bbe189eccc500a67", "flask-61d40335c345a646", "flask cap-296f434296d5ffa2", "item2-8b0783856f44904d", "flask2-4aa62165160d928e", "spoon1-e1aa368ddeb095f6", "teapot-e6cad21e551cf5bb" ], "reasoning": "1. I looked for the time the user pours oatmeal from a bag (bag of porridge-556cd1a7d40312e7) into a bowl (bowl1-5e52db782672d148), finding it from 00:35-00:45. This bowl (bowl1-5e52db782672d148) is on top of a kitchen scale (weighing scales-ebcd6fd652a7fb66) that shows a certain unit of weight. \n2. When I look at the number on the kitchen scale (weighing scales-ebcd6fd652a7fb66) after the dry oatmeal oats (bag of porridge-556cd1a7d40312e7) are poured at 00:45, I see 57 units of weight.\n3. If I were to make a second serving of oatmeal for a friend, I would need to double the number of units of weight of dry oatmeal.\n4. I perform the operation 2*57=114.\n5. 114, therefore, is my answer.", "referenced_object_masks": [ "bag of porridge-556cd1a7d40312e7", "weighing scales-ebcd6fd652a7fb66", "bowl1-5e52db782672d148" ] }, { "key": "P05-20240425-082420:36b783b03c0ef73e3809f8682d33c0bcf7a3b406", "video_id": "P05-20240425-082420", "question": "If I were to combine the time it took me to both drink water and put a banana on my plate after putting down my plate, would the combined time be longer than the time it took me to linger while looking at the microwave the second time it is microwaving?", "answer_choice_0": "You don't drink water in this video", "answer_choice_1": "Yes, the combined time would be longer", "answer_choice_2": "You don't microwave a second time", "answer_choice_3": "No, the combined time would be shorter", "answer_choice_4": "The combined time would be equal", "answer_id": 3, "answer": "No, the combined time would be shorter", "question_type": [ "Temporal Reasoning", "Counterfactual", "Numerical Reasoning", "Egocentric & Spatial Perception", "Counting", "Action Recognition", "Event Occurence", "Situational Awareness", "Object Recognition" ], "objects_of_interest": [ "mug1-82b61723316c5e6e", "bowl1-5e52db782672d148", "weighing scales-ebcd6fd652a7fb66", "bag of porridge-556cd1a7d40312e7", "bottle of milk-a94cd94cfa9d3c68", "glass-9a5506fc2fe802f8", "packet of raisins-2616586065c06836", "packet2-e23cd545de7d23c5", "plate1-41b35a6213bf3e64", "plate2-22ebe10a4fcd5b0a", "banana-45c13c9b28174df1", "knife-c2b01829bf0f18c8", "plastic bottle-70e1f0a130cb67a5", "cap of bottle-24e69164df2c9ae6", "mug2-12010544c2ea472f", "glass2-bbe189eccc500a67", "flask-61d40335c345a646", "flask cap-296f434296d5ffa2", "item2-8b0783856f44904d", "flask2-4aa62165160d928e", "spoon1-e1aa368ddeb095f6", "teapot-e6cad21e551cf5bb" ], "reasoning": "1. I look to find the time the user drank water from a glass (glass-9a5506fc2fe802f8), and I find it from 01:39-01:43.\n2. I calculate the elapsed time as 4 seconds.\n3. I look for the time it took the user put a banana (banana-45c13c9b28174df1) on their plate (plate2-22ebe10a4fcd5b0a) after first putting their plate (plate2-22ebe10a4fcd5b0a) down, and I find it from 02:05-02:10.\n4. I calculate the elapsed time and find it to be 5 seconds.\n5. Finally, I look for the time I stood lingering in front of the microwave the second time it is microwaving.\n6. I notice the user using the microwave for the first time from 01:31-04:01. The second time the user microwaves is from 04:23-04:54.\n7. The time the user spends lingering in front of the microwave happens from 04:27-04:54.\n8. I calculated the elapsed time as 27 seconds.\n9. 9 seconds is shorter than 27 seconds.\n10. Therefore, the answer is that the combined time would be shorter.", "referenced_object_masks": [ "banana-45c13c9b28174df1", "plate2-22ebe10a4fcd5b0a", "glass-9a5506fc2fe802f8" ] }, { "key": "P05-20240425-082420:3a435b40a1eeefd8b52accf62d12982229cce7fb", "video_id": "P05-20240425-082420", "question": "If I poured the banana slices into the oatmeal bowl from the left side, how many more ingredients would I have poured into the bowl from the left side than the right?", "answer_choice_0": "4", "answer_choice_1": "3", "answer_choice_2": "2", "answer_choice_3": "1", "answer_choice_4": "5", "answer_id": 2, "answer": "2", "question_type": [ "Action Recognition", "Counting", "Counterfactual", "Egocentric & Spatial Perception", "Object Recognition" ], "objects_of_interest": [ "mug1-82b61723316c5e6e", "bowl1-5e52db782672d148", "weighing scales-ebcd6fd652a7fb66", "bag of porridge-556cd1a7d40312e7", "bottle of milk-a94cd94cfa9d3c68", "glass-9a5506fc2fe802f8", "packet of raisins-2616586065c06836", "packet2-e23cd545de7d23c5", "plate1-41b35a6213bf3e64", "plate2-22ebe10a4fcd5b0a", "banana-45c13c9b28174df1", "knife-c2b01829bf0f18c8", "plastic bottle-70e1f0a130cb67a5", "cap of bottle-24e69164df2c9ae6", "mug2-12010544c2ea472f", "glass2-bbe189eccc500a67", "flask-61d40335c345a646", "flask cap-296f434296d5ffa2", "item2-8b0783856f44904d", "flask2-4aa62165160d928e", "spoon1-e1aa368ddeb095f6", "teapot-e6cad21e551cf5bb" ], "reasoning": "1. I look for the bowl (bowl1-5e52db782672d148) where the user pours oatmeal (bag of porridge-556cd1a7d40312e7).\n2. I see the bowl (bowl1-5e52db782672d148) at 00:10, but it isn't till 00:35 that oatmeal is poured from the bag (bag of porridge-556cd1a7d40312e7) into it.\n3. I notice that the bag (bag of porridge-556cd1a7d40312e7) pours in from the left side of the bowl (bowl1-5e52db782672d148) .\n4. At 0:55, the user pours milk (bottle of milk-a94cd94cfa9d3c68) into the bowl (bowl1-5e52db782672d148) from the left side.\n5. At 01:16, the user pours water from a glass (glass-9a5506fc2fe802f8) into the bowl (bowl1-5e52db782672d148) from the right side.\n6. If the user were to pour his sliced bananas (banana-45c13c9b28174df1) into the bowl (bowl1-5e52db782672d148) used for oatmeal on the left side, they would have added 1 additional ingredient on the left side.\n7. Since they have already poured the milk (bottle of milk-a94cd94cfa9d3c68) and the oatmeal )bag of porridge-556cd1a7d40312e7) in from the left side, this would come to a total of 3 ingredients poured on the left side.\n8. Since the user poured 1 ingredient in from the right side, the water from the glass (glass-9a5506fc2fe802f8), the user would have poured 2 more ingredients from the left side, because 3-1=2.\n9. Therefore, the answer is 2.", "referenced_object_masks": [ "bottle of milk-a94cd94cfa9d3c68", "bowl1-5e52db782672d148", "bag of porridge-556cd1a7d40312e7", "banana-45c13c9b28174df1", "glass-9a5506fc2fe802f8" ] }, { "key": "P05-20240425-082420:67a2bcf8fd384a24821f4ab3d66f174d56880ddd", "video_id": "P05-20240425-082420", "question": "If the amount of banana slices I was able to cut per second remained constant (not counting the time I was slicing off the peel), how many slices would I be able to cut in a minute, rounded up?", "answer_choice_0": "62 slices", "answer_choice_1": "170 slices", "answer_choice_2": "147 slices", "answer_choice_3": "101 slices", "answer_choice_4": "125 slices", "answer_id": 2, "answer": "147 slices", "question_type": [ "Temporal Reasoning", "State Changes", "Action Recognition", "Numerical Reasoning", "Counting" ], "objects_of_interest": [ "mug1-82b61723316c5e6e", "bowl1-5e52db782672d148", "weighing scales-ebcd6fd652a7fb66", "bag of porridge-556cd1a7d40312e7", "bottle of milk-a94cd94cfa9d3c68", "glass-9a5506fc2fe802f8", "packet of raisins-2616586065c06836", "packet2-e23cd545de7d23c5", "plate1-41b35a6213bf3e64", "plate2-22ebe10a4fcd5b0a", "banana-45c13c9b28174df1", "knife-c2b01829bf0f18c8", "plastic bottle-70e1f0a130cb67a5", "cap of bottle-24e69164df2c9ae6", "mug2-12010544c2ea472f", "glass2-bbe189eccc500a67", "flask-61d40335c345a646", "flask cap-296f434296d5ffa2", "item2-8b0783856f44904d", "flask2-4aa62165160d928e", "spoon1-e1aa368ddeb095f6", "teapot-e6cad21e551cf5bb" ], "reasoning": "1. I look for the time the user cuts a banana (banana-45c13c9b28174df1) after it has been peeled, and I find the time from 02:46-02:55.\n2. I see that the user makes 22 slices into the banana (banana-45c13c9b28174df1) over this time frame.\n3. I realize that the above information means that the user is cutting 22 slices every 9 seconds, or about 2.4 cuts per second.\n4. Since there are 60 seconds in a minute, I multiply 2.4 cuts per second by 60 and get about 146.6 seconds.\n5. Therefore, the answer, rounded up, is 147 slices.", "referenced_object_masks": [ "banana-45c13c9b28174df1" ] }, { "key": "P05-20240425-082420:6db39ee4974332bb48ef9c97b6a4c8389039bc1b", "video_id": "P05-20240425-082420", "question": "How many utensils am I able to put into the utensil drawer by the time the microwave finishes beeping?", "answer_choice_0": "16 utensils", "answer_choice_1": "9 utensils", "answer_choice_2": "14 utensils", "answer_choice_3": "6 utensils", "answer_choice_4": "12 utensils", "answer_id": 4, "answer": "12 utensils", "question_type": [ "Action Recognition", "Counting", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "mug1-82b61723316c5e6e", "bowl1-5e52db782672d148", "weighing scales-ebcd6fd652a7fb66", "bag of porridge-556cd1a7d40312e7", "bottle of milk-a94cd94cfa9d3c68", "glass-9a5506fc2fe802f8", "packet of raisins-2616586065c06836", "packet2-e23cd545de7d23c5", "plate1-41b35a6213bf3e64", "plate2-22ebe10a4fcd5b0a", "banana-45c13c9b28174df1", "knife-c2b01829bf0f18c8", "plastic bottle-70e1f0a130cb67a5", "cap of bottle-24e69164df2c9ae6", "mug2-12010544c2ea472f", "glass2-bbe189eccc500a67", "flask-61d40335c345a646", "flask cap-296f434296d5ffa2", "item2-8b0783856f44904d", "flask2-4aa62165160d928e", "spoon1-e1aa368ddeb095f6", "teapot-e6cad21e551cf5bb" ], "reasoning": "1. I notice the user turns the microwave on the first time at 01:31, and I notice that the microwave stays on until 04:03, at which time it finishes beeping.\n2. Before 04:03, I slow the video down and watch to see how many utensils the user is able to put away in the utensil drawer.\n3. I see that the user starts putting utensils away from 03:42-04:06, and I count 2 knives from 03:46-03:47, a fork and 2 spoons from 03:50-03:52, a fork and 2 spoons from 03:55-03:56, and a fork and 3 spoons from 04:00-04:03.\n4. Counting all these up, the user is able to put 12 utensils into the utensil drawer by the time the microwave goes off the first time.\n5. Therefore, the answer is 12 utensils.", "referenced_object_masks": [] }, { "key": "P05-20240425-083515:36384eef93aadbf112ffe73a5b431b22b092637e", "video_id": "P05-20240425-083515", "question": "If I was standing in front of and facing away from the place I filled my glass with water, in what two directions would I need to move to be facing the refrigerator?", "answer_choice_0": "Right, then left", "answer_choice_1": "Straight, then right", "answer_choice_2": "Straight, then left", "answer_choice_3": "Right, then straight", "answer_choice_4": "Left, then right", "answer_id": 2, "answer": "Straight, then left", "question_type": [ "Situational Awareness", "Object Recognition", "Counterfactual", "Egocentric & Spatial Perception", "Action Recognition" ], "objects_of_interest": [ "spoon-52888e61a7f0874f", "bowl of porridge-30c1b78ce42b562d", "glass-c30e0edbe1b4c8aa", "plate-a549d2766be70216", "knife-ece6f68112e5fdad", "raisins bag-06547d7236b91449", "pecan nuts bag-7f2411527d5ce8f1", "scales-77047543d88f4228" ], "reasoning": "1. First, I located the refrigerator when the user walked into the kitchen at 00:02. \n2. Next, I looked for the user to fill their glass (glass-c30e0edbe1b4c8aa) with water, and saw them do this at the sink at 00:31. \n3. Reasoning about the scene and taking note of the fact that the sink is to the left and a bit in front of the refrigerator , I realized that if the user was standing in front of the sink and facing away from it, they would need to walk straight then turn left to be facing the refrigerator .", "referenced_object_masks": [ "glass-c30e0edbe1b4c8aa" ] }, { "key": "P05-20240425-083515:81691fdc40f632d01eb0f48b6e2fb3e3d9ceb73e", "video_id": "P05-20240425-083515", "question": "The second time I stir the porridge with a spoon, how many times do I stir in a counter clockwise direction before stirring clockwise?", "answer_choice_0": "Ten", "answer_choice_1": "Four", "answer_choice_2": "Two", "answer_choice_3": "Six", "answer_choice_4": "Eight", "answer_id": 4, "answer": "Eight", "question_type": [ "Counting", "Action Recognition", "Egocentric & Spatial Perception", "Object Recognition" ], "objects_of_interest": [ "spoon-52888e61a7f0874f", "bowl of porridge-30c1b78ce42b562d", "glass-c30e0edbe1b4c8aa", "plate-a549d2766be70216", "knife-ece6f68112e5fdad", "raisins bag-06547d7236b91449", "pecan nuts bag-7f2411527d5ce8f1", "scales-77047543d88f4228" ], "reasoning": "1. First, I looked for the user to stir the porridge (bowl of porridge-30c1b78ce42b562d) with a spoon (spoon-52888e61a7f0874f) for the first time, and found that this begins at 00:07. \n2. Next, I looked for the user to stir the porridge (bowl of porridge-30c1b78ce42b562d) with a spoon (spoon-52888e61a7f0874f) a second time and found this moment at 01:28-01:36\n3. Then, I counted the number of times the user stirs the porridge (bowl of porridge-30c1b78ce42b562d) in a counter-clockwise direction and counted 8 instances before the user stirs in a clockwise direction at 01:33.", "referenced_object_masks": [ "bowl of porridge-30c1b78ce42b562d", "spoon-52888e61a7f0874f" ] }, { "key": "P05-20240425-083515:a908be1b61d56b98d083f2f6a4558f4d6ce1cb35", "video_id": "P05-20240425-083515", "question": "When I place the second fruit into the bowl of porridge, where is the drawer in which I place the scale in relation to me?", "answer_choice_0": "10 o'clock", "answer_choice_1": "2 o'clock", "answer_choice_2": "8 o'clock", "answer_choice_3": "4 o'clock", "answer_choice_4": "6 o'clock", "answer_id": 3, "answer": "4 o'clock", "question_type": [ "Object Recognition", "Action Recognition", "Counting", "Egocentric & Spatial Perception", "Situational Awareness" ], "objects_of_interest": [ "spoon-52888e61a7f0874f", "bowl of porridge-30c1b78ce42b562d", "glass-c30e0edbe1b4c8aa", "plate-a549d2766be70216", "knife-ece6f68112e5fdad", "raisins bag-06547d7236b91449", "pecan nuts bag-7f2411527d5ce8f1", "scales-77047543d88f4228" ], "reasoning": "1. I looked for the user to place the first fruit into the bowl of porridge (bowl of porridge-30c1b78ce42b562d) and saw them pour bananas into the bowl (bowl of porridge-30c1b78ce42b562d) at 02:09. \n2. Next, I looked for the user to place a second fruit into the bowl of porridge (bowl of porridge-30c1b78ce42b562d) and saw them pour raisins into the bowl (bowl of porridge-30c1b78ce42b562d) at 02:23. \n3. I noticed that the user is in the middle of the kitchen facing the kitchen island at this moment. \n4. Then, I looked for the user to put the scale (scales-77047543d88f4228) in a drawer and saw them do this at 03:00-03:02. \n5. I reasoned about the scene and realized that when the user adds the second fruit to the bowl of porridge (bowl of porridge-30c1b78ce42b562d), the drawer in which they place the scale (scales-77047543d88f4228) is at their relative 4 o'clock.", "referenced_object_masks": [ "scales-77047543d88f4228", "bowl of porridge-30c1b78ce42b562d" ] }, { "key": "P05-20240425-083515:e18d1967ba0f7add765a6bfdcbfc4e6a959e52aa", "video_id": "P05-20240425-083515", "question": "What is the second thing I place the bowl of porridge on after I weigh it for the final time?", "answer_choice_0": "The counter", "answer_choice_1": "The microwave", "answer_choice_2": "A plate", "answer_choice_3": "A tray", "answer_choice_4": "The sink", "answer_id": 2, "answer": "A plate", "question_type": [ "Object Recognition", "Action Recognition", "Counting" ], "objects_of_interest": [ "spoon-52888e61a7f0874f", "bowl of porridge-30c1b78ce42b562d", "glass-c30e0edbe1b4c8aa", "plate-a549d2766be70216", "knife-ece6f68112e5fdad", "raisins bag-06547d7236b91449", "pecan nuts bag-7f2411527d5ce8f1", "scales-77047543d88f4228" ], "reasoning": "1. First, I looked for the user to weigh the bowl of porridge (bowl of porridge-30c1b78ce42b562d). Although they weigh the bowl of porridge (bowl of porridge-30c1b78ce42b562d) several times during the video, the final time occurs at 02:47. \n2. Next, I looked for the user to place the porridge (bowl of porridge-30c1b78ce42b562d) somewhere, and saw them place the bowl (bowl of porridge-30c1b78ce42b562d) on the counter at 02:56. \n3. Finally, I looked for the user to move the porridge (bowl of porridge-30c1b78ce42b562d) to a second spot, and saw them place the porridge (bowl of porridge-30c1b78ce42b562d) on a plate (plate-a549d2766be70216) at 03:14.", "referenced_object_masks": [ "bowl of porridge-30c1b78ce42b562d", "plate-a549d2766be70216" ] }, { "key": "P05-20240425-083515:fe8906ebaec42d78c91f71ee72c6ea108041314f", "video_id": "P05-20240425-083515", "question": "If I subtract the weight of the lightest topping from the weight of the heaviest topping I put in the porridge, what would the total be?", "answer_choice_0": "45", "answer_choice_1": "95", "answer_choice_2": "31", "answer_choice_3": "126", "answer_choice_4": "64", "answer_id": 4, "answer": "64", "question_type": [ "Object Recognition", "Action Recognition", "Reading", "Numerical Reasoning" ], "objects_of_interest": [ "spoon-52888e61a7f0874f", "bowl of porridge-30c1b78ce42b562d", "glass-c30e0edbe1b4c8aa", "plate-a549d2766be70216", "knife-ece6f68112e5fdad", "raisins bag-06547d7236b91449", "pecan nuts bag-7f2411527d5ce8f1", "scales-77047543d88f4228" ], "reasoning": "1. I looked for the user to start weighing toppings and found the first time they do this is when they add bananas to the porridge (bowl of porridge-30c1b78ce42b562d) at 02:10-02:13. At 02:13, the number 95 is clearly visible on the scale (scales-77047543d88f4228). I counted this as the weight of the bananas . \n2. Next, I looked for the user to add a second topping to the porridge (bowl of porridge-30c1b78ce42b562d). They clear the weight on the scale (scales-77047543d88f4228) at 02:20, then add raisins to the porridge (bowl of porridge-30c1b78ce42b562d) at 02:22-02:27. At 02:27, the number 45 is clearly visible on the scale (scales-77047543d88f4228). I counted this as the weight of the raisins . \n3. Then, I looked for the user to add more toppings and found that after clearing the weight on the scale (scales-77047543d88f4228) at 02:38, they add pecans to the porridge (bowl of porridge-30c1b78ce42b562d) at 02:41-02:47. At 02:47, the number 31 is clearly visible on the scale (scales-77047543d88f4228). I counted this as the weight of the pecans . \n4. I watched the rest of the video and determined that the user never adds more toppings to the porridge (bowl of porridge-30c1b78ce42b562d). \n5. Realizing that the bananas are the heaviest topping with a weight of 95 and that the pecans are the lightest topping with a weight of 31, I subtracted 31 from 95 and got a total of 64.", "referenced_object_masks": [ "scales-77047543d88f4228", "bowl of porridge-30c1b78ce42b562d" ] }, { "key": "P05-20240425-124420:0b28d1000ec92e91e3eefb2f22b958cc9485e3f3", "video_id": "P05-20240425-124420", "question": "What is the third item I grab with my right hand?", "answer_choice_0": "The pan", "answer_choice_1": "The bowl of rice", "answer_choice_2": "The refrigerator door", "answer_choice_3": "The white plate", "answer_choice_4": "The spoon", "answer_id": 0, "answer": "The pan", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness", "Temporal Reasoning", "Action Recognition" ], "objects_of_interest": [ "plate-d3473bb56c2ee871", "pan-85f42feb378a8541", "bowl of rice-8b757c59ffdd6ade", "spoon-ba6b5b9e7b73b0b8", "pan lid-d52a2e342857ae4b", "empty plastic bottle-ef4437422ca8954d", "lid of bottle-bad1958abcfef6ed", "tupperware-9ea1f8626ee5cc69", "tupperware lid-b177292393a4dd39", "white water bottle-85ea7f7178dd22cb", "lid of white water bottle-a71b5f96fbb78a1d", "pint glass-72884ff137f00d1e" ], "reasoning": "1. I watched the video and noted that the first item the user touches with the right hand is the white plate (plate-d3473bb56c2ee871) at 00:10.\n2. I continued to watch the user and observed 00:14-00:15 when the user grabs the refrigerator door with the right hand, marking the second item grabbed with the right hand.\n3. Then, I watched the user reach into the fridge and grab the pan (pan-85f42feb378a8541) with the right hand at 00:18.\n4. This means the pan (pan-85f42feb378a8541) is the third item grabbed with the right hand.", "referenced_object_masks": [ "plate-d3473bb56c2ee871", "pan-85f42feb378a8541" ] }, { "key": "P05-20240425-124420:22b1a90323f4c391df311f08ceecd6c51d0b3f17", "video_id": "P05-20240425-124420", "question": "Which pair of actions are performed with the same hand?", "answer_choice_0": "Scooping the food onto the rice and opening the microwave door", "answer_choice_1": "Grabbing the plastic bottle and grabbing the bottle lid", "answer_choice_2": "Putting the lid back on the pan and closing the microwave door", "answer_choice_3": "Putting the rice back in the fridge and opening the refrigerator door at 00:45", "answer_choice_4": "Carrying the plate to the microwave and putting the plate into the microwave", "answer_id": 0, "answer": "Scooping the food onto the rice and opening the microwave door", "question_type": [ "Action Recognition", "Event Occurence", "State Changes", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "plate-d3473bb56c2ee871", "pan-85f42feb378a8541", "bowl of rice-8b757c59ffdd6ade", "spoon-ba6b5b9e7b73b0b8", "pan lid-d52a2e342857ae4b", "empty plastic bottle-ef4437422ca8954d", "lid of bottle-bad1958abcfef6ed", "tupperware-9ea1f8626ee5cc69", "tupperware lid-b177292393a4dd39", "white water bottle-85ea7f7178dd22cb", "lid of white water bottle-a71b5f96fbb78a1d", "pint glass-72884ff137f00d1e" ], "reasoning": "1. I watched the video to identify the event options and which hand was used to perform each.\n2. First, I observed the user opening the refrigerator at 00:45 with the right hand before putting the bowl of rice (bowl of rice-8b757c59ffdd6ade) in the fridge with the left hand from 00:46-00:47, so option A does not work.\n3. Continuing, I saw the user scoop food onto the plate (plate-d3473bb56c2ee871) with the right hand from 00:53 to 01:12\n4. Then, I watched from 01:15-01:20, when the user carries the plate (plate-d3473bb56c2ee871) to the microwave with the left hand.\nI next noticed that the user opened the microwave with the right hand from 01:20-01:21, indicating that option C is correct because the right hand was used to both scoop the food and open the microwave.\n5. I then watched and noticed that the other three options are not performed with the same hand.", "referenced_object_masks": [ "plate-d3473bb56c2ee871", "bowl of rice-8b757c59ffdd6ade" ] }, { "key": "P05-20240425-124420:251cd08b08584b19c7bea7bddfce2b42cb27fc1d", "video_id": "P05-20240425-124420", "question": "What is to the right of me when I set the spoon down for the third time?", "answer_choice_0": "A doorway", "answer_choice_1": "The microwave", "answer_choice_2": "The chair", "answer_choice_3": "The refrigerator", "answer_choice_4": "The cabinet", "answer_id": 4, "answer": "The cabinet", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "plate-d3473bb56c2ee871", "pan-85f42feb378a8541", "bowl of rice-8b757c59ffdd6ade", "spoon-ba6b5b9e7b73b0b8", "pan lid-d52a2e342857ae4b", "empty plastic bottle-ef4437422ca8954d", "lid of bottle-bad1958abcfef6ed", "tupperware-9ea1f8626ee5cc69", "tupperware lid-b177292393a4dd39", "white water bottle-85ea7f7178dd22cb", "lid of white water bottle-a71b5f96fbb78a1d", "pint glass-72884ff137f00d1e" ], "reasoning": "1. I watched the video and from 00:02-00:03, observed the layout of the kitchen, with the counter to the right, the microwave ahead of that, and cabinets across from the counter.\n2. I continued to watch and noted 01:44, in which a clear view of the kitchen layout is provided, corroborating the previous layout description.\n3. I watched the video from the beginning and saw the user take the spoon (spoon-ba6b5b9e7b73b0b8) from the drawer from 00:28-00:32. At 00:45, the user places the spoon (spoon-ba6b5b9e7b73b0b8) down for the first time on the plate (plate-d3473bb56c2ee871).\n4. From 00:35-01:17, I watch the user spoon out the rice from the bowl (bowl of rice-8b757c59ffdd6ade) and the leftovers from the pan (pan-85f42feb378a8541) onto the plate (plate-d3473bb56c2ee871) with the spoon (spoon-ba6b5b9e7b73b0b8). At 01:18, I watch the user place the spoon (spoon-ba6b5b9e7b73b0b8) down for the second time on the counter.\n5. Next, I saw the user pick the spoon (spoon-ba6b5b9e7b73b0b8) up off the counter at 02:47. The user uses the spoon (spoon-ba6b5b9e7b73b0b8) to stir and taste their food , and then at 02:12, sets the spoon (spoon-ba6b5b9e7b73b0b8) down for the third time at 03:12, at which time they are walking with the counter on their left, meaning that the cabinet is to the user's right.", "referenced_object_masks": [ "plate-d3473bb56c2ee871", "spoon-ba6b5b9e7b73b0b8", "bowl of rice-8b757c59ffdd6ade", "pan-85f42feb378a8541" ] }, { "key": "P05-20240425-124420:954b9ab384b29c67da5d6cbd651052fcd6b8dbd9", "video_id": "P05-20240425-124420", "question": "What is the difference between the number of mugs sitting on the shelf and the number of mugs hanging on the hooks below the shelf?", "answer_choice_0": "3", "answer_choice_1": "5", "answer_choice_2": "2", "answer_choice_3": "0", "answer_choice_4": "1", "answer_id": 3, "answer": "0", "question_type": [ "Object Recognition", "Numerical Reasoning", "Counting" ], "objects_of_interest": [ "plate-d3473bb56c2ee871", "pan-85f42feb378a8541", "bowl of rice-8b757c59ffdd6ade", "spoon-ba6b5b9e7b73b0b8", "pan lid-d52a2e342857ae4b", "empty plastic bottle-ef4437422ca8954d", "lid of bottle-bad1958abcfef6ed", "tupperware-9ea1f8626ee5cc69", "tupperware lid-b177292393a4dd39", "white water bottle-85ea7f7178dd22cb", "lid of white water bottle-a71b5f96fbb78a1d", "pint glass-72884ff137f00d1e" ], "reasoning": "1. I watched the video to find when the coffee mugs on the shelves and hooks were showing, identifying this to occur first from 01:54-01:57.\n2. At this time, I looked and counted 9 mugs sitting on the shelf (I slowed the video to 0.5x and at 01:55 got a good view to count the mugs)\n3. I then watched the user turn back to the cabinet before looking back towards the shelf and hooks from 02:00-02:02:05.\n4. During this time, specifically at 02:03, I got a clear view of the hooks and was able to count that there were 9 mugs on the hooks.\n5. To finish, I did the math, which is 9 - 9 = 0.", "referenced_object_masks": [] }, { "key": "P05-20240425-124420:96db429d0b723d9601903476f16af12cb6490e5b", "video_id": "P05-20240425-124420", "question": "Which of these items is the same color as the heart-shaped magnets on the left side of the refrigerator?", "answer_choice_0": "The rice bowl", "answer_choice_1": "The water bottle", "answer_choice_2": "The spoon", "answer_choice_3": "The body of the island", "answer_choice_4": "The microwave", "answer_id": 3, "answer": "The body of the island", "question_type": [ "Object Recognition", "Egocentric & Spatial Perception", "State Changes", "Situational Awareness" ], "objects_of_interest": [ "plate-d3473bb56c2ee871", "pan-85f42feb378a8541", "bowl of rice-8b757c59ffdd6ade", "spoon-ba6b5b9e7b73b0b8", "pan lid-d52a2e342857ae4b", "empty plastic bottle-ef4437422ca8954d", "lid of bottle-bad1958abcfef6ed", "tupperware-9ea1f8626ee5cc69", "tupperware lid-b177292393a4dd39", "white water bottle-85ea7f7178dd22cb", "lid of white water bottle-a71b5f96fbb78a1d", "pint glass-72884ff137f00d1e" ], "reasoning": "1. I watched the video to first find when the magnets are shown on the fridge, which I noted occurs at 03:08.\n2. Here, I observed that there are two heart-shaped magnets , both of which are blue.\n3. I then started at the beginning of the video to identify the color of each item option.\n4. First, at 00:03, I observed that the body of the island is blue.\n5. Next, at 00:08, I saw that the microwave is white, so it doesn't match.\n6. Then, I saw at 00:28 that the bowl of rice (bowl of rice-8b757c59ffdd6ade) is silver, which also does not match. \n7. Continuing, I observed at 00:30 that the spoon (spoon-ba6b5b9e7b73b0b8) is also silver.\n8. Lastly, at 02:15, I observed that the water bottle (white water bottle-85ea7f7178dd22cb) is white.\n9. Therefore, the body of the island is the item that is the same color as the magnets.", "referenced_object_masks": [ "white water bottle-85ea7f7178dd22cb", "bowl of rice-8b757c59ffdd6ade", "spoon-ba6b5b9e7b73b0b8" ] }, { "key": "P05-20240425-125459:59bde134c605fe1db4d09133943a7d60b5d2b1ac", "video_id": "P05-20240425-125459", "question": "What is the fifth total action that I perform with my right hand during the video?", "answer_choice_0": "Opening the dishwasher door", "answer_choice_1": "Placing the fork in the dishwasher", "answer_choice_2": "Opening the kitchen door", "answer_choice_3": "Closing the kitchen door", "answer_choice_4": "Opening the top dishwasher drawer", "answer_id": 1, "answer": "Placing the fork in the dishwasher", "question_type": [ "Action Recognition", "Counting", "Situational Awareness" ], "objects_of_interest": [ "plate-32a4614fb1f021b4", "fork-77ace36e394c8169", "spoon-b6ed17cf336f2f4d", "bowl-c90a6129492f82ce" ], "reasoning": "1. I watch the video to identify actions performed by the user with their right hand, and count them: the first of these actions is the pushing open of the kitchen door at 00:01.\n2. The second action I observe the user performing with their right hand is the closing of the kitchen door behind them from 00:03-00:05.\n3. The third action I observe the user performing with their right hand is the opening of the dishwasher drawer at 00:07.\n4. The fourth action I observe the user performing with their right hand is the pulling out of the top dishwasher drawer from 00:08-00:09.\n5. The fifth action I observe the user performing with their right hand is picking up the fork (fork-77ace36e394c8169) at 00:10 and placing it in the dishwasher, making C the correct answer.", "referenced_object_masks": [ "fork-77ace36e394c8169" ] }, { "key": "P05-20240425-125459:5b0b958c52da2f8ded6f424dbfd5b9ee5626f2f6", "video_id": "P05-20240425-125459", "question": "If the items placed in the dishwasher were placed in reverse chronological order, with the first and third items in the original order flipped, which item would be the last placed into the dishwasher?", "answer_choice_0": "Plate", "answer_choice_1": "Mug", "answer_choice_2": "Fork", "answer_choice_3": "Spoon", "answer_choice_4": "Bowl", "answer_id": 4, "answer": "Bowl", "question_type": [ "Temporal Reasoning", "Action Recognition", "Counterfactual" ], "objects_of_interest": [ "plate-32a4614fb1f021b4", "fork-77ace36e394c8169", "spoon-b6ed17cf336f2f4d", "bowl-c90a6129492f82ce" ], "reasoning": "1. I first note the order in which the original items are placed into the dishwasher : the first of these is the fork (fork-77ace36e394c8169) placed in at 00:10.\n2. I then note that the second item placed into the dishwasher by the user is the spoon (spoon-b6ed17cf336f2f4d) placed at 00:11.\n3. I then note that the third item placed by the user is the bowl (bowl-c90a6129492f82ce) placed at 00:16.\n4. I note that the fourth and final item placed in the dishwasher by the user is the plate (plate-32a4614fb1f021b4) at 00:22.\n5. Next, I imagine counterfactually that the first and third items are switched: this means that the fork (fork-77ace36e394c8169) is now considered third in the order of items, and the bowl (bowl-c90a6129492f82ce) is considered first.\n6. Next, I counterfactually assume that this new order is reversed chronologically in the order that the user placed the items in the dishwasher - this would mean that the plate (plate-32a4614fb1f021b4) is placed first, the fork (fork-77ace36e394c8169) is placed second, the spoon (spoon-b6ed17cf336f2f4d) is placed third, and the bowl (bowl-c90a6129492f82ce) is placed fourth.\n7. I identify, based on these counterfactuals, that the bowl (bowl-c90a6129492f82ce) is the fourth and final item placed, making the answer B.", "referenced_object_masks": [ "fork-77ace36e394c8169", "spoon-b6ed17cf336f2f4d", "bowl-c90a6129492f82ce", "plate-32a4614fb1f021b4" ] }, { "key": "P05-20240425-125459:686f063b2c55421ac389e92e8f9b296f010a508e", "video_id": "P05-20240425-125459", "question": "At what point do I look to the right in the video, relative to when I open the dishwasher shelves?", "answer_choice_0": "After opening the bottom shelf and the middle shelves, but before opening the top shelf", "answer_choice_1": "After opening the middle shelf, but before opening the top or bottom shelves", "answer_choice_2": "After opening the bottom shelf, but before opening the top or middle shelves", "answer_choice_3": "After opening the top and middle shelves, but before opening the top shelf", "answer_choice_4": "After opening the top shelf, but before opening the middle and bottom shelves", "answer_id": 4, "answer": "After opening the top shelf, but before opening the middle and bottom shelves", "question_type": [ "Event Occurence", "Action Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "plate-32a4614fb1f021b4", "fork-77ace36e394c8169", "spoon-b6ed17cf336f2f4d", "bowl-c90a6129492f82ce" ], "reasoning": "1. I first watch the video to identify the order and timestamps in which the user opens the various drawers of the dishwasher : I observe the first shelf being opened, which is the top shelf at 00:09.\n2. I observe the next shelf being opened by the user, which is the middle shelf at 00:14.\n3. I observe the user open the final shelf, which is the bottom shelf at 00:20.\n4. Next, I watch the video for when the user looks to the right in the video: this rightward glance occurs from 00:12-00:13, after the user opens the top shelf , but before they open the middle shelf or bottom shelf . This makes E the correct answer.", "referenced_object_masks": [] }, { "key": "P05-20240425-125459:7c747b47b3280af7d0f1775272f3e3a43b078ddd", "video_id": "P05-20240425-125459", "question": "Where is the item, already in the dishwasher, that matches the inside color of the third item that I place in the dishwasher?", "answer_choice_0": "The front left bottom shelf", "answer_choice_1": "The front left middle shelf", "answer_choice_2": "The back right bottom shelf", "answer_choice_3": "The back right middle shelf", "answer_choice_4": "The back right top shelf", "answer_id": 1, "answer": "The front left middle shelf", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "plate-32a4614fb1f021b4", "fork-77ace36e394c8169", "spoon-b6ed17cf336f2f4d", "bowl-c90a6129492f82ce" ], "reasoning": "1. I first note the order in which the original items are placed into the dishwasher : the first of these is the fork (fork-77ace36e394c8169) placed in at 00:10.\n2. I then note that the second item placed into the dishwasher by the user is the spoon (spoon-b6ed17cf336f2f4d) placed at 00:11.\n3. I then note that the third item placed by the user is the bowl (bowl-c90a6129492f82ce) placed at 00:16.\n4. Next, I identify the inner color of the third item placed: the concave side of the bowl (bowl-c90a6129492f82ce) is a bright, patterned pink.\n5. I look in the dishwasher for more bright, patterned pink color, and identify it on the dishwasher's middle shelf on the outside of a bowl , which can be seen in the front left corner of the middle shelf, making A the correct answer.", "referenced_object_masks": [ "spoon-b6ed17cf336f2f4d", "bowl-c90a6129492f82ce", "fork-77ace36e394c8169" ] }, { "key": "P05-20240425-125459:b2de5b9c164eb8f3c6cb953a7e07348360477717", "video_id": "P05-20240425-125459", "question": "If I exchanged where I placed the second and fourth items in the dishwasher, how many accompanying dishes would rest on the same shelf as the second item in its new place?", "answer_choice_0": "3", "answer_choice_1": "4", "answer_choice_2": "6", "answer_choice_3": "5", "answer_choice_4": "7", "answer_id": 3, "answer": "5", "question_type": [ "Action Recognition", "Temporal Reasoning", "Egocentric & Spatial Perception", "Counterfactual" ], "objects_of_interest": [ "plate-32a4614fb1f021b4", "fork-77ace36e394c8169", "spoon-b6ed17cf336f2f4d", "bowl-c90a6129492f82ce" ], "reasoning": "1. I first note the order in which the original items are placed into the dishwasher , and where they are placed: the first of these is the fork (fork-77ace36e394c8169) placed in at 00:10, on the top dishwasher shelf .\n2. I then note that the second item placed into the dishwasher by the user is the spoon (spoon-b6ed17cf336f2f4d) placed at 00:11, on the top dishwasher shelf .\n3. I then note that the third item placed by the user is the bowl (bowl-c90a6129492f82ce) placed at 00:16, on the middle dishwasher shelf .\n4. I note that the fourth and final item placed in the dishwasher by the user is the plate (plate-32a4614fb1f021b4) at 00:22, on the bottom dishwasher shelf .\n5. Next, I imagine counterfactually that the second item and fourth item are switched in the shelves they were placed on: this would mean that the spoon (spoon-b6ed17cf336f2f4d) was placed on the bottom shelf , and the plate (plate-32a4614fb1f021b4) was placed on the top shelf .\n6. Next, imagining the spoon (spoon-b6ed17cf336f2f4d) is now on the bottom shelf , I count the number of dishes that accompany it on that shelf. On the bottom shelf , I observe 2 large plates , 1 small plate , 1 large metal pot and 1 cutting board , for a total of 5 accompanying items on the bottom shelf.", "referenced_object_masks": [ "fork-77ace36e394c8169", "spoon-b6ed17cf336f2f4d", "bowl-c90a6129492f82ce", "plate-32a4614fb1f021b4" ] }, { "key": "P05-20240425-171455:25cf332bd416d7b7cfc5ec9e63338092cbe94719", "video_id": "P05-20240425-171455", "question": "How many times do I cross the room, from the counter by the stove top to the counter with the sink, after I adjust the pet bed, before I pick up the bowl I use to deposit egg whites?", "answer_choice_0": "0 times", "answer_choice_1": "5 times", "answer_choice_2": "9 times", "answer_choice_3": "6 times", "answer_choice_4": "3 times", "answer_id": 2, "answer": "9 times", "question_type": [ "Egocentric & Spatial Perception", "Temporal Reasoning", "Object Recognition", "Action Recognition", "Counting" ], "objects_of_interest": [ "tub of peanut butter-547d97278d20f202", "ramekin-2c72710322a992ad", "dirty knife-cd75b945f5ded97f", "dirty spoon-7852cb9b136deb10", "dirty spoon2-dd929219536661dc", "dirty bowl-9e6ea28a14f9b649", "plate-6938b4e909b5f09f", "pan-079265a4e1b348b2", "dirty plate-c3219b471b4dda5a", "dirty bowl2-762645d994b0cd69", "dirty plate2-5c4673b4d76f9442", "toast rack-64ea49f9c49520cc", "splatter guard-9b657c27ec8f206d", "bag of potatoes-8c34fa6ece799d60", "knife-d5a90e358104cf8e", "rich tea biscuits-63873256a79ca593", "wooden board-b1fb86ec1af9f832", "knife2-2467ef7e5f81c335", "pastry knife-aeeba31b852c7759", "lid-54e1957ed0240c00", "dirty spoon3-34337b9883d46881", "cheese grater-0a3e35504dfa2516", "kettle-03640175a5d194e0", "packet of bacon-f6d2c80fdca32296", "scales-98351d719a7cdf76", "skewer-2abb23c1b9e2019a", "knife3-5231b12bb0800120", "chopping board-c4d82758bae0192b", "clove of garlic-7fbfcb585485667e", "frying pan-afeb3171f9d73b44", "piece of bacon-06e2bc8b7c7bf6c9", "bottle of oil-3d46f688ba1d4199", "laundry piece7-4753e90504806056", "laundry piece5-792e09156bd8e44e", "laundry piece2-8453c379098ce721", "laundry piece3-9aceee288f771adc", "wooden spoon-90ce22c5bf54dd37", "metal bowl-4d94a6a18235cb96", "packet of eggs-cc647d4f21492fca", "egg1-b4d86558aacef4ae", "egg2-49b3c1df5fbf5e04", "bowl-4e3748333958fdab", "egg3-10d0494e14414d2b", "egg4-95cdfe5e7f75973d", "egg5-27bb2d87c2b69223", "eggshells-ec30a182d2ce3de2", "pot-a05582f9ce801bc1", "salt container-8080ccc21342a025", "pot lid-62c43e7df95c3a55", "cheese package-7efa514770bf8626", "cheese block-944d1022a942e9e0", "cheese grater2-74015c87812a6675", "washing up brush-2e8d9079c8abb0aa", "rind-0113d936a69507ff", "packet of pasta-9df8300cac513437", "phone-c847575caf5d82d2", "fine cheese grater-4c58ed0f9b012cf3", "pot of pepper-ae7e26371bb94891", "fork-d265c3d8289354aa", "mushrooms-5616728ad4e82788", "lettuce-7af8afa4306535dc", "metal bowl2-df50795bfd325957" ], "reasoning": "1. I look for the time the user adjusts the blanket on the pet bed . I find it in the very beginning of the video from 00:13-00:20.\n2. At this point the user is at the counter by the stove, and so I count the first time the user crosses to the counter with the sink 00:22-00:25.\n3. After this I count the user crossing the room 8 more times: from 01:37-01:40, 02:13-02:16, 03:24-03:28, 06:46-06:49, 07:04-07:07, 08:01-08:04, 08:08-08:11, and 08:35-08:37.\n4. From 08:39-08:40, the user grabs a bowl (bowl-4e3748333958fdab). From 08:49-09:03, the user separates the whites of an egg (egg4-95cdfe5e7f75973d) over this bowl (bowl-4e3748333958fdab).\n5. Since I have found the bowl (bowl-4e3748333958fdab) in which the user deposits egg whites , I stop counting, and come to the answer of 9.", "referenced_object_masks": [ "bowl-4e3748333958fdab", "egg4-95cdfe5e7f75973d" ] }, { "key": "P05-20240425-171455:469c00d91d966540ca9c460f5b60a4f30fc3f859", "video_id": "P05-20240425-171455", "question": "After putting the pot in the dishwasher, what is the second object I touch with my right hand?", "answer_choice_0": "Sink", "answer_choice_1": "Spray arm", "answer_choice_2": "Dirty spoon", "answer_choice_3": "Dirty plate", "answer_choice_4": "Pan", "answer_id": 3, "answer": "Dirty plate", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counting", "Numerical Reasoning" ], "objects_of_interest": [ "tub of peanut butter-547d97278d20f202", "ramekin-2c72710322a992ad", "dirty knife-cd75b945f5ded97f", "dirty spoon-7852cb9b136deb10", "dirty spoon2-dd929219536661dc", "dirty bowl-9e6ea28a14f9b649", "plate-6938b4e909b5f09f", "pan-079265a4e1b348b2", "dirty plate-c3219b471b4dda5a", "dirty bowl2-762645d994b0cd69", "dirty plate2-5c4673b4d76f9442", "toast rack-64ea49f9c49520cc", "splatter guard-9b657c27ec8f206d", "bag of potatoes-8c34fa6ece799d60", "knife-d5a90e358104cf8e", "rich tea biscuits-63873256a79ca593", "wooden board-b1fb86ec1af9f832", "knife2-2467ef7e5f81c335", "pastry knife-aeeba31b852c7759", "lid-54e1957ed0240c00", "dirty spoon3-34337b9883d46881", "cheese grater-0a3e35504dfa2516", "kettle-03640175a5d194e0", "packet of bacon-f6d2c80fdca32296", "scales-98351d719a7cdf76", "skewer-2abb23c1b9e2019a", "knife3-5231b12bb0800120", "chopping board-c4d82758bae0192b", "clove of garlic-7fbfcb585485667e", "frying pan-afeb3171f9d73b44", "piece of bacon-06e2bc8b7c7bf6c9", "bottle of oil-3d46f688ba1d4199", "laundry piece7-4753e90504806056", "laundry piece5-792e09156bd8e44e", "laundry piece2-8453c379098ce721", "laundry piece3-9aceee288f771adc", "wooden spoon-90ce22c5bf54dd37", "metal bowl-4d94a6a18235cb96", "packet of eggs-cc647d4f21492fca", "egg1-b4d86558aacef4ae", "egg2-49b3c1df5fbf5e04", "bowl-4e3748333958fdab", "egg3-10d0494e14414d2b", "egg4-95cdfe5e7f75973d", "egg5-27bb2d87c2b69223", "eggshells-ec30a182d2ce3de2", "pot-a05582f9ce801bc1", "salt container-8080ccc21342a025", "pot lid-62c43e7df95c3a55", "cheese package-7efa514770bf8626", "cheese block-944d1022a942e9e0", "cheese grater2-74015c87812a6675", "washing up brush-2e8d9079c8abb0aa", "rind-0113d936a69507ff", "packet of pasta-9df8300cac513437", "phone-c847575caf5d82d2", "fine cheese grater-4c58ed0f9b012cf3", "pot of pepper-ae7e26371bb94891", "fork-d265c3d8289354aa", "mushrooms-5616728ad4e82788", "lettuce-7af8afa4306535dc", "metal bowl2-df50795bfd325957" ], "reasoning": "1. I located the dishwasher at 00:23 to the left of the user as they approach the kitchen.\n2. I located the pot (pan-079265a4e1b348b2) at 00:24 in the sink in front of the user.\n2. From 01:15 to 01:20, I observed the user placing the pot (pan-079265a4e1b348b2) into the dishwasher . \n3. After 01:20, I observed the user touch the spray arm at 01:22 and the plate (dirty plate-c3219b471b4dda5a) at 01:25 with their right hand.\n4. I determined that the plate (dirty plate-c3219b471b4dda5a) is the second object the user touches with their right hand after putting the pot (pan-079265a4e1b348b2) in the dishwasher .", "referenced_object_masks": [ "pan-079265a4e1b348b2", "dirty plate-c3219b471b4dda5a" ] }, { "key": "P05-20240425-171455:5a073d2fbcf9b4646b7b5e7534467cabda08e062", "video_id": "P05-20240425-171455", "question": "What object do I touch with my right hand before grabbing the dog bed with both hands?", "answer_choice_0": "Door", "answer_choice_1": "Counter", "answer_choice_2": "Peanut butter", "answer_choice_3": "Blue cabinet", "answer_choice_4": "Stuffed animal", "answer_id": 3, "answer": "Blue cabinet", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "tub of peanut butter-547d97278d20f202", "ramekin-2c72710322a992ad", "dirty knife-cd75b945f5ded97f", "dirty spoon-7852cb9b136deb10", "dirty spoon2-dd929219536661dc", "dirty bowl-9e6ea28a14f9b649", "plate-6938b4e909b5f09f", "pan-079265a4e1b348b2", "dirty plate-c3219b471b4dda5a", "dirty bowl2-762645d994b0cd69", "dirty plate2-5c4673b4d76f9442", "toast rack-64ea49f9c49520cc", "splatter guard-9b657c27ec8f206d", "bag of potatoes-8c34fa6ece799d60", "knife-d5a90e358104cf8e", "rich tea biscuits-63873256a79ca593", "wooden board-b1fb86ec1af9f832", "knife2-2467ef7e5f81c335", "pastry knife-aeeba31b852c7759", "lid-54e1957ed0240c00", "dirty spoon3-34337b9883d46881", "cheese grater-0a3e35504dfa2516", "kettle-03640175a5d194e0", "packet of bacon-f6d2c80fdca32296", "scales-98351d719a7cdf76", "skewer-2abb23c1b9e2019a", "knife3-5231b12bb0800120", "chopping board-c4d82758bae0192b", "clove of garlic-7fbfcb585485667e", "frying pan-afeb3171f9d73b44", "piece of bacon-06e2bc8b7c7bf6c9", "bottle of oil-3d46f688ba1d4199", "laundry piece7-4753e90504806056", "laundry piece5-792e09156bd8e44e", "laundry piece2-8453c379098ce721", "laundry piece3-9aceee288f771adc", "wooden spoon-90ce22c5bf54dd37", "metal bowl-4d94a6a18235cb96", "packet of eggs-cc647d4f21492fca", "egg1-b4d86558aacef4ae", "egg2-49b3c1df5fbf5e04", "bowl-4e3748333958fdab", "egg3-10d0494e14414d2b", "egg4-95cdfe5e7f75973d", "egg5-27bb2d87c2b69223", "eggshells-ec30a182d2ce3de2", "pot-a05582f9ce801bc1", "salt container-8080ccc21342a025", "pot lid-62c43e7df95c3a55", "cheese package-7efa514770bf8626", "cheese block-944d1022a942e9e0", "cheese grater2-74015c87812a6675", "washing up brush-2e8d9079c8abb0aa", "rind-0113d936a69507ff", "packet of pasta-9df8300cac513437", "phone-c847575caf5d82d2", "fine cheese grater-4c58ed0f9b012cf3", "pot of pepper-ae7e26371bb94891", "fork-d265c3d8289354aa", "mushrooms-5616728ad4e82788", "lettuce-7af8afa4306535dc", "metal bowl2-df50795bfd325957" ], "reasoning": "1. I locate the dog bed at 00:01 to the right of the table in front of the user. \n2. I observe the user grabbing the dog bed with both hands at 00:13.\n3. I rewind from 00:13 to determine that the user's right hand touches the blue cabinet at 00:12 as the user puts away the peanut butter (tub of peanut butter-547d97278d20f202).\n4. I determine that the user's right hand touches the blue cabinet before they grab the dog bed with both hands.", "referenced_object_masks": [ "tub of peanut butter-547d97278d20f202" ] }, { "key": "P05-20240425-171455:9e6fc3fe438e3292b97d74af627257db6585a029", "video_id": "P05-20240425-171455", "question": "As I straighten out the dog bed, how does the stuffed animal on top of it rotate?", "answer_choice_0": "Counterclockwise 180 degrees", "answer_choice_1": "Clockwise 180 degrees", "answer_choice_2": "Clockwise 90 degrees", "answer_choice_3": "Counterclockwise 90 degrees", "answer_choice_4": "Counterclockwise 45 degrees", "answer_id": 3, "answer": "Counterclockwise 90 degrees", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning", "Numerical Reasoning" ], "objects_of_interest": [ "tub of peanut butter-547d97278d20f202", "ramekin-2c72710322a992ad", "dirty knife-cd75b945f5ded97f", "dirty spoon-7852cb9b136deb10", "dirty spoon2-dd929219536661dc", "dirty bowl-9e6ea28a14f9b649", "plate-6938b4e909b5f09f", "pan-079265a4e1b348b2", "dirty plate-c3219b471b4dda5a", "dirty bowl2-762645d994b0cd69", "dirty plate2-5c4673b4d76f9442", "toast rack-64ea49f9c49520cc", "splatter guard-9b657c27ec8f206d", "bag of potatoes-8c34fa6ece799d60", "knife-d5a90e358104cf8e", "rich tea biscuits-63873256a79ca593", "wooden board-b1fb86ec1af9f832", "knife2-2467ef7e5f81c335", "pastry knife-aeeba31b852c7759", "lid-54e1957ed0240c00", "dirty spoon3-34337b9883d46881", "cheese grater-0a3e35504dfa2516", "kettle-03640175a5d194e0", "packet of bacon-f6d2c80fdca32296", "scales-98351d719a7cdf76", "skewer-2abb23c1b9e2019a", "knife3-5231b12bb0800120", "chopping board-c4d82758bae0192b", "clove of garlic-7fbfcb585485667e", "frying pan-afeb3171f9d73b44", "piece of bacon-06e2bc8b7c7bf6c9", "bottle of oil-3d46f688ba1d4199", "laundry piece7-4753e90504806056", "laundry piece5-792e09156bd8e44e", "laundry piece2-8453c379098ce721", "laundry piece3-9aceee288f771adc", "wooden spoon-90ce22c5bf54dd37", "metal bowl-4d94a6a18235cb96", "packet of eggs-cc647d4f21492fca", "egg1-b4d86558aacef4ae", "egg2-49b3c1df5fbf5e04", "bowl-4e3748333958fdab", "egg3-10d0494e14414d2b", "egg4-95cdfe5e7f75973d", "egg5-27bb2d87c2b69223", "eggshells-ec30a182d2ce3de2", "pot-a05582f9ce801bc1", "salt container-8080ccc21342a025", "pot lid-62c43e7df95c3a55", "cheese package-7efa514770bf8626", "cheese block-944d1022a942e9e0", "cheese grater2-74015c87812a6675", "washing up brush-2e8d9079c8abb0aa", "rind-0113d936a69507ff", "packet of pasta-9df8300cac513437", "phone-c847575caf5d82d2", "fine cheese grater-4c58ed0f9b012cf3", "pot of pepper-ae7e26371bb94891", "fork-d265c3d8289354aa", "mushrooms-5616728ad4e82788", "lettuce-7af8afa4306535dc", "metal bowl2-df50795bfd325957" ], "reasoning": "1. I locate the dog bed at 00:01 to the right of the table in front of the user.\n2. I locate the stuffed animal on top of the dog bed at 00:07 to the left of the user. \n3. I observe the user straightening out the dog bed from 00:13 to 00:19.\n4. From 00:14 to 00:16, I observe the tail of the stuffed animal rotate counterclockwise 90 degrees.", "referenced_object_masks": [] }, { "key": "P05-20240425-171455:a4355eba404f044b8555fe87bf81bc5d3e5885b5", "video_id": "P05-20240425-171455", "question": "How many more egg yolks do I drop into the metal bowl after placing the pink bowl on the counter compared to before?", "answer_choice_0": "3", "answer_choice_1": "2", "answer_choice_2": "5", "answer_choice_3": "4", "answer_choice_4": "1", "answer_id": 4, "answer": "1", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counterfactual", "Numerical Reasoning" ], "objects_of_interest": [ "tub of peanut butter-547d97278d20f202", "ramekin-2c72710322a992ad", "dirty knife-cd75b945f5ded97f", "dirty spoon-7852cb9b136deb10", "dirty spoon2-dd929219536661dc", "dirty bowl-9e6ea28a14f9b649", "plate-6938b4e909b5f09f", "pan-079265a4e1b348b2", "dirty plate-c3219b471b4dda5a", "dirty bowl2-762645d994b0cd69", "dirty plate2-5c4673b4d76f9442", "toast rack-64ea49f9c49520cc", "splatter guard-9b657c27ec8f206d", "bag of potatoes-8c34fa6ece799d60", "knife-d5a90e358104cf8e", "rich tea biscuits-63873256a79ca593", "wooden board-b1fb86ec1af9f832", "knife2-2467ef7e5f81c335", "pastry knife-aeeba31b852c7759", "lid-54e1957ed0240c00", "dirty spoon3-34337b9883d46881", "cheese grater-0a3e35504dfa2516", "kettle-03640175a5d194e0", "packet of bacon-f6d2c80fdca32296", "scales-98351d719a7cdf76", "skewer-2abb23c1b9e2019a", "knife3-5231b12bb0800120", "chopping board-c4d82758bae0192b", "clove of garlic-7fbfcb585485667e", "frying pan-afeb3171f9d73b44", "piece of bacon-06e2bc8b7c7bf6c9", "bottle of oil-3d46f688ba1d4199", "laundry piece7-4753e90504806056", "laundry piece5-792e09156bd8e44e", "laundry piece2-8453c379098ce721", "laundry piece3-9aceee288f771adc", "wooden spoon-90ce22c5bf54dd37", "metal bowl-4d94a6a18235cb96", "packet of eggs-cc647d4f21492fca", "egg1-b4d86558aacef4ae", "egg2-49b3c1df5fbf5e04", "bowl-4e3748333958fdab", "egg3-10d0494e14414d2b", "egg4-95cdfe5e7f75973d", "egg5-27bb2d87c2b69223", "eggshells-ec30a182d2ce3de2", "pot-a05582f9ce801bc1", "salt container-8080ccc21342a025", "pot lid-62c43e7df95c3a55", "cheese package-7efa514770bf8626", "cheese block-944d1022a942e9e0", "cheese grater2-74015c87812a6675", "washing up brush-2e8d9079c8abb0aa", "rind-0113d936a69507ff", "packet of pasta-9df8300cac513437", "phone-c847575caf5d82d2", "fine cheese grater-4c58ed0f9b012cf3", "pot of pepper-ae7e26371bb94891", "fork-d265c3d8289354aa", "mushrooms-5616728ad4e82788", "lettuce-7af8afa4306535dc", "metal bowl2-df50795bfd325957" ], "reasoning": "1. I observe the user picking up a metal bowl (metal bowl-4d94a6a18235cb96) from under the sink at 08:07.\n2. At 08:11, I observe the user placing the metal bowl (metal bowl-4d94a6a18235cb96) to the left of a yellow egg carton (packet of eggs-cc647d4f21492fca).\n3. I observe the user picking up a bowl (bowl-4e3748333958fdab) from under the sink at 08:39.\n4. At 08:44, I observe the user placing the pink bowl (bowl-4e3748333958fdab) to the right of the metal bowl (metal bowl-4d94a6a18235cb96).\n5. Between 08:11 and 08:39, I observe the user drop 2 yolks into the metal bowl (metal bowl-4d94a6a18235cb96) from the first egg (egg1-b4d86558aacef4ae) and the second egg (egg2-49b3c1df5fbf5e04).\n6. After 08:44, I observe the user drop 3 yolks into the metal bowl (metal bowl-4d94a6a18235cb96) from the third egg (egg3-10d0494e14414d2b), fourth egg (egg4-95cdfe5e7f75973d), and fifth egg (egg5-27bb2d87c2b69223).\n7. I subtract 2 from 3 to get 1.", "referenced_object_masks": [ "egg1-b4d86558aacef4ae", "bowl-4e3748333958fdab", "packet of eggs-cc647d4f21492fca", "egg3-10d0494e14414d2b", "egg4-95cdfe5e7f75973d", "metal bowl-4d94a6a18235cb96", "egg5-27bb2d87c2b69223", "egg2-49b3c1df5fbf5e04" ] }, { "key": "P05-20240425-171455:a4db1ed26f83801a7befd2d38a46a9a5ec87e2cb", "video_id": "P05-20240425-171455", "question": "How many more times do I vigorously shake the pan filled with bacon on the fifth time of shaking it than the first?", "answer_choice_0": "4 times", "answer_choice_1": "3 times", "answer_choice_2": "1 time", "answer_choice_3": "2 times", "answer_choice_4": "5 times", "answer_id": 3, "answer": "2 times", "question_type": [ "Action Recognition", "Counting", "Object Recognition" ], "objects_of_interest": [ "tub of peanut butter-547d97278d20f202", "ramekin-2c72710322a992ad", "dirty knife-cd75b945f5ded97f", "dirty spoon-7852cb9b136deb10", "dirty spoon2-dd929219536661dc", "dirty bowl-9e6ea28a14f9b649", "plate-6938b4e909b5f09f", "pan-079265a4e1b348b2", "dirty plate-c3219b471b4dda5a", "dirty bowl2-762645d994b0cd69", "dirty plate2-5c4673b4d76f9442", "toast rack-64ea49f9c49520cc", "splatter guard-9b657c27ec8f206d", "bag of potatoes-8c34fa6ece799d60", "knife-d5a90e358104cf8e", "rich tea biscuits-63873256a79ca593", "wooden board-b1fb86ec1af9f832", "knife2-2467ef7e5f81c335", "pastry knife-aeeba31b852c7759", "lid-54e1957ed0240c00", "dirty spoon3-34337b9883d46881", "cheese grater-0a3e35504dfa2516", "kettle-03640175a5d194e0", "packet of bacon-f6d2c80fdca32296", "scales-98351d719a7cdf76", "skewer-2abb23c1b9e2019a", "knife3-5231b12bb0800120", "chopping board-c4d82758bae0192b", "clove of garlic-7fbfcb585485667e", "frying pan-afeb3171f9d73b44", "piece of bacon-06e2bc8b7c7bf6c9", "bottle of oil-3d46f688ba1d4199", "laundry piece7-4753e90504806056", "laundry piece5-792e09156bd8e44e", "laundry piece2-8453c379098ce721", "laundry piece3-9aceee288f771adc", "wooden spoon-90ce22c5bf54dd37", "metal bowl-4d94a6a18235cb96", "packet of eggs-cc647d4f21492fca", "egg1-b4d86558aacef4ae", "egg2-49b3c1df5fbf5e04", "bowl-4e3748333958fdab", "egg3-10d0494e14414d2b", "egg4-95cdfe5e7f75973d", "egg5-27bb2d87c2b69223", "eggshells-ec30a182d2ce3de2", "pot-a05582f9ce801bc1", "salt container-8080ccc21342a025", "pot lid-62c43e7df95c3a55", "cheese package-7efa514770bf8626", "cheese block-944d1022a942e9e0", "cheese grater2-74015c87812a6675", "washing up brush-2e8d9079c8abb0aa", "rind-0113d936a69507ff", "packet of pasta-9df8300cac513437", "phone-c847575caf5d82d2", "fine cheese grater-4c58ed0f9b012cf3", "pot of pepper-ae7e26371bb94891", "fork-d265c3d8289354aa", "mushrooms-5616728ad4e82788", "lettuce-7af8afa4306535dc", "metal bowl2-df50795bfd325957" ], "reasoning": "1. I look for the first time the user starts shaking the pan (frying pan-afeb3171f9d73b44) filled with bacon bits, finding the time from 07:45-07:57.\n2. During this time range, I notice the user shaking the pan (frying pan-afeb3171f9d73b44) once.\n3. I look for the fifth time the user started shaking the pan (frying pan-afeb3171f9d73b44). I pass 09:45-09:52, 10:27-10:39, and 12:02-12:14, and I finally notice the fifth time the user shakes the pan (frying pan-afeb3171f9d73b44) from 12:35-12:51.\n4. During this time range, the user shakes the pan (frying pan-afeb3171f9d73b44) 3 times.\n5. Therefore, the user shakes the pan (frying pan-afeb3171f9d73b44) 2 times more the fifth time than they do the second time.", "referenced_object_masks": [ "frying pan-afeb3171f9d73b44" ] }, { "key": "P05-20240425-171455:c48a1d27782616483c739a9fc5887a1368c8bcea", "video_id": "P05-20240425-171455", "question": "By the time I set the dishwasher to run, how many even numbers were shown on its screen?", "answer_choice_0": "2", "answer_choice_1": "6", "answer_choice_2": "5", "answer_choice_3": "4", "answer_choice_4": "3", "answer_id": 1, "answer": "6", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counting", "Reading", "Numerical Reasoning" ], "objects_of_interest": [ "tub of peanut butter-547d97278d20f202", "ramekin-2c72710322a992ad", "dirty knife-cd75b945f5ded97f", "dirty spoon-7852cb9b136deb10", "dirty spoon2-dd929219536661dc", "dirty bowl-9e6ea28a14f9b649", "plate-6938b4e909b5f09f", "pan-079265a4e1b348b2", "dirty plate-c3219b471b4dda5a", "dirty bowl2-762645d994b0cd69", "dirty plate2-5c4673b4d76f9442", "toast rack-64ea49f9c49520cc", "splatter guard-9b657c27ec8f206d", "bag of potatoes-8c34fa6ece799d60", "knife-d5a90e358104cf8e", "rich tea biscuits-63873256a79ca593", "wooden board-b1fb86ec1af9f832", "knife2-2467ef7e5f81c335", "pastry knife-aeeba31b852c7759", "lid-54e1957ed0240c00", "dirty spoon3-34337b9883d46881", "cheese grater-0a3e35504dfa2516", "kettle-03640175a5d194e0", "packet of bacon-f6d2c80fdca32296", "scales-98351d719a7cdf76", "skewer-2abb23c1b9e2019a", "knife3-5231b12bb0800120", "chopping board-c4d82758bae0192b", "clove of garlic-7fbfcb585485667e", "frying pan-afeb3171f9d73b44", "piece of bacon-06e2bc8b7c7bf6c9", "bottle of oil-3d46f688ba1d4199", "laundry piece7-4753e90504806056", "laundry piece5-792e09156bd8e44e", "laundry piece2-8453c379098ce721", "laundry piece3-9aceee288f771adc", "wooden spoon-90ce22c5bf54dd37", "metal bowl-4d94a6a18235cb96", "packet of eggs-cc647d4f21492fca", "egg1-b4d86558aacef4ae", "egg2-49b3c1df5fbf5e04", "bowl-4e3748333958fdab", "egg3-10d0494e14414d2b", "egg4-95cdfe5e7f75973d", "egg5-27bb2d87c2b69223", "eggshells-ec30a182d2ce3de2", "pot-a05582f9ce801bc1", "salt container-8080ccc21342a025", "pot lid-62c43e7df95c3a55", "cheese package-7efa514770bf8626", "cheese block-944d1022a942e9e0", "cheese grater2-74015c87812a6675", "washing up brush-2e8d9079c8abb0aa", "rind-0113d936a69507ff", "packet of pasta-9df8300cac513437", "phone-c847575caf5d82d2", "fine cheese grater-4c58ed0f9b012cf3", "pot of pepper-ae7e26371bb94891", "fork-d265c3d8289354aa", "mushrooms-5616728ad4e82788", "lettuce-7af8afa4306535dc", "metal bowl2-df50795bfd325957" ], "reasoning": "1. I locate the dishwasher at 00:23 to the left of the user as they approach the kitchen.\n2. I observe the user loading the dishwasher from 00:43 to 02:39.\n3. I observe the user setting the dishwasher to run from 02:41 to 02:52.\n4. I identify the number sequences 424 and 208 that show up on the dishwasher screen by 02:52.\n5. Given the numbers 4, 2, 4, 2, 0, and 8, I count 6 even numbers.", "referenced_object_masks": [] }, { "key": "P05-20240425-171455:e5588f36376804a1dab95fa745f807112a211e35", "video_id": "P05-20240425-171455", "question": "If I switched the pan with the pot 3 more times before the video ended, where would it's final place be?", "answer_choice_0": "The shelf", "answer_choice_1": "The left stovetop", "answer_choice_2": "The counter", "answer_choice_3": "The right stovetop", "answer_choice_4": "The middle stovetop", "answer_id": 3, "answer": "The right stovetop", "question_type": [ "Counterfactual", "Action Recognition", "Numerical Reasoning", "Egocentric & Spatial Perception", "State Changes" ], "objects_of_interest": [ "tub of peanut butter-547d97278d20f202", "ramekin-2c72710322a992ad", "dirty knife-cd75b945f5ded97f", "dirty spoon-7852cb9b136deb10", "dirty spoon2-dd929219536661dc", "dirty bowl-9e6ea28a14f9b649", "plate-6938b4e909b5f09f", "pan-079265a4e1b348b2", "dirty plate-c3219b471b4dda5a", "dirty bowl2-762645d994b0cd69", "dirty plate2-5c4673b4d76f9442", "toast rack-64ea49f9c49520cc", "splatter guard-9b657c27ec8f206d", "bag of potatoes-8c34fa6ece799d60", "knife-d5a90e358104cf8e", "rich tea biscuits-63873256a79ca593", "wooden board-b1fb86ec1af9f832", "knife2-2467ef7e5f81c335", "pastry knife-aeeba31b852c7759", "lid-54e1957ed0240c00", "dirty spoon3-34337b9883d46881", "cheese grater-0a3e35504dfa2516", "kettle-03640175a5d194e0", "packet of bacon-f6d2c80fdca32296", "scales-98351d719a7cdf76", "skewer-2abb23c1b9e2019a", "knife3-5231b12bb0800120", "chopping board-c4d82758bae0192b", "clove of garlic-7fbfcb585485667e", "frying pan-afeb3171f9d73b44", "piece of bacon-06e2bc8b7c7bf6c9", "bottle of oil-3d46f688ba1d4199", "laundry piece7-4753e90504806056", "laundry piece5-792e09156bd8e44e", "laundry piece2-8453c379098ce721", "laundry piece3-9aceee288f771adc", "wooden spoon-90ce22c5bf54dd37", "metal bowl-4d94a6a18235cb96", "packet of eggs-cc647d4f21492fca", "egg1-b4d86558aacef4ae", "egg2-49b3c1df5fbf5e04", "bowl-4e3748333958fdab", "egg3-10d0494e14414d2b", "egg4-95cdfe5e7f75973d", "egg5-27bb2d87c2b69223", "eggshells-ec30a182d2ce3de2", "pot-a05582f9ce801bc1", "salt container-8080ccc21342a025", "pot lid-62c43e7df95c3a55", "cheese package-7efa514770bf8626", "cheese block-944d1022a942e9e0", "cheese grater2-74015c87812a6675", "washing up brush-2e8d9079c8abb0aa", "rind-0113d936a69507ff", "packet of pasta-9df8300cac513437", "phone-c847575caf5d82d2", "fine cheese grater-4c58ed0f9b012cf3", "pot of pepper-ae7e26371bb94891", "fork-d265c3d8289354aa", "mushrooms-5616728ad4e82788", "lettuce-7af8afa4306535dc", "metal bowl2-df50795bfd325957" ], "reasoning": "1. I notice the user first put the pan (frying pan-afeb3171f9d73b44) on the left stovetop 07:24, and I notice the user first put the pot (pot-a05582f9ce801bc1) on the right stovetop from 09:58-09:59.\n2. From 12:51-12:54, I notice the user switch the pan (frying pan-afeb3171f9d73b44) with the pot (pot-a05582f9ce801bc1), leaving the pan on the right stovetop.\n3. From 14:07-14:10, I notice the user switch the pan (frying pan-afeb3171f9d73b44) with the pot (pot-a05582f9ce801bc1) again, leaving the pan (frying pan-afeb3171f9d73b44) at its original place on the left stovetop.\n4. After this, I do not notice the user switch the pan (frying pan-afeb3171f9d73b44) with the pot (pot-a05582f9ce801bc1) any more times.\n5. I imagine if the user switched the pan (frying pan-afeb3171f9d73b44) with the pot (pot-a05582f9ce801bc1) 3 more times. That would mean that the pot (pot-a05582f9ce801bc1) would be placed on the right, the left, and finally the right stovetop by the end of the exchanges.\n6. Therefore, the answer is right stovetop.", "referenced_object_masks": [ "pot-a05582f9ce801bc1", "frying pan-afeb3171f9d73b44" ] }, { "key": "P05-20240425-171455:f2acc767dc54487bb6e2450580faf05947881cd0", "video_id": "P05-20240425-171455", "question": "If the number of bacon bits that was accidentally pushed off the pan was multiplied to the weight of the oil added to the pan that cooked the bacon bits, would the result be greater or lesser than the weight of the fatty bacon ends from the first round of chopping up the bacon?", "answer_choice_0": "Greater", "answer_choice_1": "Impossible to say", "answer_choice_2": "I don't cook with oil", "answer_choice_3": "Equal", "answer_choice_4": "Lesser", "answer_id": 4, "answer": "Lesser", "question_type": [ "Numerical Reasoning", "Action Recognition", "Counting", "Object Recognition", "Reading", "Counterfactual" ], "objects_of_interest": [ "tub of peanut butter-547d97278d20f202", "ramekin-2c72710322a992ad", "dirty knife-cd75b945f5ded97f", "dirty spoon-7852cb9b136deb10", "dirty spoon2-dd929219536661dc", "dirty bowl-9e6ea28a14f9b649", "plate-6938b4e909b5f09f", "pan-079265a4e1b348b2", "dirty plate-c3219b471b4dda5a", "dirty bowl2-762645d994b0cd69", "dirty plate2-5c4673b4d76f9442", "toast rack-64ea49f9c49520cc", "splatter guard-9b657c27ec8f206d", "bag of potatoes-8c34fa6ece799d60", "knife-d5a90e358104cf8e", "rich tea biscuits-63873256a79ca593", "wooden board-b1fb86ec1af9f832", "knife2-2467ef7e5f81c335", "pastry knife-aeeba31b852c7759", "lid-54e1957ed0240c00", "dirty spoon3-34337b9883d46881", "cheese grater-0a3e35504dfa2516", "kettle-03640175a5d194e0", "packet of bacon-f6d2c80fdca32296", "scales-98351d719a7cdf76", "skewer-2abb23c1b9e2019a", "knife3-5231b12bb0800120", "chopping board-c4d82758bae0192b", "clove of garlic-7fbfcb585485667e", "frying pan-afeb3171f9d73b44", "piece of bacon-06e2bc8b7c7bf6c9", "bottle of oil-3d46f688ba1d4199", "laundry piece7-4753e90504806056", "laundry piece5-792e09156bd8e44e", "laundry piece2-8453c379098ce721", "laundry piece3-9aceee288f771adc", "wooden spoon-90ce22c5bf54dd37", "metal bowl-4d94a6a18235cb96", "packet of eggs-cc647d4f21492fca", "egg1-b4d86558aacef4ae", "egg2-49b3c1df5fbf5e04", "bowl-4e3748333958fdab", "egg3-10d0494e14414d2b", "egg4-95cdfe5e7f75973d", "egg5-27bb2d87c2b69223", "eggshells-ec30a182d2ce3de2", "pot-a05582f9ce801bc1", "salt container-8080ccc21342a025", "pot lid-62c43e7df95c3a55", "cheese package-7efa514770bf8626", "cheese block-944d1022a942e9e0", "cheese grater2-74015c87812a6675", "washing up brush-2e8d9079c8abb0aa", "rind-0113d936a69507ff", "packet of pasta-9df8300cac513437", "phone-c847575caf5d82d2", "fine cheese grater-4c58ed0f9b012cf3", "pot of pepper-ae7e26371bb94891", "fork-d265c3d8289354aa", "mushrooms-5616728ad4e82788", "lettuce-7af8afa4306535dc", "metal bowl2-df50795bfd325957" ], "reasoning": "1. First, I look for the number of bacon bits pushed off the pan (frying pan-afeb3171f9d73b44), and I find the moment at 16:54.\n2. I count the number of bits pushed off the pan (frying pan-afeb3171f9d73b44) to be 4.\n3. I look for the weight of oil poured into the pan (frying pan-afeb3171f9d73b44) that cooked the bacon bits , finding my answer at 07:20, when the oil bottle (bottle of oil-3d46f688ba1d4199) has been poured and the scale (scales-98351d719a7cdf76) shows a \"7\".\n4. Finally, I try to find the weight of the fatty bacon ends weighed from the first round of chopping bacon (piece of bacon-06e2bc8b7c7bf6c9).\n5. I notice that from 05:19-05:31, the user chops the fatty ends of bacon strips . From 05:41-05:42, I notice the user pushes these bits into the pan (frying pan-afeb3171f9d73b44).\n6. Around 05:46, I get a good view of the weight of the scale (scales-98351d719a7cdf76): it reads \"31\".\n7. I multiple 4 by 7 and get 28.\n8. 28 is less than 31.\n9. Therefore, the answer is that the number of bacon bits multiplied to the weight of the oil is not more than the weight of the fatty bacon ends chopped in the first round of chopping up bacon (piece of bacon-06e2bc8b7c7bf6c9).", "referenced_object_masks": [ "bottle of oil-3d46f688ba1d4199", "frying pan-afeb3171f9d73b44", "scales-98351d719a7cdf76", "piece of bacon-06e2bc8b7c7bf6c9" ] }, { "key": "P05-20240425-171455:fd72675be39bcaf5087b1b4ad7699baeec47a374", "video_id": "P05-20240425-171455", "question": "If I had set my timer with my voice the second time I looked at my phone, for the same amount of time as I set it for my first time, what would the phone's clock read when the timer was finished, if it was in standard time?", "answer_choice_0": "11:35", "answer_choice_1": "7:41", "answer_choice_2": "12:41", "answer_choice_3": "1:57", "answer_choice_4": "12:27", "answer_id": 2, "answer": "12:41", "question_type": [ "Audio Reasoning", "Counting", "Counterfactual", "Numerical Reasoning", "Object Recognition", "Reading" ], "objects_of_interest": [ "tub of peanut butter-547d97278d20f202", "ramekin-2c72710322a992ad", "dirty knife-cd75b945f5ded97f", "dirty spoon-7852cb9b136deb10", "dirty spoon2-dd929219536661dc", "dirty bowl-9e6ea28a14f9b649", "plate-6938b4e909b5f09f", "pan-079265a4e1b348b2", "dirty plate-c3219b471b4dda5a", "dirty bowl2-762645d994b0cd69", "dirty plate2-5c4673b4d76f9442", "toast rack-64ea49f9c49520cc", "splatter guard-9b657c27ec8f206d", "bag of potatoes-8c34fa6ece799d60", "knife-d5a90e358104cf8e", "rich tea biscuits-63873256a79ca593", "wooden board-b1fb86ec1af9f832", "knife2-2467ef7e5f81c335", "pastry knife-aeeba31b852c7759", "lid-54e1957ed0240c00", "dirty spoon3-34337b9883d46881", "cheese grater-0a3e35504dfa2516", "kettle-03640175a5d194e0", "packet of bacon-f6d2c80fdca32296", "scales-98351d719a7cdf76", "skewer-2abb23c1b9e2019a", "knife3-5231b12bb0800120", "chopping board-c4d82758bae0192b", "clove of garlic-7fbfcb585485667e", "frying pan-afeb3171f9d73b44", "piece of bacon-06e2bc8b7c7bf6c9", "bottle of oil-3d46f688ba1d4199", "laundry piece7-4753e90504806056", "laundry piece5-792e09156bd8e44e", "laundry piece2-8453c379098ce721", "laundry piece3-9aceee288f771adc", "wooden spoon-90ce22c5bf54dd37", "metal bowl-4d94a6a18235cb96", "packet of eggs-cc647d4f21492fca", "egg1-b4d86558aacef4ae", "egg2-49b3c1df5fbf5e04", "bowl-4e3748333958fdab", "egg3-10d0494e14414d2b", "egg4-95cdfe5e7f75973d", "egg5-27bb2d87c2b69223", "eggshells-ec30a182d2ce3de2", "pot-a05582f9ce801bc1", "salt container-8080ccc21342a025", "pot lid-62c43e7df95c3a55", "cheese package-7efa514770bf8626", "cheese block-944d1022a942e9e0", "cheese grater2-74015c87812a6675", "washing up brush-2e8d9079c8abb0aa", "rind-0113d936a69507ff", "packet of pasta-9df8300cac513437", "phone-c847575caf5d82d2", "fine cheese grater-4c58ed0f9b012cf3", "pot of pepper-ae7e26371bb94891", "fork-d265c3d8289354aa", "mushrooms-5616728ad4e82788", "lettuce-7af8afa4306535dc", "metal bowl2-df50795bfd325957" ], "reasoning": "1. I look for the first time the user set a timer on their phone (phone-c847575caf5d82d2) with their voice, finding the time at 13:10.\n2. At this time range, the user audibly sets a timer for \"10 minutes\".\n3. Additionally, at this time range, the user also looks at their phone (phone-c847575caf5d82d2) and their clock reads \"17:27,\" or 12:27 in standard time.\n4. I look for the second time the user checks their phone (phone-c847575caf5d82d2), and I find it briefly at 17:16. At this time frame, the phone (phone-c847575caf5d82d2) reads \"17:31,\" or 12:31 in standard time.\n5. If the user had set a 10 minute timer when their phone (phone-c847575caf5d82d2) read 17:31, then that means that the timer would have finished at 17:41, or 12:41 in standard time.\n6. Therefore, the answer is 12:41.", "referenced_object_masks": [ "phone-c847575caf5d82d2" ] }, { "key": "P05-20240425-173353:25168af71b0446e6fab1fa61b8ec31107e0b72ea", "video_id": "P05-20240425-173353", "question": "When I scooped pasta onto the first plate, approximately how many degrees would I turn clockwise to face the refrigerator?", "answer_choice_0": "270", "answer_choice_1": "135", "answer_choice_2": "180", "answer_choice_3": "90", "answer_choice_4": "225", "answer_id": 2, "answer": "180", "question_type": [ "Egocentric & Spatial Perception", "Situational Awareness", "Temporal Reasoning", "Action Recognition", "Object Recognition" ], "objects_of_interest": [ "ladle-2180bde6b5cbbec3", "lid-e06733c55b1b5ccb", "cheese-bd54c418bfe6a7ab", "meat-dc95cd7226d0ac14", "wooden spoon-11ef0de2fcb16cc1", "mixing bowl-bee6460b96ee746e", "fork-12db0e74f75b91df", "lettuce packaging-e751e8d546652d6b", "lettuce-549caf39ed2e8665", "knife-1b8d3c9e5af693d0", "chopping board-b8faff1c06fb869b", "frying pan-bf30de9476415d5f", "jar of spice-9ad8de05d2bc5a4f", "tomato-c2602de382e6e677", "tomato2-8932c740383122c3", "tomato3-6e1896a290bfadb3", "tomato4-96ca4d8053ce0216", "tomato5-abeefb8a147a4e1d", "tomato6-b4d8a81da87b8e42", "tomato7-dbe12492aa05204e", "tomato8-fd1f762d9ed6cd23", "tomato9-0b27fc848cd531fc", "phone-ecc45f9ddd54a635", "box of food-26e15deaa3d0fc20", "carton of eggs-68b38e2700817cba", "bottle-0705a30917828e8b", "bottle2-65ed537430b62fc3", "bottle3-1d649c3fdf85bffb", "mixing bowl2-7f361e88d5e2a5b1", "bowl of spinach-e2247ad2420091db", "plastic wrapper-cde3638439eddbc1", "washing up bowl-9187cc33e40d4952", "dishwashing brush-4db5453244b604e2", "metal colander-54176de4a206395e", "glass mixing bowl-6d6bfa38e2130f08", "spoon2-380607ad7fcaa061", "mug-78aa653c95602a62", "left oven glove-f441e571c2e9825f", "right oven glove-b8276c0e340b13ef", "pot-35829abe74421e18", "plastic wrapper2-7ded76dd3a5bfc2e", "weighing scales-ba7110d8bcc51071", "plate-ddc680b2316d6643", "spoon3-775f0973d5292017", "fork2-e6c4e1c758b02ccc" ], "reasoning": "1. I watched the video and looked for when the user scooped pasta onto a plate (plate-ddc680b2316d6643). I saw that this occurred from 09:32-09:53. \n2. After 09:53, I studied the surroundings until the refrigerator was visible for the first time at 10:27. From 10:24-10:27, I saw the user turn approximately 180 degrees counterclockwise.", "referenced_object_masks": [ "plate-ddc680b2316d6643" ] }, { "key": "P05-20240425-173353:4a41f6d7facaa9de6a270bd559c8d83960998a79", "video_id": "P05-20240425-173353", "question": "When I used the ladle to add liquid to the pan, which of the following sets of items are arranged from the furthest away to the closest?", "answer_choice_0": "Dishwashing brush; knife; bottle of oil", "answer_choice_1": "Dishwashing brush; bottle of oil; knife", "answer_choice_2": "Knife; bottle of oil; dishwashing brush", "answer_choice_3": "Bottle of oil; knife; dishwashing brush", "answer_choice_4": "Knife; dishwashing brush; bottle of oil", "answer_id": 2, "answer": "Knife; bottle of oil; dishwashing brush", "question_type": [ "Egocentric & Spatial Perception", "Temporal Reasoning", "Action Recognition", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "ladle-2180bde6b5cbbec3", "lid-e06733c55b1b5ccb", "cheese-bd54c418bfe6a7ab", "meat-dc95cd7226d0ac14", "wooden spoon-11ef0de2fcb16cc1", "mixing bowl-bee6460b96ee746e", "fork-12db0e74f75b91df", "lettuce packaging-e751e8d546652d6b", "lettuce-549caf39ed2e8665", "knife-1b8d3c9e5af693d0", "chopping board-b8faff1c06fb869b", "frying pan-bf30de9476415d5f", "jar of spice-9ad8de05d2bc5a4f", "tomato-c2602de382e6e677", "tomato2-8932c740383122c3", "tomato3-6e1896a290bfadb3", "tomato4-96ca4d8053ce0216", "tomato5-abeefb8a147a4e1d", "tomato6-b4d8a81da87b8e42", "tomato7-dbe12492aa05204e", "tomato8-fd1f762d9ed6cd23", "tomato9-0b27fc848cd531fc", "phone-ecc45f9ddd54a635", "box of food-26e15deaa3d0fc20", "carton of eggs-68b38e2700817cba", "bottle-0705a30917828e8b", "bottle2-65ed537430b62fc3", "bottle3-1d649c3fdf85bffb", "mixing bowl2-7f361e88d5e2a5b1", "bowl of spinach-e2247ad2420091db", "plastic wrapper-cde3638439eddbc1", "washing up bowl-9187cc33e40d4952", "dishwashing brush-4db5453244b604e2", "metal colander-54176de4a206395e", "glass mixing bowl-6d6bfa38e2130f08", "spoon2-380607ad7fcaa061", "mug-78aa653c95602a62", "left oven glove-f441e571c2e9825f", "right oven glove-b8276c0e340b13ef", "pot-35829abe74421e18", "plastic wrapper2-7ded76dd3a5bfc2e", "weighing scales-ba7110d8bcc51071", "plate-ddc680b2316d6643", "spoon3-775f0973d5292017", "fork2-e6c4e1c758b02ccc" ], "reasoning": "1. I watched the video and saw that the user used the ladle (ladle-2180bde6b5cbbec3) to add liquid to the pan (frying pan-bf30de9476415d5f) from 00:10-00:22. \n2. I saw the user use a knife (knife-1b8d3c9e5af693d0) to chop lettuce (lettuce-549caf39ed2e8665) from 01:10-01:22. I looked for the first time the knife (knife-1b8d3c9e5af693d0) was visible and saw it at 00:28 on the chopping board (chopping board-b8faff1c06fb869b). \n3. I observed the surroundings from 00:22-00:28 and observed that from 00:25-00:28 the user turned 90 degrees counterclockwise from the stove where the liquid was added to the pan (frying pan-bf30de9476415d5f) to face the counter with the chopping board (chopping board-b8faff1c06fb869b). I also saw the user step slightly forward toward the counter. \n4. I looked for the dishwashing brush (dishwashing brush-4db5453244b604e2) and first saw it at 04:10 in the sink. \n5. I observed the surroundings from 04:10-04:30. At 04:15 I noticed that the sink was in the corner of a room, with a counter running perpendicular on the user's right side. From 04:23-04:24 I saw the user turn about 90 degrees counterclockwise and I saw the rest of the kitchen further down the room, with the windows at the far end. From 04:24-04:27 I saw the user walk several steps forward and turn slightly clockwise to face the stove with the frying pan (frying pan-bf30de9476415d5f). \n6. I looked for the bottle of oil (bottle2-65ed537430b62fc3) and saw it for the first time at 02:48 when the user took it out of a cabinet. \n7. I observed the surroundings from 02:48-02:51 and saw that the user turned counterclockwise about 45 degrees to face the window and countertop with the chopping board (chopping board-b8faff1c06fb869b). From 02:51-02:52, I saw the user step a few steps forward to approach the counter. \n8. Based on the number of steps taken to come back to the stove and the counter next to it, I determined that when the ladle (ladle-2180bde6b5cbbec3) was used from 00:10-00:22, the knife (knife-1b8d3c9e5af693d0) was closest, the bottle of oil (bottle2-65ed537430b62fc3) was second closest since it was in the cabinet at this point, and the dishwashing brush (dishwashing brush-4db5453244b604e2) was furthest since it was beyond the cabinet.", "referenced_object_masks": [ "dishwashing brush-4db5453244b604e2", "lettuce-549caf39ed2e8665", "frying pan-bf30de9476415d5f", "chopping board-b8faff1c06fb869b", "ladle-2180bde6b5cbbec3", "bottle2-65ed537430b62fc3", "knife-1b8d3c9e5af693d0" ] }, { "key": "P05-20240425-173353:4ab7a2cb9bdcc31ec5ede3bb19b6590638ad239f", "video_id": "P05-20240425-173353", "question": "Rounding to the nearest 10 seconds, how much time passed between when I opened the window and when I opened the refrigerator for the second time?", "answer_choice_0": "2 minutes and 10 seconds", "answer_choice_1": "1 minute and 50 seconds", "answer_choice_2": "50 seconds", "answer_choice_3": "1 minute and 10 seconds", "answer_choice_4": "1 minute and 30 seconds", "answer_id": 2, "answer": "50 seconds", "question_type": [ "Action Recognition", "Situational Awareness", "Object Recognition", "Temporal Reasoning", "Numerical Reasoning" ], "objects_of_interest": [ "ladle-2180bde6b5cbbec3", "lid-e06733c55b1b5ccb", "cheese-bd54c418bfe6a7ab", "meat-dc95cd7226d0ac14", "wooden spoon-11ef0de2fcb16cc1", "mixing bowl-bee6460b96ee746e", "fork-12db0e74f75b91df", "lettuce packaging-e751e8d546652d6b", "lettuce-549caf39ed2e8665", "knife-1b8d3c9e5af693d0", "chopping board-b8faff1c06fb869b", "frying pan-bf30de9476415d5f", "jar of spice-9ad8de05d2bc5a4f", "tomato-c2602de382e6e677", "tomato2-8932c740383122c3", "tomato3-6e1896a290bfadb3", "tomato4-96ca4d8053ce0216", "tomato5-abeefb8a147a4e1d", "tomato6-b4d8a81da87b8e42", "tomato7-dbe12492aa05204e", "tomato8-fd1f762d9ed6cd23", "tomato9-0b27fc848cd531fc", "phone-ecc45f9ddd54a635", "box of food-26e15deaa3d0fc20", "carton of eggs-68b38e2700817cba", "bottle-0705a30917828e8b", "bottle2-65ed537430b62fc3", "bottle3-1d649c3fdf85bffb", "mixing bowl2-7f361e88d5e2a5b1", "bowl of spinach-e2247ad2420091db", "plastic wrapper-cde3638439eddbc1", "washing up bowl-9187cc33e40d4952", "dishwashing brush-4db5453244b604e2", "metal colander-54176de4a206395e", "glass mixing bowl-6d6bfa38e2130f08", "spoon2-380607ad7fcaa061", "mug-78aa653c95602a62", "left oven glove-f441e571c2e9825f", "right oven glove-b8276c0e340b13ef", "pot-35829abe74421e18", "plastic wrapper2-7ded76dd3a5bfc2e", "weighing scales-ba7110d8bcc51071", "plate-ddc680b2316d6643", "spoon3-775f0973d5292017", "fork2-e6c4e1c758b02ccc" ], "reasoning": "1. I watched the video and saw that the user opened a window from 02:07-02:08. \n2. I identified the first 2 times that the user opened the refrigerator after 02:08: 02:34, 03:01. \n3. I found the time that elapsed by subtracting: 03:01-02:08=00:53. I rounded to the nearest 10 seconds: 00:50.", "referenced_object_masks": [] }, { "key": "P05-20240425-173353:5dcdd85ec8d2512fd789040ea287eed3e81df049", "video_id": "P05-20240425-173353", "question": "What would I get if I added the number of times I ladled liquid into the pan to the number of scoops of pasta I put on the plate?", "answer_choice_0": "7", "answer_choice_1": "8", "answer_choice_2": "9", "answer_choice_3": "6", "answer_choice_4": "10", "answer_id": 2, "answer": "9", "question_type": [ "Action Recognition", "Object Recognition", "Counting", "Numerical Reasoning" ], "objects_of_interest": [ "ladle-2180bde6b5cbbec3", "lid-e06733c55b1b5ccb", "cheese-bd54c418bfe6a7ab", "meat-dc95cd7226d0ac14", "wooden spoon-11ef0de2fcb16cc1", "mixing bowl-bee6460b96ee746e", "fork-12db0e74f75b91df", "lettuce packaging-e751e8d546652d6b", "lettuce-549caf39ed2e8665", "knife-1b8d3c9e5af693d0", "chopping board-b8faff1c06fb869b", "frying pan-bf30de9476415d5f", "jar of spice-9ad8de05d2bc5a4f", "tomato-c2602de382e6e677", "tomato2-8932c740383122c3", "tomato3-6e1896a290bfadb3", "tomato4-96ca4d8053ce0216", "tomato5-abeefb8a147a4e1d", "tomato6-b4d8a81da87b8e42", "tomato7-dbe12492aa05204e", "tomato8-fd1f762d9ed6cd23", "tomato9-0b27fc848cd531fc", "phone-ecc45f9ddd54a635", "box of food-26e15deaa3d0fc20", "carton of eggs-68b38e2700817cba", "bottle-0705a30917828e8b", "bottle2-65ed537430b62fc3", "bottle3-1d649c3fdf85bffb", "mixing bowl2-7f361e88d5e2a5b1", "bowl of spinach-e2247ad2420091db", "plastic wrapper-cde3638439eddbc1", "washing up bowl-9187cc33e40d4952", "dishwashing brush-4db5453244b604e2", "metal colander-54176de4a206395e", "glass mixing bowl-6d6bfa38e2130f08", "spoon2-380607ad7fcaa061", "mug-78aa653c95602a62", "left oven glove-f441e571c2e9825f", "right oven glove-b8276c0e340b13ef", "pot-35829abe74421e18", "plastic wrapper2-7ded76dd3a5bfc2e", "weighing scales-ba7110d8bcc51071", "plate-ddc680b2316d6643", "spoon3-775f0973d5292017", "fork2-e6c4e1c758b02ccc" ], "reasoning": "1. I watched the video and looked for the user to use the ladle (ladle-2180bde6b5cbbec3) to add liquid into the pan (frying pan-bf30de9476415d5f). I saw the user grab the ladle (ladle-2180bde6b5cbbec3) at 00:07. I noted the segments of each scoop: 00:10-00:12, 00:13-00:15, and 00:17-00:22. This makes 3 scoops of liquid into the pan (frying pan-bf30de9476415d5f).\n2. At 09:32, the user began to scoop pasta onto a plate (plate-ddc680b2316d6643) with a spoon (spoon3-775f0973d5292017). I noted the segments of each scoop: 09:32-09:35, 09:36-09:39, 09:40-09:42, 09:43-09:45, 09:47-09:48, and 09:51-09:53. This makes 6 scoops of pasta onto the plate (plate-ddc680b2316d6643).\n3. I added the numbers together: 3+6=9.", "referenced_object_masks": [ "ladle-2180bde6b5cbbec3", "spoon3-775f0973d5292017", "frying pan-bf30de9476415d5f", "plate-ddc680b2316d6643" ] }, { "key": "P05-20240425-173353:f18c758f3431964cd6769a1d21d860195381f68b", "video_id": "P05-20240425-173353", "question": "Rank these from highest to lowest: The number of times I chopped the lettuce; the number of tomato halves after slicing all my tomatoes; the number of times I chopped the spinach?", "answer_choice_0": "Lettuce chops; tomato halves; spinach chops", "answer_choice_1": "Tomato halves; lettuce chops; spinach chops", "answer_choice_2": "Spinach chops; lettuce chops; tomato halves", "answer_choice_3": "Tomato halves; spinach chops; lettuce chops", "answer_choice_4": "Lettuce chops; spinach chops; tomato halves", "answer_id": 1, "answer": "Tomato halves; lettuce chops; spinach chops", "question_type": [ "Counting", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "ladle-2180bde6b5cbbec3", "lid-e06733c55b1b5ccb", "cheese-bd54c418bfe6a7ab", "meat-dc95cd7226d0ac14", "wooden spoon-11ef0de2fcb16cc1", "mixing bowl-bee6460b96ee746e", "fork-12db0e74f75b91df", "lettuce packaging-e751e8d546652d6b", "lettuce-549caf39ed2e8665", "knife-1b8d3c9e5af693d0", "chopping board-b8faff1c06fb869b", "frying pan-bf30de9476415d5f", "jar of spice-9ad8de05d2bc5a4f", "tomato-c2602de382e6e677", "tomato2-8932c740383122c3", "tomato3-6e1896a290bfadb3", "tomato4-96ca4d8053ce0216", "tomato5-abeefb8a147a4e1d", "tomato6-b4d8a81da87b8e42", "tomato7-dbe12492aa05204e", "tomato8-fd1f762d9ed6cd23", "tomato9-0b27fc848cd531fc", "phone-ecc45f9ddd54a635", "box of food-26e15deaa3d0fc20", "carton of eggs-68b38e2700817cba", "bottle-0705a30917828e8b", "bottle2-65ed537430b62fc3", "bottle3-1d649c3fdf85bffb", "mixing bowl2-7f361e88d5e2a5b1", "bowl of spinach-e2247ad2420091db", "plastic wrapper-cde3638439eddbc1", "washing up bowl-9187cc33e40d4952", "dishwashing brush-4db5453244b604e2", "metal colander-54176de4a206395e", "glass mixing bowl-6d6bfa38e2130f08", "spoon2-380607ad7fcaa061", "mug-78aa653c95602a62", "left oven glove-f441e571c2e9825f", "right oven glove-b8276c0e340b13ef", "pot-35829abe74421e18", "plastic wrapper2-7ded76dd3a5bfc2e", "weighing scales-ba7110d8bcc51071", "plate-ddc680b2316d6643", "spoon3-775f0973d5292017", "fork2-e6c4e1c758b02ccc" ], "reasoning": "1. I watched the video and saw the user begin to chop the lettuce (lettuce-549caf39ed2e8665) at 01:10. I counted each chop and noted the times: 01:11, 2 chops at 01:12, 01:13, 2 chops at 01:14, 01:15, 01:16, 01:17, 01:18, 01:19, 2 chops at 01:20, 01:21, and 01:22. This makes a total of 15 chops of lettuce (lettuce-549caf39ed2e8665). \n2. I saw the user begin to slice the first tomato (tomato-c2602de382e6e677) in half at 01:52. With this being the first, I noted the times the subsequent fruits were sliced in half: The second tomato (tomato2-8932c740383122c3) at 01:53, the third tomato (tomato3-6e1896a290bfadb3) at 01:55, the fourth tomato (tomato4-96ca4d8053ce0216) at 01:57, the fifth tomato (tomato5-abeefb8a147a4e1d) at 01:59, the sixth tomato (tomato6-b4d8a81da87b8e42) at 02:01, the seventh tomato (tomato7-dbe12492aa05204e) at 02:03, and the eighth tomato (tomato8-fd1f762d9ed6cd23) at 02:04. This makes a total of 8 tomatoes (tomato-c2602de382e6e677, tomato2-8932c740383122c3, tomato3-6e1896a290bfadb3, tomato4-96ca4d8053ce0216, tomato5-abeefb8a147a4e1d, tomato6-b4d8a81da87b8e42, tomato7-dbe12492aa05204e, tomato8-fd1f762d9ed6cd23) sliced in half, for a total of 16 halves. \n3. I saw the user grab a handful of spinach from the bowl of spinach (bowl of spinach-e2247ad2420091db) in the refrigerator at 03:07. I saw the user bring the handful of spinach to the chopping board (chopping board-b8faff1c06fb869b) and begin to chop it at 03:18. I counted each chop and noted the times: 2 chops at 03:18, 03:19, 03:20, 03:21, 03:22, 03:24, 03:25, 2 chops at 03:27, and 3 chops at 03:29. This makes a total of 13 chops of the handful of spinach .", "referenced_object_masks": [ "tomato7-dbe12492aa05204e", "lettuce-549caf39ed2e8665", "tomato2-8932c740383122c3", "chopping board-b8faff1c06fb869b", "tomato8-fd1f762d9ed6cd23", "tomato6-b4d8a81da87b8e42", "tomato5-abeefb8a147a4e1d", "tomato-c2602de382e6e677", "tomato3-6e1896a290bfadb3", "tomato4-96ca4d8053ce0216", "bowl of spinach-e2247ad2420091db" ] }, { "key": "P05-20240425-180658:13e89cf5c9a9e76bab20203401a83bbe5e69d86e", "video_id": "P05-20240425-180658", "question": "How much time passes between putting away the biscuits and putting the lid on the pot with pasta?", "answer_choice_0": "43 seconds", "answer_choice_1": "1 minute and 32 seconds", "answer_choice_2": "2 minutes and 3 seconds", "answer_choice_3": "1 minute and 8 seconds", "answer_choice_4": "2 minutes and 50 seconds", "answer_id": 3, "answer": "1 minute and 8 seconds", "question_type": [ "Action Recognition", "Object Recognition", "Temporal Reasoning", "Numerical Reasoning" ], "objects_of_interest": [ "knife-9df1087635edc06e", "colander-550f8747c24d1366", "washing up brush-35411674f020c9fc", "dirty bowl-0778e8f320a1a9e6", "pan-36dbc5aa91ba83cc", "cheese-0bdc0df93718ded0", "plastic bag-7705a7e312d6a683", "jar of pepper-0e9673af2459a70e", "cheese grater-eabd82e07e8deabd", "oil-8e6c876e926a8c0b", "vinegar-5647d68e91dadbae", "packet of rich teas-4e7116b91ffaa32d", "rich tea-724518bc8f5b8f80", "cup-5931716f0d47be9d", "olive oil-491ba83f2cb9bff2", "chopping board-0bb91b8118c5f36c", "kettle-ccb21ce7c00bdf7a", "colander2-fa0b38cc520bbf32", "silver foil-fa25f1ccb6af3dad", "kitchen roll-e3670f1a2067a406", "bowl of egg white-9232d97279643adf", "lid-5c2461f9b45e9249", "pan2-4145e715b1046368", "butter paper-081b8db5a8acd503", "lid of butter dish-8a935bc26aac2849", "bowl2-a22c09c05ba854a8", "towel-54e34efd23213f55" ], "reasoning": "1. I watched the video and saw that the user put away the biscuits (packet of rich teas-4e7116b91ffaa32d) in a cabinet from 01:36-01:40. \n2. I saw that the user put a lid (lid-5c2461f9b45e9249) on the pot of pasta (pan2-4145e715b1046368) from 02:45-02:48. \n3. I found the time that elapsed by subtracting the ends of each segment: 02:48-01:40=01:08.", "referenced_object_masks": [ "lid-5c2461f9b45e9249", "pan2-4145e715b1046368", "packet of rich teas-4e7116b91ffaa32d" ] }, { "key": "P05-20240425-180658:7ab1c0994c7c444a074556b563c3e2a39074f670", "video_id": "P05-20240425-180658", "question": "How many items did I scrub with the dish scrubber before putting the butter in the butter dish?", "answer_choice_0": "3", "answer_choice_1": "2", "answer_choice_2": "1", "answer_choice_3": "0", "answer_choice_4": "4", "answer_id": 1, "answer": "2", "question_type": [ "Action Recognition", "Object Recognition", "Counting", "Temporal Reasoning" ], "objects_of_interest": [ "knife-9df1087635edc06e", "colander-550f8747c24d1366", "washing up brush-35411674f020c9fc", "dirty bowl-0778e8f320a1a9e6", "pan-36dbc5aa91ba83cc", "cheese-0bdc0df93718ded0", "plastic bag-7705a7e312d6a683", "jar of pepper-0e9673af2459a70e", "cheese grater-eabd82e07e8deabd", "oil-8e6c876e926a8c0b", "vinegar-5647d68e91dadbae", "packet of rich teas-4e7116b91ffaa32d", "rich tea-724518bc8f5b8f80", "cup-5931716f0d47be9d", "olive oil-491ba83f2cb9bff2", "chopping board-0bb91b8118c5f36c", "kettle-ccb21ce7c00bdf7a", "colander2-fa0b38cc520bbf32", "silver foil-fa25f1ccb6af3dad", "kitchen roll-e3670f1a2067a406", "bowl of egg white-9232d97279643adf", "lid-5c2461f9b45e9249", "pan2-4145e715b1046368", "butter paper-081b8db5a8acd503", "lid of butter dish-8a935bc26aac2849", "bowl2-a22c09c05ba854a8", "towel-54e34efd23213f55" ], "reasoning": "1. I watched the video to find when the user put the butter in the butter dish . At 02:50 I saw the user lift the butter dish lid (lid of butter dish-8a935bc26aac2849) and release a stick of butter from a butter wrapper (butter paper-081b8db5a8acd503) onto the base of the butter dish from 03:01-03:13. \n2. I looked for the segments before 03:01 when the user scrubbed an item with the dish scrubber (washing up brush-35411674f020c9fc). The first occurrence was when the user scrubbed a colander (colander-550f8747c24d1366) from 00:15-00:18. \n3. The second occurrence was when the user scrubbed a knife (knife-9df1087635edc06e) from 00:24-00:29. \n4. There were no more occurrences, so I counted a total of 2.", "referenced_object_masks": [ "lid of butter dish-8a935bc26aac2849", "colander-550f8747c24d1366", "knife-9df1087635edc06e", "washing up brush-35411674f020c9fc", "butter paper-081b8db5a8acd503" ] }, { "key": "P05-20240425-180658:b3951acdb2f4a71643b46c98383edddac85cc9ef", "video_id": "P05-20240425-180658", "question": "Not counting doors or drawers, what was the 4th object with metal components I touched after drying the knife?", "answer_choice_0": "Dirty bowl", "answer_choice_1": "Grater", "answer_choice_2": "Colander", "answer_choice_3": "Jar with a metal lid", "answer_choice_4": "Pan", "answer_id": 1, "answer": "Grater", "question_type": [ "Object Recognition", "Counting", "Temporal Reasoning", "Action Recognition" ], "objects_of_interest": [ "knife-9df1087635edc06e", "colander-550f8747c24d1366", "washing up brush-35411674f020c9fc", "dirty bowl-0778e8f320a1a9e6", "pan-36dbc5aa91ba83cc", "cheese-0bdc0df93718ded0", "plastic bag-7705a7e312d6a683", "jar of pepper-0e9673af2459a70e", "cheese grater-eabd82e07e8deabd", "oil-8e6c876e926a8c0b", "vinegar-5647d68e91dadbae", "packet of rich teas-4e7116b91ffaa32d", "rich tea-724518bc8f5b8f80", "cup-5931716f0d47be9d", "olive oil-491ba83f2cb9bff2", "chopping board-0bb91b8118c5f36c", "kettle-ccb21ce7c00bdf7a", "colander2-fa0b38cc520bbf32", "silver foil-fa25f1ccb6af3dad", "kitchen roll-e3670f1a2067a406", "bowl of egg white-9232d97279643adf", "lid-5c2461f9b45e9249", "pan2-4145e715b1046368", "butter paper-081b8db5a8acd503", "lid of butter dish-8a935bc26aac2849", "bowl2-a22c09c05ba854a8", "towel-54e34efd23213f55" ], "reasoning": "1. I watched the video and looked for when the user dried the knife (knife-9df1087635edc06e). This occurred from 00:34-00:37. \n2. I identified the first 4 objects with metal components (not including doors or drawers) that the user touched after 00:37. The first was a dirty bowl (dirty bowl-0778e8f320a1a9e6) at 00:44.\n3. The second was a pan (pan-36dbc5aa91ba83cc) at 00:46.\n4. The third was a jar (jar of pepper-0e9673af2459a70e) with a metal lid at 01:15. \n5. The fourth was a grater (cheese grater-eabd82e07e8deabd) at 01:16.", "referenced_object_masks": [ "pan-36dbc5aa91ba83cc", "cheese grater-eabd82e07e8deabd", "jar of pepper-0e9673af2459a70e", "knife-9df1087635edc06e", "dirty bowl-0778e8f320a1a9e6" ] }, { "key": "P05-20240425-180658:c95d6fd2051a8cb036724e091d09a605bd4b737b", "video_id": "P05-20240425-180658", "question": "When I put away the tin foil, approximately how many degrees clockwise would I turn to face the bowl of egg white?", "answer_choice_0": "270", "answer_choice_1": "0", "answer_choice_2": "90", "answer_choice_3": "315", "answer_choice_4": "45", "answer_id": 4, "answer": "45", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Situational Awareness", "Temporal Reasoning" ], "objects_of_interest": [ "knife-9df1087635edc06e", "colander-550f8747c24d1366", "washing up brush-35411674f020c9fc", "dirty bowl-0778e8f320a1a9e6", "pan-36dbc5aa91ba83cc", "cheese-0bdc0df93718ded0", "plastic bag-7705a7e312d6a683", "jar of pepper-0e9673af2459a70e", "cheese grater-eabd82e07e8deabd", "oil-8e6c876e926a8c0b", "vinegar-5647d68e91dadbae", "packet of rich teas-4e7116b91ffaa32d", "rich tea-724518bc8f5b8f80", "cup-5931716f0d47be9d", "olive oil-491ba83f2cb9bff2", "chopping board-0bb91b8118c5f36c", "kettle-ccb21ce7c00bdf7a", "colander2-fa0b38cc520bbf32", "silver foil-fa25f1ccb6af3dad", "kitchen roll-e3670f1a2067a406", "bowl of egg white-9232d97279643adf", "lid-5c2461f9b45e9249", "pan2-4145e715b1046368", "butter paper-081b8db5a8acd503", "lid of butter dish-8a935bc26aac2849", "bowl2-a22c09c05ba854a8", "towel-54e34efd23213f55" ], "reasoning": "1. I watched the video and saw that the user picked up a wad of tin foil (silver foil-fa25f1ccb6af3dad) at 02:14. I saw the user put the wad of tin foil (silver foil-fa25f1ccb6af3dad) away on a shelf on an island at 02:18. \n2. I observed the surroundings from 02:14-02:18 and noted that from 02:14-02:16, the user faced a wall between a refrigerator on the left and an island on the right. From 02:16-02:18, I saw the user turn about 45 degrees to the right as they put away the wad of tin foil (silver foil-fa25f1ccb6af3dad). \n3. I looked for the first time the user handled the bowl of egg whites (bowl of egg white-9232d97279643adf). This occurred at 02:28.\n4. I observed the surroundings at 02:28 and noticed that the bowl of egg whites (bowl of egg white-9232d97279643adf) was on a counter between the user and the windows. I further observed from 02:28-02:30 that the user turned about 180 degrees clockwise. At 02:30, the refrigerator was ahead on their right. \n5. Since the bowl of egg whites (bowl of egg white-9232d97279643adf) were on the counter by the windows when the user put away the wad of tin foil (silver foil-fa25f1ccb6af3dad), I calculated that they would have had to turn approximately 45 degrees clockwise at 02:18 to face the bowl of egg whites (bowl of egg white-9232d97279643adf).", "referenced_object_masks": [ "silver foil-fa25f1ccb6af3dad", "bowl of egg white-9232d97279643adf" ] }, { "key": "P05-20240425-180658:dff16eb44ea834c7e440ac9e295a7130497b1958", "video_id": "P05-20240425-180658", "question": "Which was greater, and by how much: The number of times I opened the refrigerator, or the number of times I handled the gray towel?", "answer_choice_0": "Opening the refrigerator--by 2", "answer_choice_1": "Handling the towel--by 2", "answer_choice_2": "Opening the refrigerator--by 1", "answer_choice_3": "Handling the towel--by 1", "answer_choice_4": "Neither--the numbers are the same", "answer_id": 4, "answer": "Neither--the numbers are the same", "question_type": [ "Action Recognition", "Object Recognition", "Situational Awareness", "Counting", "Numerical Reasoning" ], "objects_of_interest": [ "knife-9df1087635edc06e", "colander-550f8747c24d1366", "washing up brush-35411674f020c9fc", "dirty bowl-0778e8f320a1a9e6", "pan-36dbc5aa91ba83cc", "cheese-0bdc0df93718ded0", "plastic bag-7705a7e312d6a683", "jar of pepper-0e9673af2459a70e", "cheese grater-eabd82e07e8deabd", "oil-8e6c876e926a8c0b", "vinegar-5647d68e91dadbae", "packet of rich teas-4e7116b91ffaa32d", "rich tea-724518bc8f5b8f80", "cup-5931716f0d47be9d", "olive oil-491ba83f2cb9bff2", "chopping board-0bb91b8118c5f36c", "kettle-ccb21ce7c00bdf7a", "colander2-fa0b38cc520bbf32", "silver foil-fa25f1ccb6af3dad", "kitchen roll-e3670f1a2067a406", "bowl of egg white-9232d97279643adf", "lid-5c2461f9b45e9249", "pan2-4145e715b1046368", "butter paper-081b8db5a8acd503", "lid of butter dish-8a935bc26aac2849", "bowl2-a22c09c05ba854a8", "towel-54e34efd23213f55" ], "reasoning": "1. I watched the video and counted each time the user opened the refrigerator : 01:08, 02:34, and 03:27. This makes a total of 3 times that the refrigerator was opened. \n2. I counted each time that the user handled the gray towel (towel-54e34efd23213f55): 00:34-00:37, 03:45-03:47, and 03:57-04:01. This makes a total of 3 times the gray towel (towel-54e34efd23213f55) was handled.\n3. I found the difference: 3-3=0.", "referenced_object_masks": [ "towel-54e34efd23213f55" ] }, { "key": "P05-20240425-201444:1f846aac3d31caea02777ea4dfa24a96995fcc20", "video_id": "P05-20240425-201444", "question": "After I pull the bag of sugar open, where are the next 2 places I go, in order?", "answer_choice_0": "Sink and refrigerator", "answer_choice_1": "Refrigerator and sink", "answer_choice_2": "Sink and kitchen island", "answer_choice_3": "Kitchen island and sink", "answer_choice_4": "Kitchen island and refrigerator", "answer_id": 2, "answer": "Sink and kitchen island", "question_type": [ "Temporal Reasoning", "Situational Awareness", "Egocentric & Spatial Perception", "Action Recognition", "Counting" ], "objects_of_interest": [ "scales-e5f94ff2ef748b44", "box of eggs-6f5955c996970c42", "glass bowl-9ead0b968b087d4d", "bowl-51070525ae5442f0", "egg1-792f0d07fe1965c9", "egg2-69f6dfec4c0366f8", "pan-415bb186d7c64fbb", "pot with handle-6b147ad932dd52f7", "lid-92522a1d6b473649", "mayonnaise-b4f81e16b9298126", "sugar bag-6b8295a9c890e045", "jar of chocolate buttons-a43fe58a7fc17d24", "knife-b576cfca96e4ec89", "lid of butter dish-6b58b7e6471533ee", "butter-443c0eca4519eb04", "mixer-39c3d84478cf12b4", "spatula-f17a414fbeb3a42d", "mixer blades-0f441d7a6c363730", "brush-01d619fc593fd587", "pot of flour-9d0764bbfcfc3fb5", "lid of pot of flour-0d9005db79ccadc9", "measuring spoon-7caab10dcdfb71b7", "ramekin1-8d01db3c12a50615", "ramekin2-9bec6b34ffbade1d", "spoon4-0193ffc50496414a", "both ramekins-c8d27ede390ee6d5", "oven glove-5a63c9197e7afae9", "shelf-a99e3b90f8d1bbfd", "phone-803857b825b8f51a", "mixer blade2-267d3f6f3eba6b99", "knife2-b33172dac6ba454e", "knife3-1c44a6a26dde1a07", "knife4-7a69c2bb038c8669", "spoon2-86c02722e7cea3d3", "knife5-d0069a2816a47845", "knife8-0dad6bda5d775d06", "knife7-d51e27b5053cba96", "knife6-1890e40baa4edeb5", "knife9-22560d351a0b0c50", "plate-37e72c60571ded88", "spoons-2d7a6bb9912c3b73", "spoon new-09072dcce260d73a", "spoons2-b38ba311c4c442e7", "glasses-5bd69b9526fd3d51", "glasses2-664b175a05040dcd", "stack of bowls-c0a809568ede7bc4", "one bowl-9eae99aed09da7de", "mug-73b5e0386ec68dec", "mug2-24e1861fb6b0fca8", "mug3-9cc6c399ef2c6e39", "mug4-35fff1b757f5b99f", "mug5-768afe13050e82c3", "mug6-8939a76897476123", "mug7-d505bad238e63f06", "mug8-1b25f5e5178f15b4", "mug9-94f6dd826e47f1b2", "mug10-77a0b62fa81bd84b", "mug11-018f3398e343825d", "chopping boards-66ebb8ef6ec3ad67", "chopping board2-5be0d9bb802d0f6d", "cheese graters-b4429764d99d8e7c", "cheese graters2-ed3b72b1f101a313", "plate2-ee08f4693dc8bbba", "plate new-4abe44aabed425f4", "plate3-e4d0047b5e5d7307", "plate4-af1e0144ba9babf3", "plate5-eb97acea80f79441", "plate6-f668c22ad7efce0b", "plate7-42dec1104fe5315e", "plate8-287dc6766f016846", "bowl3-1e9a912fe544b3b9", "bowl4-a8572ac2ae26311f", "pan2-502f7630bb110a5b", "pan lid-15de218aeb0c7847", "pan3-aa7f43cc002d3768", "knife10-6395d93998b8e8f6", "plate9-78d8adba0a04ef0f", "dirty knife-be36f0ad60eaa3b7", "dirty spoons-a02a7c960b14f0ba", "wooden spoon-b205fd5dfc8bc8c9", "colander-d5ad959f49bf0481", "fork-ed6ecf29d0dd02c3", "dirty bowl-b460fa60612f6964", "dirty pan-e80bc3f91737777f", "mug14-6f3c3fe67d4f83e1", "dirty plate1-b054beee5b63b90f", "dirty plate2-3d924d8074966e28", "cheese grater3-77677ab65ebb2e6a", "chopping board3-1abc2c84e321fcde", "mug15-da692c59f2043650", "mug13-0ad8ff6d30b226a9", "lid3-d793c3ec1efb7e6b", "glass3-9dc1c5b3fa216efe", "glass4-ffdc5feb83e3f50f", "spoon3-54fb87555b92d1b4", "plate10-cce905c4306a3e52", "plate12-446944685e823e8c", "dough scraper-6a7e9cef9abd18cb", "colander2-c2638075ddf22750", "glass5-c822cebc7060d90d", "plate11-8fd71216e19302f5", "towel-f29807236c61d991", "towel2-9d8f46337e93b48b", "knife11-00e4cdd01c1a5a07", "pot of double cream-d6acda522fef2f03", "second spoon-b55b10750f9affc2", "oven glove2-575a91f4240fef16", "spoon-bfd5ddf2bd38b4b6", "plate13-426eb6c9bf399566" ], "reasoning": "1. I look for the time I pull open the bag of sugar (sugar bag-6b8295a9c890e045), and I find it at 02:17.\n2. After this, I watch to see where the user goes.\n3. From 02:17-02:30, I watch the user move to the sink to wash their hands.\n4. Then, from 02:33-02:40, I watch the user move toward the counter, before pausing and moving toward the kitchen island to pull a jar of chocolate buttons (jar of chocolate buttons-a43fe58a7fc17d24).\n5. Therefore, the first 2 places, in order, that the user goes after pulling open the bag of sugar (sugar bag-6b8295a9c890e045) are the sink and the kitchen island.", "referenced_object_masks": [ "sugar bag-6b8295a9c890e045", "jar of chocolate buttons-a43fe58a7fc17d24" ] }, { "key": "P05-20240425-201444:3a4ce8379db344200f92c265553b0b17df6bebf5", "video_id": "P05-20240425-201444", "question": "How long does it take me to begin to ladle chocolate batter into the ramekin furthest to my right, counting from after I begin to ladle it into the first ramekin?", "answer_choice_0": "10 seconds", "answer_choice_1": "21 seconds", "answer_choice_2": "32 seconds", "answer_choice_3": "15 seconds", "answer_choice_4": "40 seconds", "answer_id": 1, "answer": "21 seconds", "question_type": [ "Egocentric & Spatial Perception", "Action Recognition", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "scales-e5f94ff2ef748b44", "box of eggs-6f5955c996970c42", "glass bowl-9ead0b968b087d4d", "bowl-51070525ae5442f0", "egg1-792f0d07fe1965c9", "egg2-69f6dfec4c0366f8", "pan-415bb186d7c64fbb", "pot with handle-6b147ad932dd52f7", "lid-92522a1d6b473649", "mayonnaise-b4f81e16b9298126", "sugar bag-6b8295a9c890e045", "jar of chocolate buttons-a43fe58a7fc17d24", "knife-b576cfca96e4ec89", "lid of butter dish-6b58b7e6471533ee", "butter-443c0eca4519eb04", "mixer-39c3d84478cf12b4", "spatula-f17a414fbeb3a42d", "mixer blades-0f441d7a6c363730", "brush-01d619fc593fd587", "pot of flour-9d0764bbfcfc3fb5", "lid of pot of flour-0d9005db79ccadc9", "measuring spoon-7caab10dcdfb71b7", "ramekin1-8d01db3c12a50615", "ramekin2-9bec6b34ffbade1d", "spoon4-0193ffc50496414a", "both ramekins-c8d27ede390ee6d5", "oven glove-5a63c9197e7afae9", "shelf-a99e3b90f8d1bbfd", "phone-803857b825b8f51a", "mixer blade2-267d3f6f3eba6b99", "knife2-b33172dac6ba454e", "knife3-1c44a6a26dde1a07", "knife4-7a69c2bb038c8669", "spoon2-86c02722e7cea3d3", "knife5-d0069a2816a47845", "knife8-0dad6bda5d775d06", "knife7-d51e27b5053cba96", "knife6-1890e40baa4edeb5", "knife9-22560d351a0b0c50", "plate-37e72c60571ded88", "spoons-2d7a6bb9912c3b73", "spoon new-09072dcce260d73a", "spoons2-b38ba311c4c442e7", "glasses-5bd69b9526fd3d51", "glasses2-664b175a05040dcd", "stack of bowls-c0a809568ede7bc4", "one bowl-9eae99aed09da7de", "mug-73b5e0386ec68dec", "mug2-24e1861fb6b0fca8", "mug3-9cc6c399ef2c6e39", "mug4-35fff1b757f5b99f", "mug5-768afe13050e82c3", "mug6-8939a76897476123", "mug7-d505bad238e63f06", "mug8-1b25f5e5178f15b4", "mug9-94f6dd826e47f1b2", "mug10-77a0b62fa81bd84b", "mug11-018f3398e343825d", "chopping boards-66ebb8ef6ec3ad67", "chopping board2-5be0d9bb802d0f6d", "cheese graters-b4429764d99d8e7c", "cheese graters2-ed3b72b1f101a313", "plate2-ee08f4693dc8bbba", "plate new-4abe44aabed425f4", "plate3-e4d0047b5e5d7307", "plate4-af1e0144ba9babf3", "plate5-eb97acea80f79441", "plate6-f668c22ad7efce0b", "plate7-42dec1104fe5315e", "plate8-287dc6766f016846", "bowl3-1e9a912fe544b3b9", "bowl4-a8572ac2ae26311f", "pan2-502f7630bb110a5b", "pan lid-15de218aeb0c7847", "pan3-aa7f43cc002d3768", "knife10-6395d93998b8e8f6", "plate9-78d8adba0a04ef0f", "dirty knife-be36f0ad60eaa3b7", "dirty spoons-a02a7c960b14f0ba", "wooden spoon-b205fd5dfc8bc8c9", "colander-d5ad959f49bf0481", "fork-ed6ecf29d0dd02c3", "dirty bowl-b460fa60612f6964", "dirty pan-e80bc3f91737777f", "mug14-6f3c3fe67d4f83e1", "dirty plate1-b054beee5b63b90f", "dirty plate2-3d924d8074966e28", "cheese grater3-77677ab65ebb2e6a", "chopping board3-1abc2c84e321fcde", "mug15-da692c59f2043650", "mug13-0ad8ff6d30b226a9", "lid3-d793c3ec1efb7e6b", "glass3-9dc1c5b3fa216efe", "glass4-ffdc5feb83e3f50f", "spoon3-54fb87555b92d1b4", "plate10-cce905c4306a3e52", "plate12-446944685e823e8c", "dough scraper-6a7e9cef9abd18cb", "colander2-c2638075ddf22750", "glass5-c822cebc7060d90d", "plate11-8fd71216e19302f5", "towel-f29807236c61d991", "towel2-9d8f46337e93b48b", "knife11-00e4cdd01c1a5a07", "pot of double cream-d6acda522fef2f03", "second spoon-b55b10750f9affc2", "oven glove2-575a91f4240fef16", "spoon-bfd5ddf2bd38b4b6", "plate13-426eb6c9bf399566" ], "reasoning": "1. I look for the time the user ladles chocolate batter into the ramekins (both ramekins-c8d27ede390ee6d5), finding the time around 13:43. Here, the user uses a spoon (spoon4-0193ffc50496414a) to ladle the batter out of a glass bowl (glass bowl-9ead0b968b087d4d).\n2. At this point, I see that there are 2 ramekins (both ramekins-c8d27ede390ee6d5) beside the bowl (glass bowl-9ead0b968b087d4d). One (ramekin1-8d01db3c12a50615) is further to the right of the user than the other (ramekin2-9bec6b34ffbade1d).\n3. At 13:43, the user begins to ladle chocolate batter into the closest ramekin (ramekin2-9bec6b34ffbade1d).\n4. At 14:04, the user begins to ladle chocolate batter into the ramekin which is furthest right (ramekin1-8d01db3c12a50615).\n5. The amount of time that elapses between 13:43 and 14:04 is 21 seconds.\n6. Therefore, the answer is 21 seconds.", "referenced_object_masks": [ "glass bowl-9ead0b968b087d4d", "ramekin2-9bec6b34ffbade1d", "ramekin1-8d01db3c12a50615", "spoon4-0193ffc50496414a", "both ramekins-c8d27ede390ee6d5" ] }, { "key": "P05-20240425-201444:8da7ce96649ed7006ca43a88684b8516bc9de002", "video_id": "P05-20240425-201444", "question": "What objects do I bring to my mouth?", "answer_choice_0": "Spoon", "answer_choice_1": "Spoon, Spatula, and Glass", "answer_choice_2": "Spoon and Spatula", "answer_choice_3": "Spatula", "answer_choice_4": "Glass", "answer_id": 1, "answer": "Spoon, Spatula, and Glass", "question_type": [ "Egocentric & Spatial Perception", "Action Recognition", "Object Recognition" ], "objects_of_interest": [ "scales-e5f94ff2ef748b44", "box of eggs-6f5955c996970c42", "glass bowl-9ead0b968b087d4d", "bowl-51070525ae5442f0", "egg1-792f0d07fe1965c9", "egg2-69f6dfec4c0366f8", "pan-415bb186d7c64fbb", "pot with handle-6b147ad932dd52f7", "lid-92522a1d6b473649", "mayonnaise-b4f81e16b9298126", "sugar bag-6b8295a9c890e045", "jar of chocolate buttons-a43fe58a7fc17d24", "knife-b576cfca96e4ec89", "lid of butter dish-6b58b7e6471533ee", "butter-443c0eca4519eb04", "mixer-39c3d84478cf12b4", "spatula-f17a414fbeb3a42d", "mixer blades-0f441d7a6c363730", "brush-01d619fc593fd587", "pot of flour-9d0764bbfcfc3fb5", "lid of pot of flour-0d9005db79ccadc9", "measuring spoon-7caab10dcdfb71b7", "ramekin1-8d01db3c12a50615", "ramekin2-9bec6b34ffbade1d", "spoon4-0193ffc50496414a", "both ramekins-c8d27ede390ee6d5", "oven glove-5a63c9197e7afae9", "shelf-a99e3b90f8d1bbfd", "phone-803857b825b8f51a", "mixer blade2-267d3f6f3eba6b99", "knife2-b33172dac6ba454e", "knife3-1c44a6a26dde1a07", "knife4-7a69c2bb038c8669", "spoon2-86c02722e7cea3d3", "knife5-d0069a2816a47845", "knife8-0dad6bda5d775d06", "knife7-d51e27b5053cba96", "knife6-1890e40baa4edeb5", "knife9-22560d351a0b0c50", "plate-37e72c60571ded88", "spoons-2d7a6bb9912c3b73", "spoon new-09072dcce260d73a", "spoons2-b38ba311c4c442e7", "glasses-5bd69b9526fd3d51", "glasses2-664b175a05040dcd", "stack of bowls-c0a809568ede7bc4", "one bowl-9eae99aed09da7de", "mug-73b5e0386ec68dec", "mug2-24e1861fb6b0fca8", "mug3-9cc6c399ef2c6e39", "mug4-35fff1b757f5b99f", "mug5-768afe13050e82c3", "mug6-8939a76897476123", "mug7-d505bad238e63f06", "mug8-1b25f5e5178f15b4", "mug9-94f6dd826e47f1b2", "mug10-77a0b62fa81bd84b", "mug11-018f3398e343825d", "chopping boards-66ebb8ef6ec3ad67", "chopping board2-5be0d9bb802d0f6d", "cheese graters-b4429764d99d8e7c", "cheese graters2-ed3b72b1f101a313", "plate2-ee08f4693dc8bbba", "plate new-4abe44aabed425f4", "plate3-e4d0047b5e5d7307", "plate4-af1e0144ba9babf3", "plate5-eb97acea80f79441", "plate6-f668c22ad7efce0b", "plate7-42dec1104fe5315e", "plate8-287dc6766f016846", "bowl3-1e9a912fe544b3b9", "bowl4-a8572ac2ae26311f", "pan2-502f7630bb110a5b", "pan lid-15de218aeb0c7847", "pan3-aa7f43cc002d3768", "knife10-6395d93998b8e8f6", "plate9-78d8adba0a04ef0f", "dirty knife-be36f0ad60eaa3b7", "dirty spoons-a02a7c960b14f0ba", "wooden spoon-b205fd5dfc8bc8c9", "colander-d5ad959f49bf0481", "fork-ed6ecf29d0dd02c3", "dirty bowl-b460fa60612f6964", "dirty pan-e80bc3f91737777f", "mug14-6f3c3fe67d4f83e1", "dirty plate1-b054beee5b63b90f", "dirty plate2-3d924d8074966e28", "cheese grater3-77677ab65ebb2e6a", "chopping board3-1abc2c84e321fcde", "mug15-da692c59f2043650", "mug13-0ad8ff6d30b226a9", "lid3-d793c3ec1efb7e6b", "glass3-9dc1c5b3fa216efe", "glass4-ffdc5feb83e3f50f", "spoon3-54fb87555b92d1b4", "plate10-cce905c4306a3e52", "plate12-446944685e823e8c", "dough scraper-6a7e9cef9abd18cb", "colander2-c2638075ddf22750", "glass5-c822cebc7060d90d", "plate11-8fd71216e19302f5", "towel-f29807236c61d991", "towel2-9d8f46337e93b48b", "knife11-00e4cdd01c1a5a07", "pot of double cream-d6acda522fef2f03", "second spoon-b55b10750f9affc2", "oven glove2-575a91f4240fef16", "spoon-bfd5ddf2bd38b4b6", "plate13-426eb6c9bf399566" ], "reasoning": "1. I look for objects the user brings to their mouth.\n2. I notice at 14:42 and 14:46 the user brings a spoon (spoon4-0193ffc50496414a) to their mouth, in addition to another spoon (spoon3-54fb87555b92d1b4) at 26:47.\n3. I notice from 15:06-15:12 that the user brings a spatula (spatula-f17a414fbeb3a42d) to their mouth several times.\n4. I also notice at 21:29 that the user brings a glass of water (glass4-ffdc5feb83e3f50f) to their mouth.\n5. The answer to the question, therefore, is that the user brings a spoon (spoon4-0193ffc50496414a, spoon3-54fb87555b92d1b4), a spatula (spatula-f17a414fbeb3a42d), and a glass (glass4-ffdc5feb83e3f50f) to their mouth.", "referenced_object_masks": [ "spoon4-0193ffc50496414a", "spoon3-54fb87555b92d1b4", "spatula-f17a414fbeb3a42d", "glass4-ffdc5feb83e3f50f" ] }, { "key": "P05-20240425-201444:93ad0aab1f4267c4977c570141ae9d063a03e200", "video_id": "P05-20240425-201444", "question": "Where is the trash can in proximity to the counter that I cleaned the scraps from?", "answer_choice_0": "On the same side of the room, far left corner", "answer_choice_1": "On the opposite side of the room, to the far left", "answer_choice_2": "On the opposite side of the room, far right corner", "answer_choice_3": "On the same side of the room, in the center", "answer_choice_4": "On the same side of the room, far right corner", "answer_id": 1, "answer": "On the opposite side of the room, to the far left", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition" ], "objects_of_interest": [ "scales-e5f94ff2ef748b44", "box of eggs-6f5955c996970c42", "glass bowl-9ead0b968b087d4d", "bowl-51070525ae5442f0", "egg1-792f0d07fe1965c9", "egg2-69f6dfec4c0366f8", "pan-415bb186d7c64fbb", "pot with handle-6b147ad932dd52f7", "lid-92522a1d6b473649", "mayonnaise-b4f81e16b9298126", "sugar bag-6b8295a9c890e045", "jar of chocolate buttons-a43fe58a7fc17d24", "knife-b576cfca96e4ec89", "lid of butter dish-6b58b7e6471533ee", "butter-443c0eca4519eb04", "mixer-39c3d84478cf12b4", "spatula-f17a414fbeb3a42d", "mixer blades-0f441d7a6c363730", "brush-01d619fc593fd587", "pot of flour-9d0764bbfcfc3fb5", "lid of pot of flour-0d9005db79ccadc9", "measuring spoon-7caab10dcdfb71b7", "ramekin1-8d01db3c12a50615", "ramekin2-9bec6b34ffbade1d", "spoon4-0193ffc50496414a", "both ramekins-c8d27ede390ee6d5", "oven glove-5a63c9197e7afae9", "shelf-a99e3b90f8d1bbfd", "phone-803857b825b8f51a", "mixer blade2-267d3f6f3eba6b99", "knife2-b33172dac6ba454e", "knife3-1c44a6a26dde1a07", "knife4-7a69c2bb038c8669", "spoon2-86c02722e7cea3d3", "knife5-d0069a2816a47845", "knife8-0dad6bda5d775d06", "knife7-d51e27b5053cba96", "knife6-1890e40baa4edeb5", "knife9-22560d351a0b0c50", "plate-37e72c60571ded88", "spoons-2d7a6bb9912c3b73", "spoon new-09072dcce260d73a", "spoons2-b38ba311c4c442e7", "glasses-5bd69b9526fd3d51", "glasses2-664b175a05040dcd", "stack of bowls-c0a809568ede7bc4", "one bowl-9eae99aed09da7de", "mug-73b5e0386ec68dec", "mug2-24e1861fb6b0fca8", "mug3-9cc6c399ef2c6e39", "mug4-35fff1b757f5b99f", "mug5-768afe13050e82c3", "mug6-8939a76897476123", "mug7-d505bad238e63f06", "mug8-1b25f5e5178f15b4", "mug9-94f6dd826e47f1b2", "mug10-77a0b62fa81bd84b", "mug11-018f3398e343825d", "chopping boards-66ebb8ef6ec3ad67", "chopping board2-5be0d9bb802d0f6d", "cheese graters-b4429764d99d8e7c", "cheese graters2-ed3b72b1f101a313", "plate2-ee08f4693dc8bbba", "plate new-4abe44aabed425f4", "plate3-e4d0047b5e5d7307", "plate4-af1e0144ba9babf3", "plate5-eb97acea80f79441", "plate6-f668c22ad7efce0b", "plate7-42dec1104fe5315e", "plate8-287dc6766f016846", "bowl3-1e9a912fe544b3b9", "bowl4-a8572ac2ae26311f", "pan2-502f7630bb110a5b", "pan lid-15de218aeb0c7847", "pan3-aa7f43cc002d3768", "knife10-6395d93998b8e8f6", "plate9-78d8adba0a04ef0f", "dirty knife-be36f0ad60eaa3b7", "dirty spoons-a02a7c960b14f0ba", "wooden spoon-b205fd5dfc8bc8c9", "colander-d5ad959f49bf0481", "fork-ed6ecf29d0dd02c3", "dirty bowl-b460fa60612f6964", "dirty pan-e80bc3f91737777f", "mug14-6f3c3fe67d4f83e1", "dirty plate1-b054beee5b63b90f", "dirty plate2-3d924d8074966e28", "cheese grater3-77677ab65ebb2e6a", "chopping board3-1abc2c84e321fcde", "mug15-da692c59f2043650", "mug13-0ad8ff6d30b226a9", "lid3-d793c3ec1efb7e6b", "glass3-9dc1c5b3fa216efe", "glass4-ffdc5feb83e3f50f", "spoon3-54fb87555b92d1b4", "plate10-cce905c4306a3e52", "plate12-446944685e823e8c", "dough scraper-6a7e9cef9abd18cb", "colander2-c2638075ddf22750", "glass5-c822cebc7060d90d", "plate11-8fd71216e19302f5", "towel-f29807236c61d991", "towel2-9d8f46337e93b48b", "knife11-00e4cdd01c1a5a07", "pot of double cream-d6acda522fef2f03", "second spoon-b55b10750f9affc2", "oven glove2-575a91f4240fef16", "spoon-bfd5ddf2bd38b4b6", "plate13-426eb6c9bf399566" ], "reasoning": "1. At 00:19, I saw the user cleaning scraps off the counter near the window.\n2. At 00:22, I saw the user walk across the kitchen, going toward the direction of the refrigerator, which is on the far left wall.\n3. At 00:25, I saw the user open the trash can with their foot.\n4. The trash can is near the refrigerator, in the far left corner.", "referenced_object_masks": [] }, { "key": "P05-20240425-201444:9951b1e31b73576dc1a4097ae5dccf29039ea2b7", "video_id": "P05-20240425-201444", "question": "Which, if any, of the first 3 unique ingredients I prepare for the dish are composed of a primary color?", "answer_choice_0": "Butter", "answer_choice_1": "Eggs, butter", "answer_choice_2": "Eggs", "answer_choice_3": "Eggs, butter, lemon", "answer_choice_4": "None", "answer_id": 1, "answer": "Eggs, butter", "question_type": [ "Object Recognition", "Temporal Reasoning", "State Changes", "Action Recognition" ], "objects_of_interest": [ "scales-e5f94ff2ef748b44", "box of eggs-6f5955c996970c42", "glass bowl-9ead0b968b087d4d", "bowl-51070525ae5442f0", "egg1-792f0d07fe1965c9", "egg2-69f6dfec4c0366f8", "pan-415bb186d7c64fbb", "pot with handle-6b147ad932dd52f7", "lid-92522a1d6b473649", "mayonnaise-b4f81e16b9298126", "sugar bag-6b8295a9c890e045", "jar of chocolate buttons-a43fe58a7fc17d24", "knife-b576cfca96e4ec89", "lid of butter dish-6b58b7e6471533ee", "butter-443c0eca4519eb04", "mixer-39c3d84478cf12b4", "spatula-f17a414fbeb3a42d", "mixer blades-0f441d7a6c363730", "brush-01d619fc593fd587", "pot of flour-9d0764bbfcfc3fb5", "lid of pot of flour-0d9005db79ccadc9", "measuring spoon-7caab10dcdfb71b7", "ramekin1-8d01db3c12a50615", "ramekin2-9bec6b34ffbade1d", "spoon4-0193ffc50496414a", "both ramekins-c8d27ede390ee6d5", "oven glove-5a63c9197e7afae9", "shelf-a99e3b90f8d1bbfd", "phone-803857b825b8f51a", "mixer blade2-267d3f6f3eba6b99", "knife2-b33172dac6ba454e", "knife3-1c44a6a26dde1a07", "knife4-7a69c2bb038c8669", "spoon2-86c02722e7cea3d3", "knife5-d0069a2816a47845", "knife8-0dad6bda5d775d06", "knife7-d51e27b5053cba96", "knife6-1890e40baa4edeb5", "knife9-22560d351a0b0c50", "plate-37e72c60571ded88", "spoons-2d7a6bb9912c3b73", "spoon new-09072dcce260d73a", "spoons2-b38ba311c4c442e7", "glasses-5bd69b9526fd3d51", "glasses2-664b175a05040dcd", "stack of bowls-c0a809568ede7bc4", "one bowl-9eae99aed09da7de", "mug-73b5e0386ec68dec", "mug2-24e1861fb6b0fca8", "mug3-9cc6c399ef2c6e39", "mug4-35fff1b757f5b99f", "mug5-768afe13050e82c3", "mug6-8939a76897476123", "mug7-d505bad238e63f06", "mug8-1b25f5e5178f15b4", "mug9-94f6dd826e47f1b2", "mug10-77a0b62fa81bd84b", "mug11-018f3398e343825d", "chopping boards-66ebb8ef6ec3ad67", "chopping board2-5be0d9bb802d0f6d", "cheese graters-b4429764d99d8e7c", "cheese graters2-ed3b72b1f101a313", "plate2-ee08f4693dc8bbba", "plate new-4abe44aabed425f4", "plate3-e4d0047b5e5d7307", "plate4-af1e0144ba9babf3", "plate5-eb97acea80f79441", "plate6-f668c22ad7efce0b", "plate7-42dec1104fe5315e", "plate8-287dc6766f016846", "bowl3-1e9a912fe544b3b9", "bowl4-a8572ac2ae26311f", "pan2-502f7630bb110a5b", "pan lid-15de218aeb0c7847", "pan3-aa7f43cc002d3768", "knife10-6395d93998b8e8f6", "plate9-78d8adba0a04ef0f", "dirty knife-be36f0ad60eaa3b7", "dirty spoons-a02a7c960b14f0ba", "wooden spoon-b205fd5dfc8bc8c9", "colander-d5ad959f49bf0481", "fork-ed6ecf29d0dd02c3", "dirty bowl-b460fa60612f6964", "dirty pan-e80bc3f91737777f", "mug14-6f3c3fe67d4f83e1", "dirty plate1-b054beee5b63b90f", "dirty plate2-3d924d8074966e28", "cheese grater3-77677ab65ebb2e6a", "chopping board3-1abc2c84e321fcde", "mug15-da692c59f2043650", "mug13-0ad8ff6d30b226a9", "lid3-d793c3ec1efb7e6b", "glass3-9dc1c5b3fa216efe", "glass4-ffdc5feb83e3f50f", "spoon3-54fb87555b92d1b4", "plate10-cce905c4306a3e52", "plate12-446944685e823e8c", "dough scraper-6a7e9cef9abd18cb", "colander2-c2638075ddf22750", "glass5-c822cebc7060d90d", "plate11-8fd71216e19302f5", "towel-f29807236c61d991", "towel2-9d8f46337e93b48b", "knife11-00e4cdd01c1a5a07", "pot of double cream-d6acda522fef2f03", "second spoon-b55b10750f9affc2", "oven glove2-575a91f4240fef16", "spoon-bfd5ddf2bd38b4b6", "plate13-426eb6c9bf399566" ], "reasoning": "1. I look for the first ingredient the user prepares that is eventually used in the final dish.\n2. I find it at 01:05, an egg with a yellow yolk (egg1-792f0d07fe1965c9). This egg (egg1-792f0d07fe1965c9) is used in the chocolate batter ladled into the ramekins (both ramekins-c8d27ede390ee6d5) from 13:43-14:41 and cooked from 15:26-25:13.\n3. I look for the second ingredient the user prepares that is eventually used in the final dish.\n4. I find it at 02:51: chocolate buttons (jar of chocolate buttons-a43fe58a7fc17d24). These buttons are eventually used in the chocolate batter cooked from 15:26-25:13.\n5. I look for the third ingredient the user prepares that is eventually used in the final dish.\n6. I find it from 03:11-03:12, slices of yellow butter (butter-443c0eca4519eb04). These slices (butter-443c0eca4519eb04) are used to make the chocolate batter that is cooked from 15:26-25:13.\n7. Butter and egg yolks are both yellow, a primary color. The primary colors are red, yellow, and blue.\n8. Therefore, the answer is eggs and butter.", "referenced_object_masks": [ "butter-443c0eca4519eb04", "egg1-792f0d07fe1965c9", "jar of chocolate buttons-a43fe58a7fc17d24", "both ramekins-c8d27ede390ee6d5" ] }, { "key": "P05-20240425-201444:9b97b242117f4db24705e5dac986284c01af1eb0", "video_id": "P05-20240425-201444", "question": "What is the difference between the amount of time that I spent blending the eggs with a mixer and the amount of time that I spent heating up the chocolate and butter on the stove?", "answer_choice_0": "4 minutes and 28 seconds", "answer_choice_1": "2 minutes", "answer_choice_2": "1 minute and 18 seconds", "answer_choice_3": "42 seconds", "answer_choice_4": "32 seconds", "answer_id": 3, "answer": "42 seconds", "question_type": [ "Event Occurence", "Object Recognition", "Situational Awareness", "Action Recognition" ], "objects_of_interest": [ "scales-e5f94ff2ef748b44", "box of eggs-6f5955c996970c42", "glass bowl-9ead0b968b087d4d", "bowl-51070525ae5442f0", "egg1-792f0d07fe1965c9", "egg2-69f6dfec4c0366f8", "pan-415bb186d7c64fbb", "pot with handle-6b147ad932dd52f7", "lid-92522a1d6b473649", "mayonnaise-b4f81e16b9298126", "sugar bag-6b8295a9c890e045", "jar of chocolate buttons-a43fe58a7fc17d24", "knife-b576cfca96e4ec89", "lid of butter dish-6b58b7e6471533ee", "butter-443c0eca4519eb04", "mixer-39c3d84478cf12b4", "spatula-f17a414fbeb3a42d", "mixer blades-0f441d7a6c363730", "brush-01d619fc593fd587", "pot of flour-9d0764bbfcfc3fb5", "lid of pot of flour-0d9005db79ccadc9", "measuring spoon-7caab10dcdfb71b7", "ramekin1-8d01db3c12a50615", "ramekin2-9bec6b34ffbade1d", "spoon4-0193ffc50496414a", "both ramekins-c8d27ede390ee6d5", "oven glove-5a63c9197e7afae9", "shelf-a99e3b90f8d1bbfd", "phone-803857b825b8f51a", "mixer blade2-267d3f6f3eba6b99", "knife2-b33172dac6ba454e", "knife3-1c44a6a26dde1a07", "knife4-7a69c2bb038c8669", "spoon2-86c02722e7cea3d3", "knife5-d0069a2816a47845", "knife8-0dad6bda5d775d06", "knife7-d51e27b5053cba96", "knife6-1890e40baa4edeb5", "knife9-22560d351a0b0c50", "plate-37e72c60571ded88", "spoons-2d7a6bb9912c3b73", "spoon new-09072dcce260d73a", "spoons2-b38ba311c4c442e7", "glasses-5bd69b9526fd3d51", "glasses2-664b175a05040dcd", "stack of bowls-c0a809568ede7bc4", "one bowl-9eae99aed09da7de", "mug-73b5e0386ec68dec", "mug2-24e1861fb6b0fca8", "mug3-9cc6c399ef2c6e39", "mug4-35fff1b757f5b99f", "mug5-768afe13050e82c3", "mug6-8939a76897476123", "mug7-d505bad238e63f06", "mug8-1b25f5e5178f15b4", "mug9-94f6dd826e47f1b2", "mug10-77a0b62fa81bd84b", "mug11-018f3398e343825d", "chopping boards-66ebb8ef6ec3ad67", "chopping board2-5be0d9bb802d0f6d", "cheese graters-b4429764d99d8e7c", "cheese graters2-ed3b72b1f101a313", "plate2-ee08f4693dc8bbba", "plate new-4abe44aabed425f4", "plate3-e4d0047b5e5d7307", "plate4-af1e0144ba9babf3", "plate5-eb97acea80f79441", "plate6-f668c22ad7efce0b", "plate7-42dec1104fe5315e", "plate8-287dc6766f016846", "bowl3-1e9a912fe544b3b9", "bowl4-a8572ac2ae26311f", "pan2-502f7630bb110a5b", "pan lid-15de218aeb0c7847", "pan3-aa7f43cc002d3768", "knife10-6395d93998b8e8f6", "plate9-78d8adba0a04ef0f", "dirty knife-be36f0ad60eaa3b7", "dirty spoons-a02a7c960b14f0ba", "wooden spoon-b205fd5dfc8bc8c9", "colander-d5ad959f49bf0481", "fork-ed6ecf29d0dd02c3", "dirty bowl-b460fa60612f6964", "dirty pan-e80bc3f91737777f", "mug14-6f3c3fe67d4f83e1", "dirty plate1-b054beee5b63b90f", "dirty plate2-3d924d8074966e28", "cheese grater3-77677ab65ebb2e6a", "chopping board3-1abc2c84e321fcde", "mug15-da692c59f2043650", "mug13-0ad8ff6d30b226a9", "lid3-d793c3ec1efb7e6b", "glass3-9dc1c5b3fa216efe", "glass4-ffdc5feb83e3f50f", "spoon3-54fb87555b92d1b4", "plate10-cce905c4306a3e52", "plate12-446944685e823e8c", "dough scraper-6a7e9cef9abd18cb", "colander2-c2638075ddf22750", "glass5-c822cebc7060d90d", "plate11-8fd71216e19302f5", "towel-f29807236c61d991", "towel2-9d8f46337e93b48b", "knife11-00e4cdd01c1a5a07", "pot of double cream-d6acda522fef2f03", "second spoon-b55b10750f9affc2", "oven glove2-575a91f4240fef16", "spoon-bfd5ddf2bd38b4b6", "plate13-426eb6c9bf399566" ], "reasoning": "1. I began by looking for when the user both heated up the chocolate and butter (butter-443c0eca4519eb04) on the stove and when the egg yolks (OBJs) were blended with a mixer (mixer-39c3d84478cf12b4). At 03:34 I saw the user put a pot (pot with handle-6b147ad932dd52f7) with the chocolate and butter on the stove to heat.\n2. At 04:38 the user takes the pot (pot with handle-6b147ad932dd52f7) with the chocolate and butter off of the stove to mix and cool.\n3. At 05:30, I saw the user begin mixing the egg yolks (OBJs) with the mixer (mixer-39c3d84478cf12b4).\n4. At 07:20, the user stopped mixing in a circular motion and helped the mixer (mixer-39c3d84478cf12b4) idle in the middle of the bowl (bowl3-1e9a912fe544b3b9).\n5. At 07:30, I saw the user stop mixing and raise the mixer (mixer-39c3d84478cf12b4) out of the bowl (bowl3-1e9a912fe544b3b9).\n6. At 07:48 I saw the user put the pot (pot with handle-6b147ad932dd52f7) with the chocolate and butter back on the stove to heat a little more, taking it off at 08:02. \n7. Next, I calculated the total amount of time of both the mixing and the heating. I subtracted 5:30 from 7:30 to get a total mixing time of 2 minutes. I then subtracted 03:34 from 04:38 and 07:48 from 08:02 to get a total heating time of 01:04 + 00:14 = 1 minute and 18 seconds.\n7. Lastly, I calculated the difference by subtracting 01:18 from 02:00 to get 42 seconds.", "referenced_object_masks": [ "bowl3-1e9a912fe544b3b9", "butter-443c0eca4519eb04", "mixer-39c3d84478cf12b4", "pot with handle-6b147ad932dd52f7" ] }, { "key": "P05-20240425-201444:cdec7353b301d9b4c6f889b92480038f703bd23b", "video_id": "P05-20240425-201444", "question": "What is true of the weights of the second and third ingredients I weigh, as they show up on my kitchen scale?", "answer_choice_0": "The weights are the same", "answer_choice_1": "The weights are 3 units apart", "answer_choice_2": "The weights are 10 units apart", "answer_choice_3": "The weights are each in the 60s", "answer_choice_4": "The weights are each in the 50s", "answer_id": 4, "answer": "The weights are each in the 50s", "question_type": [ "Numerical Reasoning", "Object Recognition", "Reading", "Counting" ], "objects_of_interest": [ "scales-e5f94ff2ef748b44", "box of eggs-6f5955c996970c42", "glass bowl-9ead0b968b087d4d", "bowl-51070525ae5442f0", "egg1-792f0d07fe1965c9", "egg2-69f6dfec4c0366f8", "pan-415bb186d7c64fbb", "pot with handle-6b147ad932dd52f7", "lid-92522a1d6b473649", "mayonnaise-b4f81e16b9298126", "sugar bag-6b8295a9c890e045", "jar of chocolate buttons-a43fe58a7fc17d24", "knife-b576cfca96e4ec89", "lid of butter dish-6b58b7e6471533ee", "butter-443c0eca4519eb04", "mixer-39c3d84478cf12b4", "spatula-f17a414fbeb3a42d", "mixer blades-0f441d7a6c363730", "brush-01d619fc593fd587", "pot of flour-9d0764bbfcfc3fb5", "lid of pot of flour-0d9005db79ccadc9", "measuring spoon-7caab10dcdfb71b7", "ramekin1-8d01db3c12a50615", "ramekin2-9bec6b34ffbade1d", "spoon4-0193ffc50496414a", "both ramekins-c8d27ede390ee6d5", "oven glove-5a63c9197e7afae9", "shelf-a99e3b90f8d1bbfd", "phone-803857b825b8f51a", "mixer blade2-267d3f6f3eba6b99", "knife2-b33172dac6ba454e", "knife3-1c44a6a26dde1a07", "knife4-7a69c2bb038c8669", "spoon2-86c02722e7cea3d3", "knife5-d0069a2816a47845", "knife8-0dad6bda5d775d06", "knife7-d51e27b5053cba96", "knife6-1890e40baa4edeb5", "knife9-22560d351a0b0c50", "plate-37e72c60571ded88", "spoons-2d7a6bb9912c3b73", "spoon new-09072dcce260d73a", "spoons2-b38ba311c4c442e7", "glasses-5bd69b9526fd3d51", "glasses2-664b175a05040dcd", "stack of bowls-c0a809568ede7bc4", "one bowl-9eae99aed09da7de", "mug-73b5e0386ec68dec", "mug2-24e1861fb6b0fca8", "mug3-9cc6c399ef2c6e39", "mug4-35fff1b757f5b99f", "mug5-768afe13050e82c3", "mug6-8939a76897476123", "mug7-d505bad238e63f06", "mug8-1b25f5e5178f15b4", "mug9-94f6dd826e47f1b2", "mug10-77a0b62fa81bd84b", "mug11-018f3398e343825d", "chopping boards-66ebb8ef6ec3ad67", "chopping board2-5be0d9bb802d0f6d", "cheese graters-b4429764d99d8e7c", "cheese graters2-ed3b72b1f101a313", "plate2-ee08f4693dc8bbba", "plate new-4abe44aabed425f4", "plate3-e4d0047b5e5d7307", "plate4-af1e0144ba9babf3", "plate5-eb97acea80f79441", "plate6-f668c22ad7efce0b", "plate7-42dec1104fe5315e", "plate8-287dc6766f016846", "bowl3-1e9a912fe544b3b9", "bowl4-a8572ac2ae26311f", "pan2-502f7630bb110a5b", "pan lid-15de218aeb0c7847", "pan3-aa7f43cc002d3768", "knife10-6395d93998b8e8f6", "plate9-78d8adba0a04ef0f", "dirty knife-be36f0ad60eaa3b7", "dirty spoons-a02a7c960b14f0ba", "wooden spoon-b205fd5dfc8bc8c9", "colander-d5ad959f49bf0481", "fork-ed6ecf29d0dd02c3", "dirty bowl-b460fa60612f6964", "dirty pan-e80bc3f91737777f", "mug14-6f3c3fe67d4f83e1", "dirty plate1-b054beee5b63b90f", "dirty plate2-3d924d8074966e28", "cheese grater3-77677ab65ebb2e6a", "chopping board3-1abc2c84e321fcde", "mug15-da692c59f2043650", "mug13-0ad8ff6d30b226a9", "lid3-d793c3ec1efb7e6b", "glass3-9dc1c5b3fa216efe", "glass4-ffdc5feb83e3f50f", "spoon3-54fb87555b92d1b4", "plate10-cce905c4306a3e52", "plate12-446944685e823e8c", "dough scraper-6a7e9cef9abd18cb", "colander2-c2638075ddf22750", "glass5-c822cebc7060d90d", "plate11-8fd71216e19302f5", "towel-f29807236c61d991", "towel2-9d8f46337e93b48b", "knife11-00e4cdd01c1a5a07", "pot of double cream-d6acda522fef2f03", "second spoon-b55b10750f9affc2", "oven glove2-575a91f4240fef16", "spoon-bfd5ddf2bd38b4b6", "plate13-426eb6c9bf399566" ], "reasoning": "1. I look for the first ingredient weighed and I find it at 03:01: chocolate buttons from a jar (jar of chocolate buttons-a43fe58a7fc17d24) weighing 61 units.\n2. I look for the second ingredient weighed and find it at 03:36: slices of butter (butter-443c0eca4519eb04) weighing 58 units.\n3. Finally, I look for the third ingredient weighed and find it at 03:48: a glass bowl filled with 1 yolk (egg1-792f0d07fe1965c9) and 1 yolk with the white still attached (egg2-69f6dfec4c0366f8), weighing 53 units.\n4. The butter (butter-443c0eca4519eb04) and the eggs (egg1-792f0d07fe1965c9) are each measured in the 50s, and are not equal, in the 60s, or 3 or 10 units apart.\n5. Therefore, the answer is that each weight is in the 50s.", "referenced_object_masks": [ "butter-443c0eca4519eb04", "jar of chocolate buttons-a43fe58a7fc17d24", "egg1-792f0d07fe1965c9", "egg2-69f6dfec4c0366f8" ] }, { "key": "P05-20240425-201444:dcd8b89afcc9694fa0820ecacaa29b4e180b30b6", "video_id": "P05-20240425-201444", "question": "What contents in the refrigerator match the contents on the table?", "answer_choice_0": "Apples", "answer_choice_1": "Eggs", "answer_choice_2": "Butter", "answer_choice_3": "Bananas", "answer_choice_4": "Cucumbers", "answer_id": 3, "answer": "Bananas", "question_type": [ "Object Recognition", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "scales-e5f94ff2ef748b44", "box of eggs-6f5955c996970c42", "glass bowl-9ead0b968b087d4d", "bowl-51070525ae5442f0", "egg1-792f0d07fe1965c9", "egg2-69f6dfec4c0366f8", "pan-415bb186d7c64fbb", "pot with handle-6b147ad932dd52f7", "lid-92522a1d6b473649", "mayonnaise-b4f81e16b9298126", "sugar bag-6b8295a9c890e045", "jar of chocolate buttons-a43fe58a7fc17d24", "knife-b576cfca96e4ec89", "lid of butter dish-6b58b7e6471533ee", "butter-443c0eca4519eb04", "mixer-39c3d84478cf12b4", "spatula-f17a414fbeb3a42d", "mixer blades-0f441d7a6c363730", "brush-01d619fc593fd587", "pot of flour-9d0764bbfcfc3fb5", "lid of pot of flour-0d9005db79ccadc9", "measuring spoon-7caab10dcdfb71b7", "ramekin1-8d01db3c12a50615", "ramekin2-9bec6b34ffbade1d", "spoon4-0193ffc50496414a", "both ramekins-c8d27ede390ee6d5", "oven glove-5a63c9197e7afae9", "shelf-a99e3b90f8d1bbfd", "phone-803857b825b8f51a", "mixer blade2-267d3f6f3eba6b99", "knife2-b33172dac6ba454e", "knife3-1c44a6a26dde1a07", "knife4-7a69c2bb038c8669", "spoon2-86c02722e7cea3d3", "knife5-d0069a2816a47845", "knife8-0dad6bda5d775d06", "knife7-d51e27b5053cba96", "knife6-1890e40baa4edeb5", "knife9-22560d351a0b0c50", "plate-37e72c60571ded88", "spoons-2d7a6bb9912c3b73", "spoon new-09072dcce260d73a", "spoons2-b38ba311c4c442e7", "glasses-5bd69b9526fd3d51", "glasses2-664b175a05040dcd", "stack of bowls-c0a809568ede7bc4", "one bowl-9eae99aed09da7de", "mug-73b5e0386ec68dec", "mug2-24e1861fb6b0fca8", "mug3-9cc6c399ef2c6e39", "mug4-35fff1b757f5b99f", "mug5-768afe13050e82c3", "mug6-8939a76897476123", "mug7-d505bad238e63f06", "mug8-1b25f5e5178f15b4", "mug9-94f6dd826e47f1b2", "mug10-77a0b62fa81bd84b", "mug11-018f3398e343825d", "chopping boards-66ebb8ef6ec3ad67", "chopping board2-5be0d9bb802d0f6d", "cheese graters-b4429764d99d8e7c", "cheese graters2-ed3b72b1f101a313", "plate2-ee08f4693dc8bbba", "plate new-4abe44aabed425f4", "plate3-e4d0047b5e5d7307", "plate4-af1e0144ba9babf3", "plate5-eb97acea80f79441", "plate6-f668c22ad7efce0b", "plate7-42dec1104fe5315e", "plate8-287dc6766f016846", "bowl3-1e9a912fe544b3b9", "bowl4-a8572ac2ae26311f", "pan2-502f7630bb110a5b", "pan lid-15de218aeb0c7847", "pan3-aa7f43cc002d3768", "knife10-6395d93998b8e8f6", "plate9-78d8adba0a04ef0f", "dirty knife-be36f0ad60eaa3b7", "dirty spoons-a02a7c960b14f0ba", "wooden spoon-b205fd5dfc8bc8c9", "colander-d5ad959f49bf0481", "fork-ed6ecf29d0dd02c3", "dirty bowl-b460fa60612f6964", "dirty pan-e80bc3f91737777f", "mug14-6f3c3fe67d4f83e1", "dirty plate1-b054beee5b63b90f", "dirty plate2-3d924d8074966e28", "cheese grater3-77677ab65ebb2e6a", "chopping board3-1abc2c84e321fcde", "mug15-da692c59f2043650", "mug13-0ad8ff6d30b226a9", "lid3-d793c3ec1efb7e6b", "glass3-9dc1c5b3fa216efe", "glass4-ffdc5feb83e3f50f", "spoon3-54fb87555b92d1b4", "plate10-cce905c4306a3e52", "plate12-446944685e823e8c", "dough scraper-6a7e9cef9abd18cb", "colander2-c2638075ddf22750", "glass5-c822cebc7060d90d", "plate11-8fd71216e19302f5", "towel-f29807236c61d991", "towel2-9d8f46337e93b48b", "knife11-00e4cdd01c1a5a07", "pot of double cream-d6acda522fef2f03", "second spoon-b55b10750f9affc2", "oven glove2-575a91f4240fef16", "spoon-bfd5ddf2bd38b4b6", "plate13-426eb6c9bf399566" ], "reasoning": "1. At 00:27, I saw the user open the refrigerator . The contents in the refrigerator , especially on the first shelf, have a clearer view at 00:28.\n2. There is a bowl (bowl3-1e9a912fe544b3b9), bananas , eggs and blocks of cheese .\n3. At 02:31, the camera has a full view of the table. There are bananas , lemons , a plant and a bowl of fruits .\n4. The refrigerator and table has bananas in common.", "referenced_object_masks": [ "bowl3-1e9a912fe544b3b9" ] }, { "key": "P05-20240425-201444:e71361c87c13e9e864a51cd868913c8d1e45c767", "video_id": "P05-20240425-201444", "question": "How many cooking steps are there before the finished product?", "answer_choice_0": "12 steps", "answer_choice_1": "16 steps", "answer_choice_2": "10 steps", "answer_choice_3": "15 steps", "answer_choice_4": "8 steps", "answer_id": 1, "answer": "16 steps", "question_type": [ "Counting", "Situational Awareness" ], "objects_of_interest": [ "scales-e5f94ff2ef748b44", "box of eggs-6f5955c996970c42", "glass bowl-9ead0b968b087d4d", "bowl-51070525ae5442f0", "egg1-792f0d07fe1965c9", "egg2-69f6dfec4c0366f8", "pan-415bb186d7c64fbb", "pot with handle-6b147ad932dd52f7", "lid-92522a1d6b473649", "mayonnaise-b4f81e16b9298126", "sugar bag-6b8295a9c890e045", "jar of chocolate buttons-a43fe58a7fc17d24", "knife-b576cfca96e4ec89", "lid of butter dish-6b58b7e6471533ee", "butter-443c0eca4519eb04", "mixer-39c3d84478cf12b4", "spatula-f17a414fbeb3a42d", "mixer blades-0f441d7a6c363730", "brush-01d619fc593fd587", "pot of flour-9d0764bbfcfc3fb5", "lid of pot of flour-0d9005db79ccadc9", "measuring spoon-7caab10dcdfb71b7", "ramekin1-8d01db3c12a50615", "ramekin2-9bec6b34ffbade1d", "spoon4-0193ffc50496414a", "both ramekins-c8d27ede390ee6d5", "oven glove-5a63c9197e7afae9", "shelf-a99e3b90f8d1bbfd", "phone-803857b825b8f51a", "mixer blade2-267d3f6f3eba6b99", "knife2-b33172dac6ba454e", "knife3-1c44a6a26dde1a07", "knife4-7a69c2bb038c8669", "spoon2-86c02722e7cea3d3", "knife5-d0069a2816a47845", "knife8-0dad6bda5d775d06", "knife7-d51e27b5053cba96", "knife6-1890e40baa4edeb5", "knife9-22560d351a0b0c50", "plate-37e72c60571ded88", "spoons-2d7a6bb9912c3b73", "spoon new-09072dcce260d73a", "spoons2-b38ba311c4c442e7", "glasses-5bd69b9526fd3d51", "glasses2-664b175a05040dcd", "stack of bowls-c0a809568ede7bc4", "one bowl-9eae99aed09da7de", "mug-73b5e0386ec68dec", "mug2-24e1861fb6b0fca8", "mug3-9cc6c399ef2c6e39", "mug4-35fff1b757f5b99f", "mug5-768afe13050e82c3", "mug6-8939a76897476123", "mug7-d505bad238e63f06", "mug8-1b25f5e5178f15b4", "mug9-94f6dd826e47f1b2", "mug10-77a0b62fa81bd84b", "mug11-018f3398e343825d", "chopping boards-66ebb8ef6ec3ad67", "chopping board2-5be0d9bb802d0f6d", "cheese graters-b4429764d99d8e7c", "cheese graters2-ed3b72b1f101a313", "plate2-ee08f4693dc8bbba", "plate new-4abe44aabed425f4", "plate3-e4d0047b5e5d7307", "plate4-af1e0144ba9babf3", "plate5-eb97acea80f79441", "plate6-f668c22ad7efce0b", "plate7-42dec1104fe5315e", "plate8-287dc6766f016846", "bowl3-1e9a912fe544b3b9", "bowl4-a8572ac2ae26311f", "pan2-502f7630bb110a5b", "pan lid-15de218aeb0c7847", "pan3-aa7f43cc002d3768", "knife10-6395d93998b8e8f6", "plate9-78d8adba0a04ef0f", "dirty knife-be36f0ad60eaa3b7", "dirty spoons-a02a7c960b14f0ba", "wooden spoon-b205fd5dfc8bc8c9", "colander-d5ad959f49bf0481", "fork-ed6ecf29d0dd02c3", "dirty bowl-b460fa60612f6964", "dirty pan-e80bc3f91737777f", "mug14-6f3c3fe67d4f83e1", "dirty plate1-b054beee5b63b90f", "dirty plate2-3d924d8074966e28", "cheese grater3-77677ab65ebb2e6a", "chopping board3-1abc2c84e321fcde", "mug15-da692c59f2043650", "mug13-0ad8ff6d30b226a9", "lid3-d793c3ec1efb7e6b", "glass3-9dc1c5b3fa216efe", "glass4-ffdc5feb83e3f50f", "spoon3-54fb87555b92d1b4", "plate10-cce905c4306a3e52", "plate12-446944685e823e8c", "dough scraper-6a7e9cef9abd18cb", "colander2-c2638075ddf22750", "glass5-c822cebc7060d90d", "plate11-8fd71216e19302f5", "towel-f29807236c61d991", "towel2-9d8f46337e93b48b", "knife11-00e4cdd01c1a5a07", "pot of double cream-d6acda522fef2f03", "second spoon-b55b10750f9affc2", "oven glove2-575a91f4240fef16", "spoon-bfd5ddf2bd38b4b6", "plate13-426eb6c9bf399566" ], "reasoning": "1. At 01:07, I saw the user separate egg yolk from the egg white.\n2. At 01:27, I saw the user crack the last egg (egg1-792f0d07fe1965c9) into a bowl (bowl3-1e9a912fe544b3b9).\n3. At 02:55, I saw the user getting the exact measurements of what seems to be chocolate.\n4. At 3:15, I saw the user add butter (butter-443c0eca4519eb04) to the same pot that the chocolate was in.\n5. At 04:03, I saw the user add sugar to the bowl with the eggs (egg1-792f0d07fe1965c9).\n6. At 03:53, I saw the user begin to heat the chocolate and butter (butter-443c0eca4519eb04).\n7. At 05:30, the user began to mix the sugar and eggs (egg1-792f0d07fe1965c9) with the mixer. \n8. At 08:23, I saw the user begin to add the melted chocolate and butter (butter-443c0eca4519eb04) to the bowl (bowl3-1e9a912fe544b3b9) with the mixed eggs (egg1-792f0d07fe1965c9) and sugar.\n9. At 09:51, I saw the user add another ingredient to the bowl (bowl3-1e9a912fe544b3b9).\n10. At 10:03, I saw the user begin to mix all the contents in the bowl (bowl3-1e9a912fe544b3b9).\n11. At 11:51, I saw the user begin to rub butter around the inside of a white ramekin (ramekin1-8d01db3c12a50615).\n12. At 12:28, I saw the user add more of the previous ingredient to the bottom of the ramekin (ramekin1-8d01db3c12a50615) and shook them together.\n13. At 13:45, I saw the user begin to spoon out portions of the mixed ingredients into the ramekins (ramekin1-8d01db3c12a50615), filling them both half way.\n14. At 15:28, I saw the user place both ramekins (ramekin1-8d01db3c12a50615) in the oven on the bottom rack.\n15. The user baked the dish for about 10 minutes.\n16. At 25:19, I saw the user retrieve the ramekins (ramekin1-8d01db3c12a50615) from the oven, which completes the task.\n17. There were 16 steps to complete this baking task.", "referenced_object_masks": [ "bowl3-1e9a912fe544b3b9", "egg1-792f0d07fe1965c9", "ramekin1-8d01db3c12a50615", "butter-443c0eca4519eb04" ] }, { "key": "P05-20240425-201444:f81a733833a84b2af145788afaaf6c38d55b407b", "video_id": "P05-20240425-201444", "question": "How many egg yolks were used versus egg whites in the same clear bowl?", "answer_choice_0": "3 egg yolks, 2 egg whites", "answer_choice_1": "1 egg yolk, 2 egg whites", "answer_choice_2": "4 egg yolks, no egg whites", "answer_choice_3": "2 egg yolks, 1 egg white", "answer_choice_4": "5 egg yolks, 3 egg whites", "answer_id": 3, "answer": "2 egg yolks, 1 egg white", "question_type": [ "Counting", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "scales-e5f94ff2ef748b44", "box of eggs-6f5955c996970c42", "glass bowl-9ead0b968b087d4d", "bowl-51070525ae5442f0", "egg1-792f0d07fe1965c9", "egg2-69f6dfec4c0366f8", "pan-415bb186d7c64fbb", "pot with handle-6b147ad932dd52f7", "lid-92522a1d6b473649", "mayonnaise-b4f81e16b9298126", "sugar bag-6b8295a9c890e045", "jar of chocolate buttons-a43fe58a7fc17d24", "knife-b576cfca96e4ec89", "lid of butter dish-6b58b7e6471533ee", "butter-443c0eca4519eb04", "mixer-39c3d84478cf12b4", "spatula-f17a414fbeb3a42d", "mixer blades-0f441d7a6c363730", "brush-01d619fc593fd587", "pot of flour-9d0764bbfcfc3fb5", "lid of pot of flour-0d9005db79ccadc9", "measuring spoon-7caab10dcdfb71b7", "ramekin1-8d01db3c12a50615", "ramekin2-9bec6b34ffbade1d", "spoon4-0193ffc50496414a", "both ramekins-c8d27ede390ee6d5", "oven glove-5a63c9197e7afae9", "shelf-a99e3b90f8d1bbfd", "phone-803857b825b8f51a", "mixer blade2-267d3f6f3eba6b99", "knife2-b33172dac6ba454e", "knife3-1c44a6a26dde1a07", "knife4-7a69c2bb038c8669", "spoon2-86c02722e7cea3d3", "knife5-d0069a2816a47845", "knife8-0dad6bda5d775d06", "knife7-d51e27b5053cba96", "knife6-1890e40baa4edeb5", "knife9-22560d351a0b0c50", "plate-37e72c60571ded88", "spoons-2d7a6bb9912c3b73", "spoon new-09072dcce260d73a", "spoons2-b38ba311c4c442e7", "glasses-5bd69b9526fd3d51", "glasses2-664b175a05040dcd", "stack of bowls-c0a809568ede7bc4", "one bowl-9eae99aed09da7de", "mug-73b5e0386ec68dec", "mug2-24e1861fb6b0fca8", "mug3-9cc6c399ef2c6e39", "mug4-35fff1b757f5b99f", "mug5-768afe13050e82c3", "mug6-8939a76897476123", "mug7-d505bad238e63f06", "mug8-1b25f5e5178f15b4", "mug9-94f6dd826e47f1b2", "mug10-77a0b62fa81bd84b", "mug11-018f3398e343825d", "chopping boards-66ebb8ef6ec3ad67", "chopping board2-5be0d9bb802d0f6d", "cheese graters-b4429764d99d8e7c", "cheese graters2-ed3b72b1f101a313", "plate2-ee08f4693dc8bbba", "plate new-4abe44aabed425f4", "plate3-e4d0047b5e5d7307", "plate4-af1e0144ba9babf3", "plate5-eb97acea80f79441", "plate6-f668c22ad7efce0b", "plate7-42dec1104fe5315e", "plate8-287dc6766f016846", "bowl3-1e9a912fe544b3b9", "bowl4-a8572ac2ae26311f", "pan2-502f7630bb110a5b", "pan lid-15de218aeb0c7847", "pan3-aa7f43cc002d3768", "knife10-6395d93998b8e8f6", "plate9-78d8adba0a04ef0f", "dirty knife-be36f0ad60eaa3b7", "dirty spoons-a02a7c960b14f0ba", "wooden spoon-b205fd5dfc8bc8c9", "colander-d5ad959f49bf0481", "fork-ed6ecf29d0dd02c3", "dirty bowl-b460fa60612f6964", "dirty pan-e80bc3f91737777f", "mug14-6f3c3fe67d4f83e1", "dirty plate1-b054beee5b63b90f", "dirty plate2-3d924d8074966e28", "cheese grater3-77677ab65ebb2e6a", "chopping board3-1abc2c84e321fcde", "mug15-da692c59f2043650", "mug13-0ad8ff6d30b226a9", "lid3-d793c3ec1efb7e6b", "glass3-9dc1c5b3fa216efe", "glass4-ffdc5feb83e3f50f", "spoon3-54fb87555b92d1b4", "plate10-cce905c4306a3e52", "plate12-446944685e823e8c", "dough scraper-6a7e9cef9abd18cb", "colander2-c2638075ddf22750", "glass5-c822cebc7060d90d", "plate11-8fd71216e19302f5", "towel-f29807236c61d991", "towel2-9d8f46337e93b48b", "knife11-00e4cdd01c1a5a07", "pot of double cream-d6acda522fef2f03", "second spoon-b55b10750f9affc2", "oven glove2-575a91f4240fef16", "spoon-bfd5ddf2bd38b4b6", "plate13-426eb6c9bf399566" ], "reasoning": "1. At 00:50, I saw the user retrieve a bowl (bowl3-1e9a912fe544b3b9) of contents from the refrigerator .\n2. At 01:01, I saw the user crack the first egg (egg1-792f0d07fe1965c9).\n3. At 01:07, I saw the user carefully separate the egg yolk from the egg white .\n4. At 01:27, I saw the user crack another egg (egg1-792f0d07fe1965c9), entirely in the same bowl (bowl3-1e9a912fe544b3b9) with the yolk.\n5. The user used 2 egg yolks and 1 egg white in the same bowl (bowl3-1e9a912fe544b3b9).", "referenced_object_masks": [ "bowl3-1e9a912fe544b3b9", "egg1-792f0d07fe1965c9" ] }, { "key": "P05-20240426-085333:15602c116ae4957ee9a5955f31b75051d3503471", "video_id": "P05-20240426-085333", "question": "What is the mathematical difference between the weight scale's two readings from after I poured the blue bag of cereal and after I poured the milk into the blue and white bowl?", "answer_choice_0": "0.0", "answer_choice_1": "5.7", "answer_choice_2": "17.7", "answer_choice_3": "29.1", "answer_choice_4": "23.4", "answer_id": 2, "answer": "17.7", "question_type": [ "Action Recognition", "Counting", "Egocentric & Spatial Perception", "Numerical Reasoning", "Object Recognition", "Reading", "State Changes", "Temporal Reasoning" ], "objects_of_interest": [ "chair-e92822c080dea09a", "chair2-4e538dac6ac0bd92", "bowl-ef6e49df5dbc5dee", "scale-42188d023098d5dd", "porridge-ecda708125e3519c", "bag-f9a2b599685c7cc5", "raisins-d9fdca4e2bd01a7a", "milk-ff50c737ccb37ed2", "glass of water-8b47fc5e594b7b50", "glass2-df9451e31b17fe59", "ladle-500ded6fe7ce69bd", "dirty cutlery-edba569c036d9cc7", "dirty cutlery2-cd4e627fe429ca6d", "dirty cutlery3-1ea3fb52433af7ae", "metal bowl-ee1dbeb86d9ddabf", "bowl2-edeeb647d026f70f", "cutlery-ec947c15bcd16270", "plate-a59636f30a73ef51", "butter packet-57ba8f21ff7b1752", "dirty butter lid-4ec97546fbad68bb", "metal bowl2-4fee056760472760", "dirty plastic bowl-9fd800f91bc6e441", "lid-0f56ce10e3df5ccd", "dirty chopping board-64d2905f3b53b134", "larger chopping board-635a02f23fa701e0", "cup-5ef8ecd9ac856132", "cup2-149f51975b290552", "cup3-2116a2918290855c", "ramekin-cc9f78fdcb75201a", "cup4-c01e5aced1b8b4f0", "plate2-53e412b8fec7edd9", "plate4-1334a673c9246c4b", "plate3-8b89929472b321c8", "chopping board-11eae8c8dfe65edd", "knife1-5514283aa2a057f5", "knife2-42ba60f61dc82f04", "spoon-d7fe84d24dcbfe36", "bananas-855aa0d163ee0199", "one banana-3fcbe0b842a47b13", "knife3-3073f91c57b874ca" ], "reasoning": "1. I saw the user pour the blue bag of cereal (porridge-ecda708125e3519c) into the blue and white bowl (bowl-ef6e49df5dbc5dee) on the scale (scale-42188d023098d5dd) at 00:43-00:51.\n2. After the user poured the blue bag of cereal (porridge-ecda708125e3519c) at 00:43-00:51, I saw the reading of the scale (scale-42188d023098d5dd) read \"5.7\" at 00:51.\n3. I saw the user pour the milk (milk-ff50c737ccb37ed2) into the blue and white bowl (bowl-ef6e49df5dbc5dee) of cereal on the scale (scale-42188d023098d5dd) at 01:03-01:10.\n4. After the user poured the milk (milk-ff50c737ccb37ed2) at 01:03-01:10, I saw the reading of the scale (scale-42188d023098d5dd) read \"23.4\" at 01:10.\n5. I calculated 23.4-5.7=17.7. Therefore, the mathematical difference between the two readings of the weight scale (scale-42188d023098d5dd) from after the user poured the blue bag of cereal (porridge-ecda708125e3519c) and after they poured the milk (milk-ff50c737ccb37ed2) into the blue and white bowl (bowl-ef6e49df5dbc5dee) is 17.7.", "referenced_object_masks": [ "porridge-ecda708125e3519c", "scale-42188d023098d5dd", "bowl-ef6e49df5dbc5dee", "milk-ff50c737ccb37ed2" ] }, { "key": "P05-20240426-085333:2e3f3e04d506ff3d9679048f17a1c8139cf7a4fb", "video_id": "P05-20240426-085333", "question": "If I turned to face the sunlight instead of opening the dishwasher, where would the microwave have been in relation to me?", "answer_choice_0": "Behind the user", "answer_choice_1": "In front of the user", "answer_choice_2": "Above the user", "answer_choice_3": "To the left of the user", "answer_choice_4": "To the right of the user", "answer_id": 0, "answer": "Behind the user", "question_type": [ "Situational Awareness", "Action Recognition", "Temporal Reasoning", "Egocentric & Spatial Perception", "Counterfactual", "Object Recognition" ], "objects_of_interest": [ "chair-e92822c080dea09a", "chair2-4e538dac6ac0bd92", "bowl-ef6e49df5dbc5dee", "scale-42188d023098d5dd", "porridge-ecda708125e3519c", "bag-f9a2b599685c7cc5", "raisins-d9fdca4e2bd01a7a", "milk-ff50c737ccb37ed2", "glass of water-8b47fc5e594b7b50", "glass2-df9451e31b17fe59", "ladle-500ded6fe7ce69bd", "dirty cutlery-edba569c036d9cc7", "dirty cutlery2-cd4e627fe429ca6d", "dirty cutlery3-1ea3fb52433af7ae", "metal bowl-ee1dbeb86d9ddabf", "bowl2-edeeb647d026f70f", "cutlery-ec947c15bcd16270", "plate-a59636f30a73ef51", "butter packet-57ba8f21ff7b1752", "dirty butter lid-4ec97546fbad68bb", "metal bowl2-4fee056760472760", "dirty plastic bowl-9fd800f91bc6e441", "lid-0f56ce10e3df5ccd", "dirty chopping board-64d2905f3b53b134", "larger chopping board-635a02f23fa701e0", "cup-5ef8ecd9ac856132", "cup2-149f51975b290552", "cup3-2116a2918290855c", "ramekin-cc9f78fdcb75201a", "cup4-c01e5aced1b8b4f0", "plate2-53e412b8fec7edd9", "plate4-1334a673c9246c4b", "plate3-8b89929472b321c8", "chopping board-11eae8c8dfe65edd", "knife1-5514283aa2a057f5", "knife2-42ba60f61dc82f04", "spoon-d7fe84d24dcbfe36", "bananas-855aa0d163ee0199", "one banana-3fcbe0b842a47b13", "knife3-3073f91c57b874ca" ], "reasoning": "1. I watched the video and saw the user open the dishwasher at 01:41.\n2. Before the user opened the dishwasher at 01:41, I saw the user turn leftward from the microwave towards the dishwasher at 01:37-01:41.\n3. Since the user turned leftward from the microwave towards the dishwasher at 01:37-01:41, the microwave was consequently to the right of the user when they opened the dishwasher at 01:41.\n4. As the user opened the dishwasher at 01:41, I saw sunlight sourcing from an off-screen window to the left of the user and the dishwasher.\n5. Therefore, if the user turned leftward to face the sunlight instead of opening the dishwasher, the microwave would have been behind the user.", "referenced_object_masks": [] }, { "key": "P05-20240426-085333:59a6a7837663def18bd189c07d7df225282755e4", "video_id": "P05-20240426-085333", "question": "What was the second item I moved to the white tabletop?", "answer_choice_0": "A white plate of cutlery", "answer_choice_1": "A black cereal bowl", "answer_choice_2": "A blue and white scale", "answer_choice_3": "A black weight scale", "answer_choice_4": "A blue and white bowl", "answer_id": 3, "answer": "A black weight scale", "question_type": [ "Counting", "Action Recognition", "Situational Awareness", "Object Recognition" ], "objects_of_interest": [ "chair-e92822c080dea09a", "chair2-4e538dac6ac0bd92", "bowl-ef6e49df5dbc5dee", "scale-42188d023098d5dd", "porridge-ecda708125e3519c", "bag-f9a2b599685c7cc5", "raisins-d9fdca4e2bd01a7a", "milk-ff50c737ccb37ed2", "glass of water-8b47fc5e594b7b50", "glass2-df9451e31b17fe59", "ladle-500ded6fe7ce69bd", "dirty cutlery-edba569c036d9cc7", "dirty cutlery2-cd4e627fe429ca6d", "dirty cutlery3-1ea3fb52433af7ae", "metal bowl-ee1dbeb86d9ddabf", "bowl2-edeeb647d026f70f", "cutlery-ec947c15bcd16270", "plate-a59636f30a73ef51", "butter packet-57ba8f21ff7b1752", "dirty butter lid-4ec97546fbad68bb", "metal bowl2-4fee056760472760", "dirty plastic bowl-9fd800f91bc6e441", "lid-0f56ce10e3df5ccd", "dirty chopping board-64d2905f3b53b134", "larger chopping board-635a02f23fa701e0", "cup-5ef8ecd9ac856132", "cup2-149f51975b290552", "cup3-2116a2918290855c", "ramekin-cc9f78fdcb75201a", "cup4-c01e5aced1b8b4f0", "plate2-53e412b8fec7edd9", "plate4-1334a673c9246c4b", "plate3-8b89929472b321c8", "chopping board-11eae8c8dfe65edd", "knife1-5514283aa2a057f5", "knife2-42ba60f61dc82f04", "spoon-d7fe84d24dcbfe36", "bananas-855aa0d163ee0199", "one banana-3fcbe0b842a47b13", "knife3-3073f91c57b874ca" ], "reasoning": "1. I watched the video and counted each item the user moved to the white tabletop.\n2. At 00:13, I saw the user move the first item to the white tabletop, which was a blue and white bowl (bowl-ef6e49df5dbc5dee).\n3. At 00:26, I saw the user move the second item to the white tabletop, which was a black weight scale (scale-42188d023098d5dd).\n4. Therefore, the second item the user moved to the white tabletop was a black weight scale (scale-42188d023098d5dd).", "referenced_object_masks": [ "scale-42188d023098d5dd", "bowl-ef6e49df5dbc5dee" ] }, { "key": "P05-20240426-085333:a1b82a1bdb44b9b4b904a7a5703fd65b09ae25b9", "video_id": "P05-20240426-085333", "question": "If I doubled the number of items I poured into the blue and white bowl before putting it in the microwave, how many items would I have poured?", "answer_choice_0": "8", "answer_choice_1": "4", "answer_choice_2": "0", "answer_choice_3": "2", "answer_choice_4": "6", "answer_id": 4, "answer": "6", "question_type": [ "Object Recognition", "Action Recognition", "Counting", "Egocentric & Spatial Perception", "Situational Awareness", "Temporal Reasoning", "Numerical Reasoning", "Counterfactual" ], "objects_of_interest": [ "chair-e92822c080dea09a", "chair2-4e538dac6ac0bd92", "bowl-ef6e49df5dbc5dee", "scale-42188d023098d5dd", "porridge-ecda708125e3519c", "bag-f9a2b599685c7cc5", "raisins-d9fdca4e2bd01a7a", "milk-ff50c737ccb37ed2", "glass of water-8b47fc5e594b7b50", "glass2-df9451e31b17fe59", "ladle-500ded6fe7ce69bd", "dirty cutlery-edba569c036d9cc7", "dirty cutlery2-cd4e627fe429ca6d", "dirty cutlery3-1ea3fb52433af7ae", "metal bowl-ee1dbeb86d9ddabf", "bowl2-edeeb647d026f70f", "cutlery-ec947c15bcd16270", "plate-a59636f30a73ef51", "butter packet-57ba8f21ff7b1752", "dirty butter lid-4ec97546fbad68bb", "metal bowl2-4fee056760472760", "dirty plastic bowl-9fd800f91bc6e441", "lid-0f56ce10e3df5ccd", "dirty chopping board-64d2905f3b53b134", "larger chopping board-635a02f23fa701e0", "cup-5ef8ecd9ac856132", "cup2-149f51975b290552", "cup3-2116a2918290855c", "ramekin-cc9f78fdcb75201a", "cup4-c01e5aced1b8b4f0", "plate2-53e412b8fec7edd9", "plate4-1334a673c9246c4b", "plate3-8b89929472b321c8", "chopping board-11eae8c8dfe65edd", "knife1-5514283aa2a057f5", "knife2-42ba60f61dc82f04", "spoon-d7fe84d24dcbfe36", "bananas-855aa0d163ee0199", "one banana-3fcbe0b842a47b13", "knife3-3073f91c57b874ca" ], "reasoning": "1. At 00:43-00:51, I saw the user pour the first item into the bowl (bowl-ef6e49df5dbc5dee), which was a blue bag of cereal (porridge-ecda708125e3519c). At 01:03-01:10, I saw the user pour the second item into the bowl (bowl-ef6e49df5dbc5dee), which was milk (milk-ff50c737ccb37ed2). At 01:22-01:24, I saw the user pour the third item into the bowl (bowl-ef6e49df5dbc5dee), which was a glass of water (glass of water-8b47fc5e594b7b50).\n2. At 01:34, I saw the user put the bowl (bowl-ef6e49df5dbc5dee) into the microwave.\n3. I counted each item the user poured into the bowl (bowl-ef6e49df5dbc5dee) before putting it in the microwave at 01:34, calculating a total of 3.\n4. I calculated 3*2=6. Therefore, if the user doubled the number of items they poured into the blue and white bowl (bowl-ef6e49df5dbc5dee) before putting it in the microwave, they would have poured 6 items.", "referenced_object_masks": [ "glass of water-8b47fc5e594b7b50", "porridge-ecda708125e3519c", "bowl-ef6e49df5dbc5dee", "milk-ff50c737ccb37ed2" ] }, { "key": "P05-20240426-085333:d71c8cb06054b1dbf09cb141ecbebb37f6f6ded0", "video_id": "P05-20240426-085333", "question": "Where did I move the metal bowl to?", "answer_choice_0": "Into the dishwasher's top shelf", "answer_choice_1": "Into the dishwasher's bottom shelf", "answer_choice_2": "Into the sink full of dishes", "answer_choice_3": "Onto the kitchen countertop", "answer_choice_4": "Onto the white tabletop", "answer_id": 1, "answer": "Into the dishwasher's bottom shelf", "question_type": [ "Object Recognition", "Action Recognition", "State Changes", "Situational Awareness", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "chair-e92822c080dea09a", "chair2-4e538dac6ac0bd92", "bowl-ef6e49df5dbc5dee", "scale-42188d023098d5dd", "porridge-ecda708125e3519c", "bag-f9a2b599685c7cc5", "raisins-d9fdca4e2bd01a7a", "milk-ff50c737ccb37ed2", "glass of water-8b47fc5e594b7b50", "glass2-df9451e31b17fe59", "ladle-500ded6fe7ce69bd", "dirty cutlery-edba569c036d9cc7", "dirty cutlery2-cd4e627fe429ca6d", "dirty cutlery3-1ea3fb52433af7ae", "metal bowl-ee1dbeb86d9ddabf", "bowl2-edeeb647d026f70f", "cutlery-ec947c15bcd16270", "plate-a59636f30a73ef51", "butter packet-57ba8f21ff7b1752", "dirty butter lid-4ec97546fbad68bb", "metal bowl2-4fee056760472760", "dirty plastic bowl-9fd800f91bc6e441", "lid-0f56ce10e3df5ccd", "dirty chopping board-64d2905f3b53b134", "larger chopping board-635a02f23fa701e0", "cup-5ef8ecd9ac856132", "cup2-149f51975b290552", "cup3-2116a2918290855c", "ramekin-cc9f78fdcb75201a", "cup4-c01e5aced1b8b4f0", "plate2-53e412b8fec7edd9", "plate4-1334a673c9246c4b", "plate3-8b89929472b321c8", "chopping board-11eae8c8dfe65edd", "knife1-5514283aa2a057f5", "knife2-42ba60f61dc82f04", "spoon-d7fe84d24dcbfe36", "bananas-855aa0d163ee0199", "one banana-3fcbe0b842a47b13", "knife3-3073f91c57b874ca" ], "reasoning": "1. I watched the video and found the metal bowl (metal bowl-ee1dbeb86d9ddabf) at 02:00.\n2. I saw the user move the metal bowl (metal bowl-ee1dbeb86d9ddabf) into the bottom shelf of the dishwasher at 02:05.\n3. Therefore, the user moved the metal bowl (metal bowl-ee1dbeb86d9ddabf) into the dishwasher's bottom shelf.", "referenced_object_masks": [ "metal bowl-ee1dbeb86d9ddabf" ] }, { "key": "P05-20240426-091011:0f29185554aa1348ea9564a1a38f82865230fd57", "video_id": "P05-20240426-091011", "question": "Where did I move the blue and white bowl of food to before grabbing the bag of food from the top of the refrigerator?", "answer_choice_0": "Onto a tabletop", "answer_choice_1": "Onto a countertop", "answer_choice_2": "On top of the refrigerator", "answer_choice_3": "Into a microwave", "answer_choice_4": "Into the refrigerator", "answer_id": 3, "answer": "Into a microwave", "question_type": [ "Action Recognition", "Situational Awareness", "Temporal Reasoning", "Object Recognition", "State Changes" ], "objects_of_interest": [ "spoon-29d30e37fcd63270", "plate-ce042b6d105191de", "bag1-068969a760fbe26a", "bag2-6d72ddb1030b040d", "bag3-82e7fb2b30853e20" ], "reasoning": "1. I watched the video and saw the user grab a bag of food (bag1-068969a760fbe26a) from the top of a refrigerator at 00:42.\n2. Before 00:42, I found the blue and white bowl of food (plate-ce042b6d105191de) at 00:06.\n3. At 00:19-00:20, I saw the user move the blue and white bowl of food (plate-ce042b6d105191de) from a tabletop and into a microwave .\n4. Therefore, the user moved the blue and white bowl of food (plate-ce042b6d105191de) into a microwave before grabbing the bag of food (bag1-068969a760fbe26a) from the top of the refrigerator.", "referenced_object_masks": [ "plate-ce042b6d105191de", "bag1-068969a760fbe26a" ] }, { "key": "P05-20240426-091011:159a8c1c6d6a2456997173d9f7ed10b12b1f7f4e", "video_id": "P05-20240426-091011", "question": "What did I use to stir the bowl of food for the second time?", "answer_choice_0": "A fork", "answer_choice_1": "A spoon", "answer_choice_2": "A finger", "answer_choice_3": "A spork", "answer_choice_4": "A knife", "answer_id": 1, "answer": "A spoon", "question_type": [ "Object Recognition", "Action Recognition", "Counting", "Cause and Effect" ], "objects_of_interest": [ "spoon-29d30e37fcd63270", "plate-ce042b6d105191de", "bag1-068969a760fbe26a", "bag2-6d72ddb1030b040d", "bag3-82e7fb2b30853e20" ], "reasoning": "1. I watched the video and found the bowl of food (plate-ce042b6d105191de) at 00:06.\n2. I saw the user stir the bowl of food (plate-ce042b6d105191de) for the first time at 00:07-00:16.\n3. I saw the user stir the bowl of food (plate-ce042b6d105191de) for the second time at 01:43-01:47.\n4. At 01:43-01:47, I saw the user stir the bowl of food (plate-ce042b6d105191de) with a spoon (spoon-29d30e37fcd63270).\n5. Therefore, the user used a spoon (spoon-29d30e37fcd63270) to stir the bowl of food (plate-ce042b6d105191de) for the second time.", "referenced_object_masks": [ "plate-ce042b6d105191de", "spoon-29d30e37fcd63270" ] }, { "key": "P05-20240426-091011:5a85e75ee18690e9f6a93ddd48bffc29c370bad8", "video_id": "P05-20240426-091011", "question": "How many green bags of food did I grab from the top of the refrigerator after putting the bowl of food in the microwave?", "answer_choice_0": "2", "answer_choice_1": "3", "answer_choice_2": "4", "answer_choice_3": "1", "answer_choice_4": "5", "answer_id": 3, "answer": "1", "question_type": [ "Action Recognition", "Situational Awareness", "Temporal Reasoning", "Egocentric & Spatial Perception", "Object Recognition", "Counting" ], "objects_of_interest": [ "spoon-29d30e37fcd63270", "plate-ce042b6d105191de", "bag1-068969a760fbe26a", "bag2-6d72ddb1030b040d", "bag3-82e7fb2b30853e20" ], "reasoning": "1. I watched the video and saw the user put the bowl of food (plate-ce042b6d105191de) in the microwave at 00:20.\n2. After 00:20, I saw the user grab the first bag of food (bag1-068969a760fbe26a) from the top of the refrigerator at 00:42, but it was primarily white.\n3. At 00:46, I saw the user grab the second bag of food (bag2-6d72ddb1030b040d) from the top of the refrigerator, but it was primarily transparent.\n4. At 00:51, I saw the user grab the third and final bag of food (bag3-82e7fb2b30853e20) from the top of the refrigerator, which was primarily green.\n5. Since the user did not grab anymore bags of food from the top of the refrigerator after 00:51, I counted each green bag of food (bag3-82e7fb2b30853e20) the user grabbed from the top of the refrigerator, calculating a total of 1.\n6. Therefore, the user grabbed 1 green bag of food (bag3-82e7fb2b30853e20) from the top of the refrigerator after putting the bowl of food (plate-ce042b6d105191de) in the microwave .", "referenced_object_masks": [ "bag3-82e7fb2b30853e20", "plate-ce042b6d105191de", "bag1-068969a760fbe26a", "bag2-6d72ddb1030b040d" ] }, { "key": "P05-20240426-091011:933cd385a278da8fc7781f81d0bfddbab8f1f74e", "video_id": "P05-20240426-091011", "question": "Where was the microwave in relation to me when I was standing in the dining area and facing a window?", "answer_choice_0": "In front", "answer_choice_1": "To the right", "answer_choice_2": "Overhead", "answer_choice_3": "Behind", "answer_choice_4": "To the left", "answer_id": 3, "answer": "Behind", "question_type": [ "Object Recognition", "Situational Awareness", "Temporal Reasoning", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "spoon-29d30e37fcd63270", "plate-ce042b6d105191de", "bag1-068969a760fbe26a", "bag2-6d72ddb1030b040d", "bag3-82e7fb2b30853e20" ], "reasoning": "1. I watched the video and found the microwave at 00:20.\n2. At 01:00, I saw the user standing in the dining area and facing a window, but the microwave was not visible.\n3. Before 01:00, I saw the user facing the microwave before turning around, away from it, at 00:58-00:59.\n4. Therefore, the microwave was behind the user when they were standing in the dining area and facing a window.", "referenced_object_masks": [] }, { "key": "P05-20240426-091011:df2a38f4daf732e5215ae48d81248b8f7c91c7ed", "video_id": "P05-20240426-091011", "question": "If I counted the number of times I stirred the food in the blue and white bowl in clockwise and counterclockwise directions before placing it in the microwave, then rounded each number to the nearest multiple of 3, what simplified ratio would I get of clockwise stirs to counterclockwise stirs?", "answer_choice_0": "2:5", "answer_choice_1": "2:3", "answer_choice_2": "7:3", "answer_choice_3": "4:3", "answer_choice_4": "5:2", "answer_id": 3, "answer": "4:3", "question_type": [ "Object Recognition", "Action Recognition", "Situational Awareness", "Temporal Reasoning", "Egocentric & Spatial Perception", "Counting", "Numerical Reasoning", "Counterfactual" ], "objects_of_interest": [ "spoon-29d30e37fcd63270", "plate-ce042b6d105191de", "bag1-068969a760fbe26a", "bag2-6d72ddb1030b040d", "bag3-82e7fb2b30853e20" ], "reasoning": "1. I watched the video and found the blue and white bowl of food (plate-ce042b6d105191de) at 00:06.\n2. I saw the user put the blue and white bowl of food (plate-ce042b6d105191de) in the microwave at 00:20.\n3. Before 00:20, I saw the user stir the blue and white bowl of food (plate-ce042b6d105191de) at 00:07-00:16.\n4. I counted the amount of times the user stirred the blue and white bowl of food (plate-ce042b6d105191de) clockwise at 00:07-00:16, calculating a total of 13.\n5. I counted the amount of times the user stirred the blue and white bowl of food (plate-ce042b6d105191de) counterclockwise at 00:07-00:16, calculating a total of 9.\n6. I then rounded each number to the closest multiple of three, and got 12 clockwise stirs vs. 9 counterclockwise stirs. \n7. I simplified the ratio for a final answer of 4:3 clockwise stirs vs. counterclockwise stirs.", "referenced_object_masks": [ "plate-ce042b6d105191de" ] }, { "key": "P05-20240426-092000:1661661bcdabf1d6866c3387cdc0d9147c62ba58", "video_id": "P05-20240426-092000", "question": "How many perfectly circular objects can be seen while I look down at the table while the bowl is on the scales with only porridge in it?", "answer_choice_0": "4", "answer_choice_1": "0", "answer_choice_2": "2", "answer_choice_3": "3", "answer_choice_4": "1", "answer_id": 3, "answer": "3", "question_type": [ "Egocentric & Spatial Perception", "Counting", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "spoon-d3eb941379f623cd", "porridge bowl-9cdb618a1c4120fe", "plate-4107414cbd633a9e", "packet of raisins-039d581b287d8cbd", "plastic bag-8c25764414d2f011", "second package of raisins-11550f72e170badb", "pecan nuts-6ae12fde6718cdb1", "scales-550e5d82a358f01c", "porridge bowl on plate-6a4ff2e77f0d2976" ], "reasoning": "1. I watched for the user to put the bowl (porridge bowl-9cdb618a1c4120fe) on the scale (scales-550e5d82a358f01c) when there is only porridge in it, which happens from 00:10-00:21.\n2. As the user looks down at the table, the circular objects in their field of vision are the bowl (porridge bowl-9cdb618a1c4120fe), the plate (plate-4107414cbd633a9e), and the silver trash can , making three circular objects.", "referenced_object_masks": [ "porridge bowl-9cdb618a1c4120fe", "plate-4107414cbd633a9e", "scales-550e5d82a358f01c" ] }, { "key": "P05-20240426-092000:274c9dbb8be9bfa657f80b77cc96ab599fbd5a61", "video_id": "P05-20240426-092000", "question": "What do I not do with the spoon?", "answer_choice_0": "Pick it up from the plate", "answer_choice_1": "Push bananas into the bowl", "answer_choice_2": "Lick off porridge", "answer_choice_3": "Wash it", "answer_choice_4": "Stir the porridge", "answer_id": 3, "answer": "Wash it", "question_type": [ "Action Recognition", "Counterfactual", "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "spoon-d3eb941379f623cd", "porridge bowl-9cdb618a1c4120fe", "plate-4107414cbd633a9e", "packet of raisins-039d581b287d8cbd", "plastic bag-8c25764414d2f011", "second package of raisins-11550f72e170badb", "pecan nuts-6ae12fde6718cdb1", "scales-550e5d82a358f01c", "porridge bowl on plate-6a4ff2e77f0d2976" ], "reasoning": "1. I watched to see everything the user does with the spoon (spoon-d3eb941379f623cd), which is lying on the plate (plate-4107414cbd633a9e) from the beginning of the video.\n2. At 00:06, the user picks up the spoon (spoon-d3eb941379f623cd) from the plate (plate-4107414cbd633a9e) and stirs the porridge (porridge bowl-9cdb618a1c4120fe).\n3. At 00:15, the user licks porridge off the spoon (spoon-d3eb941379f623cd).\n4. At 00:20, the user pushes bananas into the bowl (porridge bowl-9cdb618a1c4120fe) with the spoon (spoon-d3eb941379f623cd).\n5. I watched the rest of the video, and the user never washes the spoon (spoon-d3eb941379f623cd).", "referenced_object_masks": [ "porridge bowl-9cdb618a1c4120fe", "spoon-d3eb941379f623cd", "plate-4107414cbd633a9e" ] }, { "key": "P05-20240426-092000:421085aba0555e6f06e77a5d28024bb3d9a7af9e", "video_id": "P05-20240426-092000", "question": "What is the second to last ingredient I add to the porridge bowl?", "answer_choice_0": "Sliced bananas", "answer_choice_1": "Coconut", "answer_choice_2": "The second package of raisins", "answer_choice_3": "The first package of raisins", "answer_choice_4": "Pecans", "answer_id": 2, "answer": "The second package of raisins", "question_type": [ "Action Recognition", "Counting", "Numerical Reasoning", "Egocentric & Spatial Perception", "Object Recognition" ], "objects_of_interest": [ "spoon-d3eb941379f623cd", "porridge bowl-9cdb618a1c4120fe", "plate-4107414cbd633a9e", "packet of raisins-039d581b287d8cbd", "plastic bag-8c25764414d2f011", "second package of raisins-11550f72e170badb", "pecan nuts-6ae12fde6718cdb1", "scales-550e5d82a358f01c", "porridge bowl on plate-6a4ff2e77f0d2976" ], "reasoning": "1. I watched for the user to add ingredients to the porridge bowl (porridge bowl-9cdb618a1c4120fe).\n2. The first ingredient added to the porridge bowl (porridge bowl-9cdb618a1c4120fe) is sliced bananas at 00:21.\n3. The second ingredient added to the porridge bowl (porridge bowl-9cdb618a1c4120fe) is raisins (packet of raisins-039d581b287d8cbd) at 00:30.\n4. The third ingredient added to the porridge bowl (porridge bowl-9cdb618a1c4120fe) is a second package of raisins (second package of raisins-11550f72e170badb) at 00:48.\n5. The fourth ingredient added to the porridge bowl (porridge bowl-9cdb618a1c4120fe) is pecans (pecan nuts-6ae12fde6718cdb1) at 01:11.\n6. I continued watching, but no more ingredients are added, making the pecans (pecan nuts-6ae12fde6718cdb1) the final ingredient added.\n7. This makes the second package of raisins (second package of raisins-11550f72e170badb) the second to last ingredient added to the porridge bowl (porridge bowl-9cdb618a1c4120fe).", "referenced_object_masks": [ "pecan nuts-6ae12fde6718cdb1", "second package of raisins-11550f72e170badb", "porridge bowl-9cdb618a1c4120fe", "packet of raisins-039d581b287d8cbd" ] }, { "key": "P05-20240426-092000:7c88c18315e09478d819bab3ff9ba3dbc0c62139", "video_id": "P05-20240426-092000", "question": "How many more times do I wrap the rubber band around the second package of raisins than it was wrapped before I opened it?", "answer_choice_0": "0", "answer_choice_1": "3", "answer_choice_2": "2", "answer_choice_3": "4", "answer_choice_4": "1", "answer_id": 0, "answer": "0", "question_type": [ "Action Recognition", "Counting", "Numerical Reasoning", "Egocentric & Spatial Perception", "State Changes" ], "objects_of_interest": [ "spoon-d3eb941379f623cd", "porridge bowl-9cdb618a1c4120fe", "plate-4107414cbd633a9e", "packet of raisins-039d581b287d8cbd", "plastic bag-8c25764414d2f011", "second package of raisins-11550f72e170badb", "pecan nuts-6ae12fde6718cdb1", "scales-550e5d82a358f01c", "porridge bowl on plate-6a4ff2e77f0d2976" ], "reasoning": "1. I looked for the second package of raisins (second package of raisins-11550f72e170badb) which is seen at 0:36.\n2. The user pulls off the rubber band which encircles the package (second package of raisins-11550f72e170badb) twice.\n3. The user finishes pouring raisins out of the package (second package of raisins-11550f72e170badb) at 0:50.\n4. The user then wraps the rubber band back around the package (second package of raisins-11550f72e170badb) twice.\n5. There is no change in the number of times that the rubber band is wrapped around the second package of raisins (second package of raisins-11550f72e170badb) before and after use.", "referenced_object_masks": [ "second package of raisins-11550f72e170badb" ] }, { "key": "P05-20240426-092000:a93254f7a4262a487059dec8b2aa8c6aa39b0075", "video_id": "P05-20240426-092000", "question": "After pouring ingredients into the bowl of porridge, why couldn't I put the second package of raisins where I had put the first package of raisins?", "answer_choice_0": "Because there wasn't room", "answer_choice_1": "Because it was too high to reach", "answer_choice_2": "Because I was still holding the first package", "answer_choice_3": "Because the second package wasn't empty", "answer_choice_4": "Because I didn't have time", "answer_id": 3, "answer": "Because the second package wasn't empty", "question_type": [ "Action Recognition", "Cause and Effect", "Goal Reasoning", "Counterfactual", "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "spoon-d3eb941379f623cd", "porridge bowl-9cdb618a1c4120fe", "plate-4107414cbd633a9e", "packet of raisins-039d581b287d8cbd", "plastic bag-8c25764414d2f011", "second package of raisins-11550f72e170badb", "pecan nuts-6ae12fde6718cdb1", "scales-550e5d82a358f01c", "porridge bowl on plate-6a4ff2e77f0d2976" ], "reasoning": "1. I watched to see what the user did with the first package of raisins (packet of raisins-039d581b287d8cbd) and the second package of raisins (second package of raisins-11550f72e170badb) after the ingredients had gone into the bowl of porridge (porridge bowl-9cdb618a1c4120fe).\n2. After pouring the ingredient from the first package of raisins (packet of raisins-039d581b287d8cbd) at 00:30, the user puts the empty package under the table in a plastic bag (plastic bag-8c25764414d2f011) at 00:35, presumably for recycling purposes.\n3. The user pours from the second package of raisins (second package of raisins-11550f72e170badb) at 00:38, but there are still raisins left in the package.\n4. Therefore, it wouldn't make sense to recycle the package as it isn't empty.", "referenced_object_masks": [ "plastic bag-8c25764414d2f011", "porridge bowl-9cdb618a1c4120fe", "packet of raisins-039d581b287d8cbd", "second package of raisins-11550f72e170badb" ] }, { "key": "P05-20240427-143353:139a1af02832b0f2bf6014b31a502613c729ed1d", "video_id": "P05-20240427-143353", "question": "What are the 3 textures of the rice in this video, listed in chronological order?", "answer_choice_0": "Hard, soaked, hard from boiling", "answer_choice_1": "Hard, soaked, soft from boiling", "answer_choice_2": "Soft, soaked, soft from boiling", "answer_choice_3": "Hard, soft, hard from boiling", "answer_choice_4": "Soft, soaked, hard from boiling", "answer_id": 1, "answer": "Hard, soaked, soft from boiling", "question_type": [ "Action Recognition", "Event Occurence", "State Changes", "Temporal Reasoning", "Object Recognition" ], "objects_of_interest": [ "pot-fb17e9f4e622bef4", "scales-db03eed0a9ce7dd4", "pan lid-0276f810bedf6db5", "sieve-fe9b14d13e0a53a7", "bag of rice-86bb1bacdc3cbe77", "washing up bowl-fc72fc3a891d0063", "pint glass-d1f0609d46b95cad", "phone-647d211abc447c9b", "carrot1-ec6d69fff15f4bd5", "plastic bag-e9cb682d6b45c0a3", "carrot2-88eb3aced746e547", "spring onion-686e43cb3904d6ce", "sriracha sauce bottle-5169ad8f2088bc5f", "soy sauce bottle-0893d41f5e9167c1", "peanut butter jar-f4a3dc222966f2b5", "teaspoon-92bd9892215473ea", "tin of coconut milk-9c1bb90203fa62c7", "knife-33922bd38b374613", "chopping board-bfdb77c1d11d6af8", "red onion-e75b440968cd0281", "half of onion-e35231f8b45ce0b3", "metal bowl-c408756860db95ea", "garlic-b00c556f79137f48", "bag of cashew nuts-3c0906fba0c8f285", "tin opener-d80868cd63006336", "cover of tin of coconut milk-c92c2d64b6a2b596", "washing up bowl2-ac6149924782dd00", "washing up sponge-6d105a14aa01f13f", "towel-4ae4fac6c8e175d8", "recycling container-f57b4e3fe88d8784", "recycling container lid-ec3579675d5f7d91", "dirty spoon-1ab2dbf2b89a6131", "dirty spoon2-e6fb417280127e3f", "serving spoon-398fa55f930f234b", "pan lid2-7104c888fd283971", "dirty plastic bowl-e5b81844b067a2e3", "metal bowl2-fb7cd1d2342aac87", "dirty pan-30cbf0278630c6c9", "dirty plastic lid-f98afa38cf8f9891", "dirty bowl-60a4087190e5db2f", "dirty bowl2-bcbef987d0b796b4", "2pint glass-41ae0dff97d140ec", "bowl3-46df756dbd50c250", "pan-39d8fd751308f021", "pan lid3-70dc73e2f9834819", "pink washing up gloves-a6bbce7104392729", "bottle of oil-60b203fb5f221205", "washing up brush-a9914d804116ceac", "spatula-8a7615e5e8ce9e9b", "spoon-52a14800961b0d58" ], "reasoning": "1. I watched the whole video and saw the user pour uncooked/hard rice from a bag (bag of rice-86bb1bacdc3cbe77) into a strainer (sieve-fe9b14d13e0a53a7) on top of a pan (pot-fb17e9f4e622bef4) at 00:48.\n2. At 01:29, the user washes the uncooked/hard rice in the strainer (sieve-fe9b14d13e0a53a7) under the water faucet until 02:11, leaving the rice soaked.\n3. At 03:24, the user puts the pan (pan-39d8fd751308f021) with the rice and water on the stove to make the rice soft by boiling it.", "referenced_object_masks": [ "sieve-fe9b14d13e0a53a7", "pan-39d8fd751308f021", "pot-fb17e9f4e622bef4", "bag of rice-86bb1bacdc3cbe77" ] }, { "key": "P05-20240427-143353:39a011b00f469b2a5e93ad22899b684bebb250b1", "video_id": "P05-20240427-143353", "question": "I use a metal mixing bowl in the video. How many ingredients do I put into this bowl?", "answer_choice_0": "1", "answer_choice_1": "3", "answer_choice_2": "2", "answer_choice_3": "0", "answer_choice_4": "4", "answer_id": 2, "answer": "2", "question_type": [ "Action Recognition", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "pot-fb17e9f4e622bef4", "scales-db03eed0a9ce7dd4", "pan lid-0276f810bedf6db5", "sieve-fe9b14d13e0a53a7", "bag of rice-86bb1bacdc3cbe77", "washing up bowl-fc72fc3a891d0063", "pint glass-d1f0609d46b95cad", "phone-647d211abc447c9b", "carrot1-ec6d69fff15f4bd5", "plastic bag-e9cb682d6b45c0a3", "carrot2-88eb3aced746e547", "spring onion-686e43cb3904d6ce", "sriracha sauce bottle-5169ad8f2088bc5f", "soy sauce bottle-0893d41f5e9167c1", "peanut butter jar-f4a3dc222966f2b5", "teaspoon-92bd9892215473ea", "tin of coconut milk-9c1bb90203fa62c7", "knife-33922bd38b374613", "chopping board-bfdb77c1d11d6af8", "red onion-e75b440968cd0281", "half of onion-e35231f8b45ce0b3", "metal bowl-c408756860db95ea", "garlic-b00c556f79137f48", "bag of cashew nuts-3c0906fba0c8f285", "tin opener-d80868cd63006336", "cover of tin of coconut milk-c92c2d64b6a2b596", "washing up bowl2-ac6149924782dd00", "washing up sponge-6d105a14aa01f13f", "towel-4ae4fac6c8e175d8", "recycling container-f57b4e3fe88d8784", "recycling container lid-ec3579675d5f7d91", "dirty spoon-1ab2dbf2b89a6131", "dirty spoon2-e6fb417280127e3f", "serving spoon-398fa55f930f234b", "pan lid2-7104c888fd283971", "dirty plastic bowl-e5b81844b067a2e3", "metal bowl2-fb7cd1d2342aac87", "dirty pan-30cbf0278630c6c9", "dirty plastic lid-f98afa38cf8f9891", "dirty bowl-60a4087190e5db2f", "dirty bowl2-bcbef987d0b796b4", "2pint glass-41ae0dff97d140ec", "bowl3-46df756dbd50c250", "pan-39d8fd751308f021", "pan lid3-70dc73e2f9834819", "pink washing up gloves-a6bbce7104392729", "bottle of oil-60b203fb5f221205", "washing up brush-a9914d804116ceac", "spatula-8a7615e5e8ce9e9b", "spoon-52a14800961b0d58" ], "reasoning": "1. I notice the user grab the metal mixing bowl (metal bowl-c408756860db95ea) at 08:42.\n2. At 08:50 the user places the spring onions (spring onion-686e43cb3904d6ce) into the mixing bowl (metal bowl-c408756860db95ea), but then immediately takes them out.\n3. I continue watching the video and notice the user places half of the cut red onion (half of onion-e35231f8b45ce0b3) in the mixing bowl (metal bowl-c408756860db95ea). This is 1 ingredient.\n4. At 09:10, the user dumps the cut up carrots (carrot1-ec6d69fff15f4bd5) (carrot2-88eb3aced746e547) into the mixing bowl (metal bowl-c408756860db95ea). This is the second ingredient.\n5. I continue to watch the video and notice the user does not add any other ingredients to the bowl (metal bowl-c408756860db95ea).\n6. The user added a total of 2 ingredients to the metal mixing bowl (metal bowl-c408756860db95ea).", "referenced_object_masks": [ "spring onion-686e43cb3904d6ce", "half of onion-e35231f8b45ce0b3", "metal bowl-c408756860db95ea", "carrot1-ec6d69fff15f4bd5", "carrot2-88eb3aced746e547" ] }, { "key": "P05-20240427-143353:39b0977f5aeb08515ef0c45211f33ba23f803db3", "video_id": "P05-20240427-143353", "question": "How many carrots do I cut in the video?", "answer_choice_0": "1", "answer_choice_1": "2", "answer_choice_2": "0", "answer_choice_3": "3", "answer_choice_4": "4", "answer_id": 1, "answer": "2", "question_type": [ "Action Recognition", "Numerical Reasoning", "Temporal Reasoning", "Object Recognition" ], "objects_of_interest": [ "pot-fb17e9f4e622bef4", "scales-db03eed0a9ce7dd4", "pan lid-0276f810bedf6db5", "sieve-fe9b14d13e0a53a7", "bag of rice-86bb1bacdc3cbe77", "washing up bowl-fc72fc3a891d0063", "pint glass-d1f0609d46b95cad", "phone-647d211abc447c9b", "carrot1-ec6d69fff15f4bd5", "plastic bag-e9cb682d6b45c0a3", "carrot2-88eb3aced746e547", "spring onion-686e43cb3904d6ce", "sriracha sauce bottle-5169ad8f2088bc5f", "soy sauce bottle-0893d41f5e9167c1", "peanut butter jar-f4a3dc222966f2b5", "teaspoon-92bd9892215473ea", "tin of coconut milk-9c1bb90203fa62c7", "knife-33922bd38b374613", "chopping board-bfdb77c1d11d6af8", "red onion-e75b440968cd0281", "half of onion-e35231f8b45ce0b3", "metal bowl-c408756860db95ea", "garlic-b00c556f79137f48", "bag of cashew nuts-3c0906fba0c8f285", "tin opener-d80868cd63006336", "cover of tin of coconut milk-c92c2d64b6a2b596", "washing up bowl2-ac6149924782dd00", "washing up sponge-6d105a14aa01f13f", "towel-4ae4fac6c8e175d8", "recycling container-f57b4e3fe88d8784", "recycling container lid-ec3579675d5f7d91", "dirty spoon-1ab2dbf2b89a6131", "dirty spoon2-e6fb417280127e3f", "serving spoon-398fa55f930f234b", "pan lid2-7104c888fd283971", "dirty plastic bowl-e5b81844b067a2e3", "metal bowl2-fb7cd1d2342aac87", "dirty pan-30cbf0278630c6c9", "dirty plastic lid-f98afa38cf8f9891", "dirty bowl-60a4087190e5db2f", "dirty bowl2-bcbef987d0b796b4", "2pint glass-41ae0dff97d140ec", "bowl3-46df756dbd50c250", "pan-39d8fd751308f021", "pan lid3-70dc73e2f9834819", "pink washing up gloves-a6bbce7104392729", "bottle of oil-60b203fb5f221205", "washing up brush-a9914d804116ceac", "spatula-8a7615e5e8ce9e9b", "spoon-52a14800961b0d58" ], "reasoning": "1. At 6:50, I notice the user takes a carrot (carrot1-ec6d69fff15f4bd5) and begins cutting it into pieces on a cutting board. This is the first carrot (carrot1-ec6d69fff15f4bd5) that is cut.\n2. At 07:34, I notice the user takes a second carrot (carrot2-88eb3aced746e547) and cuts it into pieces. This is the second carrot (carrot2-88eb3aced746e547) cut. \n3. I continue to watch the video and notice the user does not cut any more carrots. \n4. There were 2 carrots cut in the video.", "referenced_object_masks": [ "carrot2-88eb3aced746e547", "carrot1-ec6d69fff15f4bd5" ] }, { "key": "P05-20240427-143353:45e6e5869392f9a8811e27cdf335482de7b38c12", "video_id": "P05-20240427-143353", "question": "How many distinct vegetables do I cut with a knife?", "answer_choice_0": "3", "answer_choice_1": "1", "answer_choice_2": "2", "answer_choice_3": "0", "answer_choice_4": "4", "answer_id": 4, "answer": "4", "question_type": [ "Action Recognition", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "pot-fb17e9f4e622bef4", "scales-db03eed0a9ce7dd4", "pan lid-0276f810bedf6db5", "sieve-fe9b14d13e0a53a7", "bag of rice-86bb1bacdc3cbe77", "washing up bowl-fc72fc3a891d0063", "pint glass-d1f0609d46b95cad", "phone-647d211abc447c9b", "carrot1-ec6d69fff15f4bd5", "plastic bag-e9cb682d6b45c0a3", "carrot2-88eb3aced746e547", "spring onion-686e43cb3904d6ce", "sriracha sauce bottle-5169ad8f2088bc5f", "soy sauce bottle-0893d41f5e9167c1", "peanut butter jar-f4a3dc222966f2b5", "teaspoon-92bd9892215473ea", "tin of coconut milk-9c1bb90203fa62c7", "knife-33922bd38b374613", "chopping board-bfdb77c1d11d6af8", "red onion-e75b440968cd0281", "half of onion-e35231f8b45ce0b3", "metal bowl-c408756860db95ea", "garlic-b00c556f79137f48", "bag of cashew nuts-3c0906fba0c8f285", "tin opener-d80868cd63006336", "cover of tin of coconut milk-c92c2d64b6a2b596", "washing up bowl2-ac6149924782dd00", "washing up sponge-6d105a14aa01f13f", "towel-4ae4fac6c8e175d8", "recycling container-f57b4e3fe88d8784", "recycling container lid-ec3579675d5f7d91", "dirty spoon-1ab2dbf2b89a6131", "dirty spoon2-e6fb417280127e3f", "serving spoon-398fa55f930f234b", "pan lid2-7104c888fd283971", "dirty plastic bowl-e5b81844b067a2e3", "metal bowl2-fb7cd1d2342aac87", "dirty pan-30cbf0278630c6c9", "dirty plastic lid-f98afa38cf8f9891", "dirty bowl-60a4087190e5db2f", "dirty bowl2-bcbef987d0b796b4", "2pint glass-41ae0dff97d140ec", "bowl3-46df756dbd50c250", "pan-39d8fd751308f021", "pan lid3-70dc73e2f9834819", "pink washing up gloves-a6bbce7104392729", "bottle of oil-60b203fb5f221205", "washing up brush-a9914d804116ceac", "spatula-8a7615e5e8ce9e9b", "spoon-52a14800961b0d58" ], "reasoning": "1. I notice the user grab a knife (knife-33922bd38b374613) out of a kitchen drawer at 05:58.\n2. I continue watching the video and notice the user begin cutting a red onion (red onion-e75b440968cd0281) at 06:11. This is the 1st distinct vegetable.\n3. Continuing to watch the video, I notice the user begin to cut a carrot (carrot1-ec6d69fff15f4bd5) at 06:50. The user then cuts another carrot (carrot2-88eb3aced746e547) at 07:34. This is the 2nd distinct vegetable.\n4. I continue to watch the video and notice the user cut a spring onion (spring onion-686e43cb3904d6ce) at 08:07. This is the 3rd distinct vegetable. \n5. I continue to watch the video and notice the user cut a garlic clove (garlic-b00c556f79137f48) at 10:25. This is the 4th distinct vegetable.\n6. I continue watching the video and notice the user clean and eventually put the knife (knife-33922bd38b374613) back into the kitchen drawer at 17:59. \n7. The user cut a total of 4 distinct vegetables.", "referenced_object_masks": [ "spring onion-686e43cb3904d6ce", "red onion-e75b440968cd0281", "garlic-b00c556f79137f48", "knife-33922bd38b374613", "carrot1-ec6d69fff15f4bd5", "carrot2-88eb3aced746e547" ] }, { "key": "P05-20240427-143353:49b852b36c7c255208d3769b15b18621216d16b1", "video_id": "P05-20240427-143353", "question": "How many degrees would I have to turn clockwise from where I am standing at 03:24 to get to the island where I just weighed the rice?", "answer_choice_0": "135 degrees", "answer_choice_1": "180 degrees", "answer_choice_2": "100 degrees", "answer_choice_3": "90 degrees", "answer_choice_4": "45 degrees", "answer_id": 0, "answer": "135 degrees", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Event Occurence", "Numerical Reasoning", "Temporal Reasoning" ], "objects_of_interest": [ "pot-fb17e9f4e622bef4", "scales-db03eed0a9ce7dd4", "pan lid-0276f810bedf6db5", "sieve-fe9b14d13e0a53a7", "bag of rice-86bb1bacdc3cbe77", "washing up bowl-fc72fc3a891d0063", "pint glass-d1f0609d46b95cad", "phone-647d211abc447c9b", "carrot1-ec6d69fff15f4bd5", "plastic bag-e9cb682d6b45c0a3", "carrot2-88eb3aced746e547", "spring onion-686e43cb3904d6ce", "sriracha sauce bottle-5169ad8f2088bc5f", "soy sauce bottle-0893d41f5e9167c1", "peanut butter jar-f4a3dc222966f2b5", "teaspoon-92bd9892215473ea", "tin of coconut milk-9c1bb90203fa62c7", "knife-33922bd38b374613", "chopping board-bfdb77c1d11d6af8", "red onion-e75b440968cd0281", "half of onion-e35231f8b45ce0b3", "metal bowl-c408756860db95ea", "garlic-b00c556f79137f48", "bag of cashew nuts-3c0906fba0c8f285", "tin opener-d80868cd63006336", "cover of tin of coconut milk-c92c2d64b6a2b596", "washing up bowl2-ac6149924782dd00", "washing up sponge-6d105a14aa01f13f", "towel-4ae4fac6c8e175d8", "recycling container-f57b4e3fe88d8784", "recycling container lid-ec3579675d5f7d91", "dirty spoon-1ab2dbf2b89a6131", "dirty spoon2-e6fb417280127e3f", "serving spoon-398fa55f930f234b", "pan lid2-7104c888fd283971", "dirty plastic bowl-e5b81844b067a2e3", "metal bowl2-fb7cd1d2342aac87", "dirty pan-30cbf0278630c6c9", "dirty plastic lid-f98afa38cf8f9891", "dirty bowl-60a4087190e5db2f", "dirty bowl2-bcbef987d0b796b4", "2pint glass-41ae0dff97d140ec", "bowl3-46df756dbd50c250", "pan-39d8fd751308f021", "pan lid3-70dc73e2f9834819", "pink washing up gloves-a6bbce7104392729", "bottle of oil-60b203fb5f221205", "washing up brush-a9914d804116ceac", "spatula-8a7615e5e8ce9e9b", "spoon-52a14800961b0d58" ], "reasoning": "1. I watched the whole video and saw the user weighing rice at 00:49 at the kitchen island.\n2. At 03:24, the user was standing at the stove .\n3. At 03:25, the user makes a 90 degree turn to the right.\n4. The user walks straight down the kitchen floor and makes at 45 degree right turn at 03:37 to face the kitchen island.\n5. 90 degrees plus 45 degrees is 135 degrees.", "referenced_object_masks": [] }, { "key": "P05-20240427-143353:4e5955ee3ee5cebd562f1f9d98177722fcc54a73", "video_id": "P05-20240427-143353", "question": "Calculate how much time has passed from when I check my phone for the first time and from when I check my phone for the second time by subtracting the initial clock time displayed on my phone during both occurences. The answer should be in hh:mm.", "answer_choice_0": "00:18", "answer_choice_1": "00:17", "answer_choice_2": "00:13", "answer_choice_3": "00:10", "answer_choice_4": "00:05", "answer_id": 1, "answer": "00:17", "question_type": [ "Action Recognition", "Counting", "Event Occurence", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "pot-fb17e9f4e622bef4", "scales-db03eed0a9ce7dd4", "pan lid-0276f810bedf6db5", "sieve-fe9b14d13e0a53a7", "bag of rice-86bb1bacdc3cbe77", "washing up bowl-fc72fc3a891d0063", "pint glass-d1f0609d46b95cad", "phone-647d211abc447c9b", "carrot1-ec6d69fff15f4bd5", "plastic bag-e9cb682d6b45c0a3", "carrot2-88eb3aced746e547", "spring onion-686e43cb3904d6ce", "sriracha sauce bottle-5169ad8f2088bc5f", "soy sauce bottle-0893d41f5e9167c1", "peanut butter jar-f4a3dc222966f2b5", "teaspoon-92bd9892215473ea", "tin of coconut milk-9c1bb90203fa62c7", "knife-33922bd38b374613", "chopping board-bfdb77c1d11d6af8", "red onion-e75b440968cd0281", "half of onion-e35231f8b45ce0b3", "metal bowl-c408756860db95ea", "garlic-b00c556f79137f48", "bag of cashew nuts-3c0906fba0c8f285", "tin opener-d80868cd63006336", "cover of tin of coconut milk-c92c2d64b6a2b596", "washing up bowl2-ac6149924782dd00", "washing up sponge-6d105a14aa01f13f", "towel-4ae4fac6c8e175d8", "recycling container-f57b4e3fe88d8784", "recycling container lid-ec3579675d5f7d91", "dirty spoon-1ab2dbf2b89a6131", "dirty spoon2-e6fb417280127e3f", "serving spoon-398fa55f930f234b", "pan lid2-7104c888fd283971", "dirty plastic bowl-e5b81844b067a2e3", "metal bowl2-fb7cd1d2342aac87", "dirty pan-30cbf0278630c6c9", "dirty plastic lid-f98afa38cf8f9891", "dirty bowl-60a4087190e5db2f", "dirty bowl2-bcbef987d0b796b4", "2pint glass-41ae0dff97d140ec", "bowl3-46df756dbd50c250", "pan-39d8fd751308f021", "pan lid3-70dc73e2f9834819", "pink washing up gloves-a6bbce7104392729", "bottle of oil-60b203fb5f221205", "washing up brush-a9914d804116ceac", "spatula-8a7615e5e8ce9e9b", "spoon-52a14800961b0d58" ], "reasoning": "1. I watch the video until I notice the user check the phone (phone-647d211abc447c9b) for the first time at 03:29. The initial time displayed on the phone (phone-647d211abc447c9b) is 14:36.\n2. I continue watching the video and note the clock time on the phone (phone-647d211abc447c9b) each time it is checked thereafter: 14:46 at 12:56 (2nd time), 14:49 at 15:59 (3rd time), 14:53 at 19:33 (4th time), and 14:53 again at 19:37 (5th time).\n3. I subtract the first clock time from the fifth, 14:53 - 14:36, and get a difference of 17 minutes, or 00:17.", "referenced_object_masks": [ "phone-647d211abc447c9b" ] }, { "key": "P05-20240427-143353:9721d288eb3681f9313cff787b8c323dc4ae289a", "video_id": "P05-20240427-143353", "question": "If I looked at how much time elapsed between the first time I looked at my phone and the second time I looked at my phone, multiply this number by the number of seconds it took me to drink a glass of water at 03:49 and raise it to the second power, what number would I get?", "answer_choice_0": "14,161", "answer_choice_1": "13,924", "answer_choice_2": "9,216", "answer_choice_3": "13,456", "answer_choice_4": "14,400", "answer_id": 0, "answer": "14,161", "question_type": [ "Action Recognition", "Event Occurence", "Numerical Reasoning", "Temporal Reasoning" ], "objects_of_interest": [ "pot-fb17e9f4e622bef4", "scales-db03eed0a9ce7dd4", "pan lid-0276f810bedf6db5", "sieve-fe9b14d13e0a53a7", "bag of rice-86bb1bacdc3cbe77", "washing up bowl-fc72fc3a891d0063", "pint glass-d1f0609d46b95cad", "phone-647d211abc447c9b", "carrot1-ec6d69fff15f4bd5", "plastic bag-e9cb682d6b45c0a3", "carrot2-88eb3aced746e547", "spring onion-686e43cb3904d6ce", "sriracha sauce bottle-5169ad8f2088bc5f", "soy sauce bottle-0893d41f5e9167c1", "peanut butter jar-f4a3dc222966f2b5", "teaspoon-92bd9892215473ea", "tin of coconut milk-9c1bb90203fa62c7", "knife-33922bd38b374613", "chopping board-bfdb77c1d11d6af8", "red onion-e75b440968cd0281", "half of onion-e35231f8b45ce0b3", "metal bowl-c408756860db95ea", "garlic-b00c556f79137f48", "bag of cashew nuts-3c0906fba0c8f285", "tin opener-d80868cd63006336", "cover of tin of coconut milk-c92c2d64b6a2b596", "washing up bowl2-ac6149924782dd00", "washing up sponge-6d105a14aa01f13f", "towel-4ae4fac6c8e175d8", "recycling container-f57b4e3fe88d8784", "recycling container lid-ec3579675d5f7d91", "dirty spoon-1ab2dbf2b89a6131", "dirty spoon2-e6fb417280127e3f", "serving spoon-398fa55f930f234b", "pan lid2-7104c888fd283971", "dirty plastic bowl-e5b81844b067a2e3", "metal bowl2-fb7cd1d2342aac87", "dirty pan-30cbf0278630c6c9", "dirty plastic lid-f98afa38cf8f9891", "dirty bowl-60a4087190e5db2f", "dirty bowl2-bcbef987d0b796b4", "2pint glass-41ae0dff97d140ec", "bowl3-46df756dbd50c250", "pan-39d8fd751308f021", "pan lid3-70dc73e2f9834819", "pink washing up gloves-a6bbce7104392729", "bottle of oil-60b203fb5f221205", "washing up brush-a9914d804116ceac", "spatula-8a7615e5e8ce9e9b", "spoon-52a14800961b0d58" ], "reasoning": "1. I watched the whole video and saw the user look at their phone (phone-647d211abc447c9b) at 03:29.\n2. The time on the phone (phone-647d211abc447c9b) said 14:36.\n3. The user looked at their phone (phone-647d211abc447c9b) the second time at 19:34 and the time was 14:53. \n4. 14:53 minus 14:36 is 17 minutes.\n5. At 03:49, the user began to drink a glass of water. \n6. The user finished the glass of water at 03:56.\n7. 03:56 minus 03:49 is 7 minutes.\n8. 17 times 7 is 119.\n9. 119^2 is 14,161.", "referenced_object_masks": [ "phone-647d211abc447c9b" ] }, { "key": "P05-20240427-143353:aba56179dc1731bfc19a1cc207e38f5d7d0ccdb5", "video_id": "P05-20240427-143353", "question": "When standing at the island where the bag of rice is at 04:00, how many degrees counterclockwise do I need to turn to get to the yellow pan with the yellow lid with a black utensil sticking out of it?", "answer_choice_0": "60 degrees", "answer_choice_1": "0 degrees", "answer_choice_2": "45 degrees", "answer_choice_3": "90 degrees", "answer_choice_4": "30 degrees", "answer_id": 2, "answer": "45 degrees", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Event Occurence", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "pot-fb17e9f4e622bef4", "scales-db03eed0a9ce7dd4", "pan lid-0276f810bedf6db5", "sieve-fe9b14d13e0a53a7", "bag of rice-86bb1bacdc3cbe77", "washing up bowl-fc72fc3a891d0063", "pint glass-d1f0609d46b95cad", "phone-647d211abc447c9b", "carrot1-ec6d69fff15f4bd5", "plastic bag-e9cb682d6b45c0a3", "carrot2-88eb3aced746e547", "spring onion-686e43cb3904d6ce", "sriracha sauce bottle-5169ad8f2088bc5f", "soy sauce bottle-0893d41f5e9167c1", "peanut butter jar-f4a3dc222966f2b5", "teaspoon-92bd9892215473ea", "tin of coconut milk-9c1bb90203fa62c7", "knife-33922bd38b374613", "chopping board-bfdb77c1d11d6af8", "red onion-e75b440968cd0281", "half of onion-e35231f8b45ce0b3", "metal bowl-c408756860db95ea", "garlic-b00c556f79137f48", "bag of cashew nuts-3c0906fba0c8f285", "tin opener-d80868cd63006336", "cover of tin of coconut milk-c92c2d64b6a2b596", "washing up bowl2-ac6149924782dd00", "washing up sponge-6d105a14aa01f13f", "towel-4ae4fac6c8e175d8", "recycling container-f57b4e3fe88d8784", "recycling container lid-ec3579675d5f7d91", "dirty spoon-1ab2dbf2b89a6131", "dirty spoon2-e6fb417280127e3f", "serving spoon-398fa55f930f234b", "pan lid2-7104c888fd283971", "dirty plastic bowl-e5b81844b067a2e3", "metal bowl2-fb7cd1d2342aac87", "dirty pan-30cbf0278630c6c9", "dirty plastic lid-f98afa38cf8f9891", "dirty bowl-60a4087190e5db2f", "dirty bowl2-bcbef987d0b796b4", "2pint glass-41ae0dff97d140ec", "bowl3-46df756dbd50c250", "pan-39d8fd751308f021", "pan lid3-70dc73e2f9834819", "pink washing up gloves-a6bbce7104392729", "bottle of oil-60b203fb5f221205", "washing up brush-a9914d804116ceac", "spatula-8a7615e5e8ce9e9b", "spoon-52a14800961b0d58" ], "reasoning": "1. I watched the whole video and saw the user standing at the island where the bag of rice (bag of rice-86bb1bacdc3cbe77) is at 04:00.\n2. At 04:06, the user turns left 45 degrees to walk to the refrigerator . \n3. The yellow pan with the yellow lid (pan lid2-7104c888fd283971) can be seen on the counter in front of the microwave at 04:07.", "referenced_object_masks": [ "bag of rice-86bb1bacdc3cbe77", "pan lid2-7104c888fd283971" ] }, { "key": "P05-20240427-143353:d730e9f198ad122cabe80ce18251f8e7916ec9ca", "video_id": "P05-20240427-143353", "question": "At 13:47, I opened up the dishwasher. What is the 4th object I put into the dishwasher?", "answer_choice_0": "Spoon", "answer_choice_1": "Cooking Pot", "answer_choice_2": "Plate", "answer_choice_3": "Knife", "answer_choice_4": "Cooking Pan Lid", "answer_id": 4, "answer": "Cooking Pan Lid", "question_type": [ "Action Recognition", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "pot-fb17e9f4e622bef4", "scales-db03eed0a9ce7dd4", "pan lid-0276f810bedf6db5", "sieve-fe9b14d13e0a53a7", "bag of rice-86bb1bacdc3cbe77", "washing up bowl-fc72fc3a891d0063", "pint glass-d1f0609d46b95cad", "phone-647d211abc447c9b", "carrot1-ec6d69fff15f4bd5", "plastic bag-e9cb682d6b45c0a3", "carrot2-88eb3aced746e547", "spring onion-686e43cb3904d6ce", "sriracha sauce bottle-5169ad8f2088bc5f", "soy sauce bottle-0893d41f5e9167c1", "peanut butter jar-f4a3dc222966f2b5", "teaspoon-92bd9892215473ea", "tin of coconut milk-9c1bb90203fa62c7", "knife-33922bd38b374613", "chopping board-bfdb77c1d11d6af8", "red onion-e75b440968cd0281", "half of onion-e35231f8b45ce0b3", "metal bowl-c408756860db95ea", "garlic-b00c556f79137f48", "bag of cashew nuts-3c0906fba0c8f285", "tin opener-d80868cd63006336", "cover of tin of coconut milk-c92c2d64b6a2b596", "washing up bowl2-ac6149924782dd00", "washing up sponge-6d105a14aa01f13f", "towel-4ae4fac6c8e175d8", "recycling container-f57b4e3fe88d8784", "recycling container lid-ec3579675d5f7d91", "dirty spoon-1ab2dbf2b89a6131", "dirty spoon2-e6fb417280127e3f", "serving spoon-398fa55f930f234b", "pan lid2-7104c888fd283971", "dirty plastic bowl-e5b81844b067a2e3", "metal bowl2-fb7cd1d2342aac87", "dirty pan-30cbf0278630c6c9", "dirty plastic lid-f98afa38cf8f9891", "dirty bowl-60a4087190e5db2f", "dirty bowl2-bcbef987d0b796b4", "2pint glass-41ae0dff97d140ec", "bowl3-46df756dbd50c250", "pan-39d8fd751308f021", "pan lid3-70dc73e2f9834819", "pink washing up gloves-a6bbce7104392729", "bottle of oil-60b203fb5f221205", "washing up brush-a9914d804116ceac", "spatula-8a7615e5e8ce9e9b", "spoon-52a14800961b0d58" ], "reasoning": "1. I watch the video starting at 13:47 and notice the user put two dirty spoons (dirty spoon-1ab2dbf2b89a6131, dirty spoon2-e6fb417280127e3f) into the dishwasher at 13:58. These are the first 2 objects put into the dishwasher. \n2. I continue to watch the video and notice the user put a serving spoon (serving spoon-398fa55f930f234b) into the dishwasher at 14:02. This is the 3rd object put into the dishwasher. \n3. I continue to watch the video and notice the user put a cooking pan lid (pan lid2-7104c888fd283971) into the dishwasher. This is the 4th object put into the dishwasher.", "referenced_object_masks": [ "pan lid2-7104c888fd283971", "serving spoon-398fa55f930f234b", "dirty spoon-1ab2dbf2b89a6131", "dirty spoon2-e6fb417280127e3f" ] }, { "key": "P05-20240427-143353:e77ee2a978bcadd263ef20d3e6fba08eb1c8f687", "video_id": "P05-20240427-143353", "question": "If I take the difference in weight of the rice at 00:38 and 01:17, add the weight of the water and rice at 03:01 and multiply it by the amount of seconds I wash the rice under the faucet, what number do I get?", "answer_choice_0": "16,810", "answer_choice_1": "17,514", "answer_choice_2": "17,220", "answer_choice_3": "16,880", "answer_choice_4": "17,346", "answer_id": 1, "answer": "17,514", "question_type": [ "Action Recognition", "Event Occurence", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "pot-fb17e9f4e622bef4", "scales-db03eed0a9ce7dd4", "pan lid-0276f810bedf6db5", "sieve-fe9b14d13e0a53a7", "bag of rice-86bb1bacdc3cbe77", "washing up bowl-fc72fc3a891d0063", "pint glass-d1f0609d46b95cad", "phone-647d211abc447c9b", "carrot1-ec6d69fff15f4bd5", "plastic bag-e9cb682d6b45c0a3", "carrot2-88eb3aced746e547", "spring onion-686e43cb3904d6ce", "sriracha sauce bottle-5169ad8f2088bc5f", "soy sauce bottle-0893d41f5e9167c1", "peanut butter jar-f4a3dc222966f2b5", "teaspoon-92bd9892215473ea", "tin of coconut milk-9c1bb90203fa62c7", "knife-33922bd38b374613", "chopping board-bfdb77c1d11d6af8", "red onion-e75b440968cd0281", "half of onion-e35231f8b45ce0b3", "metal bowl-c408756860db95ea", "garlic-b00c556f79137f48", "bag of cashew nuts-3c0906fba0c8f285", "tin opener-d80868cd63006336", "cover of tin of coconut milk-c92c2d64b6a2b596", "washing up bowl2-ac6149924782dd00", "washing up sponge-6d105a14aa01f13f", "towel-4ae4fac6c8e175d8", "recycling container-f57b4e3fe88d8784", "recycling container lid-ec3579675d5f7d91", "dirty spoon-1ab2dbf2b89a6131", "dirty spoon2-e6fb417280127e3f", "serving spoon-398fa55f930f234b", "pan lid2-7104c888fd283971", "dirty plastic bowl-e5b81844b067a2e3", "metal bowl2-fb7cd1d2342aac87", "dirty pan-30cbf0278630c6c9", "dirty plastic lid-f98afa38cf8f9891", "dirty bowl-60a4087190e5db2f", "dirty bowl2-bcbef987d0b796b4", "2pint glass-41ae0dff97d140ec", "bowl3-46df756dbd50c250", "pan-39d8fd751308f021", "pan lid3-70dc73e2f9834819", "pink washing up gloves-a6bbce7104392729", "bottle of oil-60b203fb5f221205", "washing up brush-a9914d804116ceac", "spatula-8a7615e5e8ce9e9b", "spoon-52a14800961b0d58" ], "reasoning": "1. I watched the whole video and saw the user weigh the rice in the pan (pan-39d8fd751308f021) at 00:38 which was 180 g.\n2. At 01:17, the new weight of the rice after being washed was 240 g.\n3. Added together, 240 minus 180 is 60.\n4. At 03:01, the user finished adding water to the rice in the pan (pan-39d8fd751308f021), so the new weight of the rice was 357.\n5. I saw the user start to wash the rice at 01:29 and finished washing the rice at 02:11 which is 42 seconds.\n6. 60 + 357 is 417.\n7. 417*42 is 17,514.", "referenced_object_masks": [ "pan-39d8fd751308f021" ] }, { "key": "P05-20240427-145526:0134a435bc99d89d148be79da4836b6f80068422", "video_id": "P05-20240427-145526", "question": "What is the combined number of times that I scoop from the can and taste from the pan?", "answer_choice_0": "5 times", "answer_choice_1": "9 times", "answer_choice_2": "11 times", "answer_choice_3": "10 times", "answer_choice_4": "7 times", "answer_id": 0, "answer": "5 times", "question_type": [ "Action Recognition", "Counting", "Numerical Reasoning", "Object Recognition" ], "objects_of_interest": [ "spatula-56887f675176f060", "pan-b9ac55679fa82dd6", "pan lid-ae0a06184751fb96", "bowl-3cd8178bd2fb02f0", "groundnut oil-3f52eb251b9429dc", "phone-6e0c4e156e434f6e", "pot-2d7a33128fd962dd", "chopping board-79f3c268834a00ee", "scales-55387c3f04db27eb", "tin opener-d3424d3b76192b58", "bowl2-3bdbc08281d71952", "spoon2-ce1084e62e2cb20d", "plate-a205745e239f9882", "cashew nuts bag-a0ca899ca4b72881", "can-9ec500193317d54e", "spoon-74fd9250b32dfa8e", "bottle-66debb3da13697fc", "bottle3-910f720cbaf30b36", "peanut butter-7506a814b7e8b969", "lid of peanut butter-bfd4f60e94a2d678" ], "reasoning": "1. I watched the video to add up the number of times that the user scooped from the can (can-9ec500193317d54e) and tasted the food in the pan (pan-b9ac55679fa82dd6). The first time either of these actions occurs is at 04:15, when the user picks up the can (can-9ec500193317d54e).\n2. Then, I watched to see how many times they scooped from the can (can-9ec500193317d54e). I found that this occurred at 04:19, 04:22, and 04:27. They place the can (can-9ec500193317d54e) down on the counter at 04:32.\n3. I continued watching, looking for when the user tastes from the pan (pan-b9ac55679fa82dd6). I found this occurred at 05:39 and 06:05. There were no more scoops from the can (can-9ec500193317d54e) or tastes from the pan (pan-b9ac55679fa82dd6) after this point.\n4. I added 3+2 in order to get a final answer of 5.", "referenced_object_masks": [ "can-9ec500193317d54e", "pan-b9ac55679fa82dd6" ] }, { "key": "P05-20240427-145526:196fca06bc12eafe142ccc2c6c55149c70fb9bf7", "video_id": "P05-20240427-145526", "question": "After setting down the bag of cashews, what is the 5th unique item that I touched?", "answer_choice_0": "The can", "answer_choice_1": "The pan lid", "answer_choice_2": "The silver mixing bowl", "answer_choice_3": "The pan", "answer_choice_4": "The spatula", "answer_id": 0, "answer": "The can", "question_type": [ "Action Recognition", "Counting", "Egocentric & Spatial Perception", "Object Recognition" ], "objects_of_interest": [ "spatula-56887f675176f060", "pan-b9ac55679fa82dd6", "pan lid-ae0a06184751fb96", "bowl-3cd8178bd2fb02f0", "groundnut oil-3f52eb251b9429dc", "phone-6e0c4e156e434f6e", "pot-2d7a33128fd962dd", "chopping board-79f3c268834a00ee", "scales-55387c3f04db27eb", "tin opener-d3424d3b76192b58", "bowl2-3bdbc08281d71952", "spoon2-ce1084e62e2cb20d", "plate-a205745e239f9882", "cashew nuts bag-a0ca899ca4b72881", "can-9ec500193317d54e", "spoon-74fd9250b32dfa8e", "bottle-66debb3da13697fc", "bottle3-910f720cbaf30b36", "peanut butter-7506a814b7e8b969", "lid of peanut butter-bfd4f60e94a2d678" ], "reasoning": "1. I watched the video in order to see when the user set down the bag of cashews (cashew nuts bag-a0ca899ca4b72881). I found this occurred at 03:14. The user does not handle the bag of cashews (cashew nuts bag-a0ca899ca4b72881) again after this point.\n2. Then, I set out to determine the 5th unique item that the user touched after setting down the bag of cashews (cashew nuts bag-a0ca899ca4b72881). The first item they touched after this point was the spatula (spatula-56887f675176f060) at 03:15.\n3. Then, the next item they touched was the pan (pan-b9ac55679fa82dd6), also at 03:15. This was the 2nd unique item.\n4. The 3rd unique item was touched at 03:28, which was the pan lid (pan lid-ae0a06184751fb96).\n5. The 4th unique item was touched at 03:31, which was the silver mixing bowl (bowl-3cd8178bd2fb02f0).\n6. After this, the user touches the previous items again, but nothing unique is touched until 04:04. At this time, the user touches the can (can-9ec500193317d54e). That makes the can (can-9ec500193317d54e) the final answer.", "referenced_object_masks": [ "can-9ec500193317d54e", "bowl-3cd8178bd2fb02f0", "cashew nuts bag-a0ca899ca4b72881", "pan-b9ac55679fa82dd6", "pan lid-ae0a06184751fb96", "spatula-56887f675176f060" ] }, { "key": "P05-20240427-145526:7fb800f5d059256fa41af54ecc51391526aeab4f", "video_id": "P05-20240427-145526", "question": "If I was standing and looking at the sink, what actions would I need to find the garden peas?", "answer_choice_0": "Turn 180 degrees around, step forward, turn left, and open the drawer", "answer_choice_1": "Turn 180 degrees around, step forward, turn right, and open the drawer", "answer_choice_2": "Turn 180 degrees around, bend down, and open the drawer", "answer_choice_3": "Turn left 90 degrees, reach up, and open the drawer", "answer_choice_4": "Turn right 90 degrees, take a step forward, turn left, and open the drawer", "answer_id": 0, "answer": "Turn 180 degrees around, step forward, turn left, and open the drawer", "question_type": [ "Action Recognition", "Counterfactual", "Egocentric & Spatial Perception", "Situational Awareness", "Object Recognition" ], "objects_of_interest": [ "spatula-56887f675176f060", "pan-b9ac55679fa82dd6", "pan lid-ae0a06184751fb96", "bowl-3cd8178bd2fb02f0", "groundnut oil-3f52eb251b9429dc", "phone-6e0c4e156e434f6e", "pot-2d7a33128fd962dd", "chopping board-79f3c268834a00ee", "scales-55387c3f04db27eb", "tin opener-d3424d3b76192b58", "bowl2-3bdbc08281d71952", "spoon2-ce1084e62e2cb20d", "plate-a205745e239f9882", "cashew nuts bag-a0ca899ca4b72881", "can-9ec500193317d54e", "spoon-74fd9250b32dfa8e", "bottle-66debb3da13697fc", "bottle3-910f720cbaf30b36", "peanut butter-7506a814b7e8b969", "lid of peanut butter-bfd4f60e94a2d678" ], "reasoning": "1. I watched the video in order to figure out how the user would get from the sink to the garden peas . First, I needed to find these objects. I found the sink at 02:19, when the user turns 180 degrees away from the window and walks a few steps forward. The sink is forward and on the left.\n2. I located the garden peas at 06:28. They were located inside of a freezer drawer that the user opened just a moment before at 06:27.\n3. In order to connect the two, I looked for when the sink was visible the last time before the freezer drawer was opened. I found this moment at 06:24, when the user walks in between the sink and the freezer . The freezer is to the right of the sink .\n4. From this context, I determined the user could turn 180 degrees around, walk forward, turn left, and then open the freezer drawer in order to find the garden peas .", "referenced_object_masks": [] }, { "key": "P05-20240427-145526:a870dc88a5910c19c9394308e500b34f31147e32", "video_id": "P05-20240427-145526", "question": "Approximately how long would I have placed the camera in the drawer if I had left it in there for twice as long?", "answer_choice_0": "1 minute", "answer_choice_1": "1 minute and 30 seconds", "answer_choice_2": "40 seconds", "answer_choice_3": "2 minutes", "answer_choice_4": "20 seconds", "answer_id": 0, "answer": "1 minute", "question_type": [ "Temporal Reasoning", "Numerical Reasoning", "Situational Awareness", "Action Recognition", "Counterfactual" ], "objects_of_interest": [ "spatula-56887f675176f060", "pan-b9ac55679fa82dd6", "pan lid-ae0a06184751fb96", "bowl-3cd8178bd2fb02f0", "groundnut oil-3f52eb251b9429dc", "phone-6e0c4e156e434f6e", "pot-2d7a33128fd962dd", "chopping board-79f3c268834a00ee", "scales-55387c3f04db27eb", "tin opener-d3424d3b76192b58", "bowl2-3bdbc08281d71952", "spoon2-ce1084e62e2cb20d", "plate-a205745e239f9882", "cashew nuts bag-a0ca899ca4b72881", "can-9ec500193317d54e", "spoon-74fd9250b32dfa8e", "bottle-66debb3da13697fc", "bottle3-910f720cbaf30b36", "peanut butter-7506a814b7e8b969", "lid of peanut butter-bfd4f60e94a2d678" ], "reasoning": "1. I watched the video to find when the user placed the camera inside of a drawer. I found this occurs at 06:29, when they place it inside a drawer next to a package of garden peas and shut the drawer at 06:30.\n2. Then, I waited to see when the camera was taken out of the drawer. I found this occurred at 06:59, after the user reopened the drawer at 06:57.\n3. I subtracted 06:29 from 06:59 to get a total of 30 seconds inside the drawer.\n4. I multipled this by 2 to double the total time (30x2) and came up with a total of exactly 1 minute.", "referenced_object_masks": [] }, { "key": "P05-20240427-145526:b0f3077d1d917b1f718b80d000c02cbe178cb238", "video_id": "P05-20240427-145526", "question": "After adding the last food items from the cutting board into the pan, what is the 2nd place I put the cutting board?", "answer_choice_0": "On the scale", "answer_choice_1": "On the dining table", "answer_choice_2": "Underneath a bowl", "answer_choice_3": "In the sink", "answer_choice_4": "In a cupboard", "answer_id": 2, "answer": "Underneath a bowl", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "spatula-56887f675176f060", "pan-b9ac55679fa82dd6", "pan lid-ae0a06184751fb96", "bowl-3cd8178bd2fb02f0", "groundnut oil-3f52eb251b9429dc", "phone-6e0c4e156e434f6e", "pot-2d7a33128fd962dd", "chopping board-79f3c268834a00ee", "scales-55387c3f04db27eb", "tin opener-d3424d3b76192b58", "bowl2-3bdbc08281d71952", "spoon2-ce1084e62e2cb20d", "plate-a205745e239f9882", "cashew nuts bag-a0ca899ca4b72881", "can-9ec500193317d54e", "spoon-74fd9250b32dfa8e", "bottle-66debb3da13697fc", "bottle3-910f720cbaf30b36", "peanut butter-7506a814b7e8b969", "lid of peanut butter-bfd4f60e94a2d678" ], "reasoning": "1. I watched the video in order to see when the user adds food items for the last time from the cutting board (chopping board-79f3c268834a00ee). This occurs from 01:58-01:59, when the user empties the cutting board (chopping board-79f3c268834a00ee) and stirs the food in the pan (pan-b9ac55679fa82dd6).\n2. Then, I noticed the user put down the cutting board (chopping board-79f3c268834a00ee) on the countertop to their left at 02:04. I counted this as the first instance of putting down the cutting board (chopping board-79f3c268834a00ee).\n3. Then, I found that the user picked up the cutting board (chopping board-79f3c268834a00ee) at 02:17. They then turned around and walked to the opposite side of the kitchen. I saw them pick up a small blue bowl (bowl2-3bdbc08281d71952) at 02:22.\n4. At 02:23, the user placed the cutting board (chopping board-79f3c268834a00ee) underneath the small blue bowl (bowl2-3bdbc08281d71952). This gave me the final answer of the small blue bowl (bowl2-3bdbc08281d71952) as the second place they put down the cutting board (chopping board-79f3c268834a00ee).", "referenced_object_masks": [ "chopping board-79f3c268834a00ee", "pan-b9ac55679fa82dd6", "bowl2-3bdbc08281d71952" ] }, { "key": "P05-20240427-150402:24a6ac2c7843efa6aa949838df5616dc0f92d88d", "video_id": "P05-20240427-150402", "question": "Why did I put the peanut butter down on the scale so slowly?", "answer_choice_0": "I was waiting for the scale to be ready", "answer_choice_1": "The jar the peanut butter in is delicate", "answer_choice_2": "I got distracted by something", "answer_choice_3": "The scale is delicate", "answer_choice_4": "I was waiting for the peanut butter to settle", "answer_id": 0, "answer": "I was waiting for the scale to be ready", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "State Changes", "Action Recognition" ], "objects_of_interest": [ "peanut butter-39f423911d78cea2", "scales-539952817bfd8d35", "spoon-6d169167a70ac479", "excess coconut milk-cd73c7e97f49739b", "sriracha-0df209a45ad42e22", "soy sauce-048b30ade2d50cac", "cashews-bf018e55723d186c", "large spoon-1f7771624f1e6018", "spoon2-b110dc3bdf287110", "lid of pan-b6ee4cca9f3e2b35", "pan of rice-2fdf483de5804278", "lid of cooker-4f01c0265a72216c", "pan-7ee735959f575a96", "forks-8583655194c12f3c", "new spoon-e6a8e442e94c0090", "one spoon-b915082b3339f524", "lid of rice pan-57774196184173ac", "plate-2f1291ee71f0412f" ], "reasoning": "1. I watched the video to find when the user places the peanut butter (peanut butter-39f423911d78cea2) on the scale (scales-539952817bfd8d35) and saw this finally happen at 00:10.\n2. The user appears to take 5 seconds to do this from 00:05-00:10, which is very slow for such a simple action.\n3. The only variable that changes between 00:05:00:10 is that at 00:09, the numbers on the scale's (scales-539952817bfd8d35) screen change from \"0000\" to \"0\", at which point the user immediately places the peanut butter (peanut butter-39f423911d78cea2) on the scale (scales-539952817bfd8d35) at a normal speed.\n4. Therefore, it appears that the user was waiting for the scale (scales-539952817bfd8d35) to show \"0\" before they placed the peanut butter (peanut butter-39f423911d78cea2) on it.\n5. This would indicate that they were waiting for the scale (scales-539952817bfd8d35) to be in a state of readiness for weighing or, more specifically, for the scale (scales-539952817bfd8d35) to \"zero out\".", "referenced_object_masks": [ "scales-539952817bfd8d35", "peanut butter-39f423911d78cea2" ] }, { "key": "P05-20240427-150402:7e3272ca0503a4b915b7ea119b7552a2fa266a88", "video_id": "P05-20240427-150402", "question": "How many more scoops of rice did I spoon from the pot to the plate than I spooned from the plate back into the pot?", "answer_choice_0": "3", "answer_choice_1": "2", "answer_choice_2": "5", "answer_choice_3": "1", "answer_choice_4": "4", "answer_id": 3, "answer": "1", "question_type": [ "Counting", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "peanut butter-39f423911d78cea2", "scales-539952817bfd8d35", "spoon-6d169167a70ac479", "excess coconut milk-cd73c7e97f49739b", "sriracha-0df209a45ad42e22", "soy sauce-048b30ade2d50cac", "cashews-bf018e55723d186c", "large spoon-1f7771624f1e6018", "spoon2-b110dc3bdf287110", "lid of pan-b6ee4cca9f3e2b35", "pan of rice-2fdf483de5804278", "lid of cooker-4f01c0265a72216c", "pan-7ee735959f575a96", "forks-8583655194c12f3c", "new spoon-e6a8e442e94c0090", "one spoon-b915082b3339f524", "lid of rice pan-57774196184173ac", "plate-2f1291ee71f0412f" ], "reasoning": "1. I watched the video and saw the user take the first scoop from the pot of rice (pan of rice-2fdf483de5804278) and put it onto the plate (plate-2f1291ee71f0412f) at 01:59.\n2. I then saw the user take the second scoop from the pot of rice (pan of rice-2fdf483de5804278) and put it onto the plate (plate-2f1291ee71f0412f) at 02:07.\n3. I saw the user take the third scoop from the pot of rice (pan of rice-2fdf483de5804278) and put it onto the plate (plate-2f1291ee71f0412f) at 02:10.\n4. I saw the user put 1 small scoop from the plate (plate-2f1291ee71f0412f) back in the pot of rice (pan of rice-2fdf483de5804278) at 02:20.\n5. I saw the user put another small scoop from the plate (plate-2f1291ee71f0412f) back in the pot of rice (pan of rice-2fdf483de5804278) at 02:24.\n6. Therefore, the user took 3 scoops from the pot of rice (pan of rice-2fdf483de5804278) and put them on the plate (plate-2f1291ee71f0412f), then put 2 small scoops back from the plate (plate-2f1291ee71f0412f) to the pot (pan of rice-2fdf483de5804278). \n7. Finally, I did the math of 3-2 for a total of 1 more scoop the user takes from the pot of rice (pan of rice-2fdf483de5804278) and puts on the plate (plate-2f1291ee71f0412f) than they take from the plate (plate-2f1291ee71f0412f) and put back into the pot (pan of rice-2fdf483de5804278).", "referenced_object_masks": [ "plate-2f1291ee71f0412f", "pan of rice-2fdf483de5804278" ] }, { "key": "P05-20240427-150402:865a1a71dac1a1b7bcd12c2f56662600e214d184", "video_id": "P05-20240427-150402", "question": "How many things did I weigh on the scale?", "answer_choice_0": "5", "answer_choice_1": "10", "answer_choice_2": "6", "answer_choice_3": "3", "answer_choice_4": "1", "answer_id": 0, "answer": "5", "question_type": [ "Counting", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "peanut butter-39f423911d78cea2", "scales-539952817bfd8d35", "spoon-6d169167a70ac479", "excess coconut milk-cd73c7e97f49739b", "sriracha-0df209a45ad42e22", "soy sauce-048b30ade2d50cac", "cashews-bf018e55723d186c", "large spoon-1f7771624f1e6018", "spoon2-b110dc3bdf287110", "lid of pan-b6ee4cca9f3e2b35", "pan of rice-2fdf483de5804278", "lid of cooker-4f01c0265a72216c", "pan-7ee735959f575a96", "forks-8583655194c12f3c", "new spoon-e6a8e442e94c0090", "one spoon-b915082b3339f524", "lid of rice pan-57774196184173ac", "plate-2f1291ee71f0412f" ], "reasoning": "1. I watched the video and saw the first item, the peanut butter (peanut butter-39f423911d78cea2), being weighed on the scale (scales-539952817bfd8d35) at 00:10.\n2. I then saw the second item, the coconut milk (excess coconut milk-cd73c7e97f49739b), being weighed on the scale (scales-539952817bfd8d35) at 00:19.\n3. I then saw the third item, the sriracha (sriracha-0df209a45ad42e22), being weighed on the scale (scales-539952817bfd8d35) at 00:23.\n4. I then saw the fourth item, the soy sauce (soy sauce-048b30ade2d50cac), being weighed on the scale (scales-539952817bfd8d35) at 00:27.\n5. I finally saw the last item, the bag of cashews (cashews-bf018e55723d186c), being weighed at 00:31.\n6. Therefore, 5 things were weighed on the scale.", "referenced_object_masks": [ "sriracha-0df209a45ad42e22", "scales-539952817bfd8d35", "peanut butter-39f423911d78cea2", "cashews-bf018e55723d186c", "excess coconut milk-cd73c7e97f49739b", "soy sauce-048b30ade2d50cac" ] }, { "key": "P05-20240427-150402:a64a2958d0016128634ba3ea4460cf8211807634", "video_id": "P05-20240427-150402", "question": "What was the event immediately preceding me taking the pan away from the cooker on the right?", "answer_choice_0": "The pan was steaming", "answer_choice_1": "I heard an alarm and shut it off", "answer_choice_2": "The cooker was flashing", "answer_choice_3": "I checked the clock on the wall", "answer_choice_4": "The stove turned off", "answer_id": 1, "answer": "I heard an alarm and shut it off", "question_type": [ "Object Recognition", "Event Occurence", "Cause and Effect" ], "objects_of_interest": [ "peanut butter-39f423911d78cea2", "scales-539952817bfd8d35", "spoon-6d169167a70ac479", "excess coconut milk-cd73c7e97f49739b", "sriracha-0df209a45ad42e22", "soy sauce-048b30ade2d50cac", "cashews-bf018e55723d186c", "large spoon-1f7771624f1e6018", "spoon2-b110dc3bdf287110", "lid of pan-b6ee4cca9f3e2b35", "pan of rice-2fdf483de5804278", "lid of cooker-4f01c0265a72216c", "pan-7ee735959f575a96", "forks-8583655194c12f3c", "new spoon-e6a8e442e94c0090", "one spoon-b915082b3339f524", "lid of rice pan-57774196184173ac", "plate-2f1291ee71f0412f" ], "reasoning": "1. I watched the video to find when the user takes the pan (pan of rice-2fdf483de5804278) away from the cooker (lid of cooker-4f01c0265a72216c) on the right and saw this happen at 01:22.\n2. I then rewatched the video to observe what happened immediately before this action and heard and observed that between 01:14 and 01:19, something that sounds like an alarm is going off and the user seems to be checking something in their back pocket with their right hand (they are presumably shutting off the alarm, as when it ceases at 01:19, the user takes their right hand out of their pocket shortly afterward at 01:20).\n3. This is the the event that immediately preceded the user taking the pan (pan of rice-2fdf483de5804278) away from the cooker (lid of cooker-4f01c0265a72216c) on the right at 01:22, and the sound from 01:14-01:19 is presumably an alarm/reminder for them to do so.", "referenced_object_masks": [ "pan of rice-2fdf483de5804278", "lid of cooker-4f01c0265a72216c" ] }, { "key": "P05-20240427-150402:f35cf0dc4918fada09c625590a5de60f5d980c29", "video_id": "P05-20240427-150402", "question": "What was the primary reason why I took out the spoon and held it in my hand?", "answer_choice_0": "To make sure I would weigh the coconut milk correctly", "answer_choice_1": "I was preparing to use it", "answer_choice_2": "I am right-handed", "answer_choice_3": "I was freeing up my left hand", "answer_choice_4": "To wash it", "answer_id": 0, "answer": "To make sure I would weigh the coconut milk correctly", "question_type": [ "Action Recognition", "Object Recognition" ], "objects_of_interest": [ "peanut butter-39f423911d78cea2", "scales-539952817bfd8d35", "spoon-6d169167a70ac479", "excess coconut milk-cd73c7e97f49739b", "sriracha-0df209a45ad42e22", "soy sauce-048b30ade2d50cac", "cashews-bf018e55723d186c", "large spoon-1f7771624f1e6018", "spoon2-b110dc3bdf287110", "lid of pan-b6ee4cca9f3e2b35", "pan of rice-2fdf483de5804278", "lid of cooker-4f01c0265a72216c", "pan-7ee735959f575a96", "forks-8583655194c12f3c", "new spoon-e6a8e442e94c0090", "one spoon-b915082b3339f524", "lid of rice pan-57774196184173ac", "plate-2f1291ee71f0412f" ], "reasoning": "1. I watched the video to find where the user takes out a spoon (spoon-6d169167a70ac479) and holds it in their hand and found that happens between 00:12 and 00:41.\n2. The user appears to take out the spoon (spoon-6d169167a70ac479) from the coconut milk (excess coconut milk-cd73c7e97f49739b) at 00:12 directly before they weigh the coconut milk (excess coconut milk-cd73c7e97f49739b) at 00:19.\n3. This indicates that they primarily didn't want the weight of the spoon (spoon-6d169167a70ac479) to throw off the weighing of the coconut milk (excess coconut milk-cd73c7e97f49739b) at 00:19.\n4. From that point forward, the user simply holds the spoon (spoon-6d169167a70ac479) in their right hand (while they continue to use their left hand) until they decide to go and put the spoon (spoon-6d169167a70ac479) in the sink at 00:41.", "referenced_object_masks": [ "excess coconut milk-cd73c7e97f49739b", "spoon-6d169167a70ac479" ] }, { "key": "P05-20240427-151808:36e3b67566f9d4284644638f984b80933bed3d0d", "video_id": "P05-20240427-151808", "question": "If I made a fraction symbolizing the number of spoonfuls of rice I scoop out of the pan and the number of pumps of hand soap I use, what would the fraction be?", "answer_choice_0": "4/4", "answer_choice_1": "3/2", "answer_choice_2": "4/3", "answer_choice_3": "3/3", "answer_choice_4": "2/3", "answer_id": 1, "answer": "3/2", "question_type": [ "Counterfactual", "Action Recognition", "Counting", "Object Recognition" ], "objects_of_interest": [ "dirty plate-3e5e43f63c5bd159", "spoon-ad016eec390896e3", "dirty bowl-13b4891bff322aa0", "bag of cashew nuts-dac44dfeec9f7b93", "spoon2-56e899d2e3bcabb4", "bowl-a4846d6111061fe8", "lid-55c6a094834f351f", "dirty metal bowl-a96d6c6e5c882f08", "washing up brush-297c1f2577afff07", "washing up liquid-63695cfdd8f731cb", "kettle-5ff09ec1cad3e429", "coconut milk-be09f324921b316c", "sriracha-5c4ab7ba91c6e7ca", "soy sauce-3c0aa6cd778708ec", "peanut butter-9dfe120dad7fa9de", "fork-5f86f55fd7ed1bef", "lid2-43b5e6905c3f7eb6" ], "reasoning": "1. First, I determine when the user scoops rice into a pan . This starts at 00:20, where the user scoops a spoonful (spoon2-56e899d2e3bcabb4) of rice into a pan at 00:26, 00:34, and 00:44, for a total of 3 spoonfuls (spoon2-56e899d2e3bcabb4).\n2. Next, I determine when the user pumps soap (washing up liquid-63695cfdd8f731cb) from a dispenser. The user pumps once at 01:03 and once at 01:04, for a total of 2 pumps. \n3. Therefore, I determine the fraction would be 3/2.", "referenced_object_masks": [ "washing up liquid-63695cfdd8f731cb", "spoon2-56e899d2e3bcabb4" ] }, { "key": "P05-20240427-151808:7c245e278d06b2b1de50f4f90e8fa6f4e00c8716", "video_id": "P05-20240427-151808", "question": "How would I travel in order to face the fridge if I started facing the place where I scraped the onion from the pot?", "answer_choice_0": "Turn 90 degrees left, walk straight, turn left, walk straight, turn right", "answer_choice_1": "Turn 90 degrees left, walk straight, turn right, walk straight, turn left", "answer_choice_2": "Turn around 180 degrees, walk straight, turn right, walk straight, turn left", "answer_choice_3": "Turn 90 degrees right, walk straight, turn left, walk straight, turn right", "answer_choice_4": "Turn around 180 degrees, walk straight, turn left, walk straight, turn right", "answer_id": 2, "answer": "Turn around 180 degrees, walk straight, turn right, walk straight, turn left", "question_type": [ "Counterfactual", "Egocentric & Spatial Perception", "Situational Awareness", "Object Recognition" ], "objects_of_interest": [ "dirty plate-3e5e43f63c5bd159", "spoon-ad016eec390896e3", "dirty bowl-13b4891bff322aa0", "bag of cashew nuts-dac44dfeec9f7b93", "spoon2-56e899d2e3bcabb4", "bowl-a4846d6111061fe8", "lid-55c6a094834f351f", "dirty metal bowl-a96d6c6e5c882f08", "washing up brush-297c1f2577afff07", "washing up liquid-63695cfdd8f731cb", "kettle-5ff09ec1cad3e429", "coconut milk-be09f324921b316c", "sriracha-5c4ab7ba91c6e7ca", "soy sauce-3c0aa6cd778708ec", "peanut butter-9dfe120dad7fa9de", "fork-5f86f55fd7ed1bef", "lid2-43b5e6905c3f7eb6" ], "reasoning": "1. First I looked for the user to scrape onion from a pot (bowl-a4846d6111061fe8) and found this moment from 00:20-00:49. I took note of the counter they were facing. \n2. At 00:56, I saw the user turn 180 degrees to the right and walk in the general direction of the fridge. I also took note of where the fridge was in the space in relation to the counter the user was initially in front of. \n3. Reasoning about the scene, I realized that from the user's position when they scraped the onion , they would need to turn around 180 degrees, walk straight, turn right, walk straight, and turn left in order to face the fridge.", "referenced_object_masks": [ "bowl-a4846d6111061fe8" ] }, { "key": "P05-20240427-151808:91d25e89f6fc82fff2aa6fdb8fcf225ebfb5e24f", "video_id": "P05-20240427-151808", "question": "Map my trajectory: How many turns do I make from when I first enter the kitchen to when I first arrive at the window?", "answer_choice_0": "7", "answer_choice_1": "6", "answer_choice_2": "5", "answer_choice_3": "4", "answer_choice_4": "3", "answer_id": 4, "answer": "3", "question_type": [ "Egocentric & Spatial Perception", "Situational Awareness", "Counting", "Action Recognition", "Object Recognition", "State Changes" ], "objects_of_interest": [ "dirty plate-3e5e43f63c5bd159", "spoon-ad016eec390896e3", "dirty bowl-13b4891bff322aa0", "bag of cashew nuts-dac44dfeec9f7b93", "spoon2-56e899d2e3bcabb4", "bowl-a4846d6111061fe8", "lid-55c6a094834f351f", "dirty metal bowl-a96d6c6e5c882f08", "washing up brush-297c1f2577afff07", "washing up liquid-63695cfdd8f731cb", "kettle-5ff09ec1cad3e429", "coconut milk-be09f324921b316c", "sriracha-5c4ab7ba91c6e7ca", "soy sauce-3c0aa6cd778708ec", "peanut butter-9dfe120dad7fa9de", "fork-5f86f55fd7ed1bef", "lid2-43b5e6905c3f7eb6" ], "reasoning": "1. First, I determine when the user first enters the kitchen. This occurs when the video begins at 00:01. \n2. Next, I determine when the user first stands at the window. This occurs at 00:14.\n3. After that, I rewind the video to trace the user's steps in between those 2 timestamps. \n4. I determine the user makes a left at 00:03 (1st turn), walks forward, makes a slight left at the sink at 00:08 (2nd turn), turns around at 00:10 (3rd turn), and walks forward, arriving at the window at 00:14. \n5. Therefore the total number of turns the user makes before arriving at the window is 3.", "referenced_object_masks": [] }, { "key": "P05-20240427-151808:ab52f3c01f5f90a70899540075adf2f2eea7feae", "video_id": "P05-20240427-151808", "question": "What is my trajectory through the space after scooping rice into a pan to pumping hand soap from the dispenser?", "answer_choice_0": "The user turns around, walks, makes a slight right", "answer_choice_1": "The user walks forward, makes a wide left", "answer_choice_2": "The user turns around, walks, makes a slight left", "answer_choice_3": "The user walks forward, makes a wide right", "answer_choice_4": "The user heads straight, makes 2 lefts", "answer_id": 2, "answer": "The user turns around, walks, makes a slight left", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Situational Awareness" ], "objects_of_interest": [ "dirty plate-3e5e43f63c5bd159", "spoon-ad016eec390896e3", "dirty bowl-13b4891bff322aa0", "bag of cashew nuts-dac44dfeec9f7b93", "spoon2-56e899d2e3bcabb4", "bowl-a4846d6111061fe8", "lid-55c6a094834f351f", "dirty metal bowl-a96d6c6e5c882f08", "washing up brush-297c1f2577afff07", "washing up liquid-63695cfdd8f731cb", "kettle-5ff09ec1cad3e429", "coconut milk-be09f324921b316c", "sriracha-5c4ab7ba91c6e7ca", "soy sauce-3c0aa6cd778708ec", "peanut butter-9dfe120dad7fa9de", "fork-5f86f55fd7ed1bef", "lid2-43b5e6905c3f7eb6" ], "reasoning": "1. First, I determine when the user scoops rice into a pan . This starts at 00:20 while standing at the counter facing the window. \n2. Next, I determine when the user pumps soap (washing up liquid-63695cfdd8f731cb) from a dispenser. This occurs at 01:03 while standing facing the sink on the kitchen's right. \n3. After that, I rewind the video to trace the user's steps in between those 2 timestamps. \n4. While facing the window, the user is facing away from the sink. The user turns around at 00:55, walks forward, and makes a slight left at the sink at 00:58. \n5. Therefore, the user turns around, walks, and makes a slight left.", "referenced_object_masks": [ "washing up liquid-63695cfdd8f731cb" ] }, { "key": "P05-20240427-151808:b835cf239a931d43ac2bd436afb53969a9c53668", "video_id": "P05-20240427-151808", "question": "What is the product if I multiplied the visible number of shelves where I place the lid by the number of visible spaces in the silverware drawer?", "answer_choice_0": "16", "answer_choice_1": "4", "answer_choice_2": "28", "answer_choice_3": "7", "answer_choice_4": "49", "answer_id": 2, "answer": "28", "question_type": [ "Numerical Reasoning", "Action Recognition", "Situational Awareness", "Counting", "Counterfactual", "Object Recognition" ], "objects_of_interest": [ "dirty plate-3e5e43f63c5bd159", "spoon-ad016eec390896e3", "dirty bowl-13b4891bff322aa0", "bag of cashew nuts-dac44dfeec9f7b93", "spoon2-56e899d2e3bcabb4", "bowl-a4846d6111061fe8", "lid-55c6a094834f351f", "dirty metal bowl-a96d6c6e5c882f08", "washing up brush-297c1f2577afff07", "washing up liquid-63695cfdd8f731cb", "kettle-5ff09ec1cad3e429", "coconut milk-be09f324921b316c", "sriracha-5c4ab7ba91c6e7ca", "soy sauce-3c0aa6cd778708ec", "peanut butter-9dfe120dad7fa9de", "fork-5f86f55fd7ed1bef", "lid2-43b5e6905c3f7eb6" ], "reasoning": "1. First, I locate when the user puts away the lid (lid2-43b5e6905c3f7eb6). This occurs at 01:58. Before the user turns at 01:59, there are 4 shelves visible on the top left. \n2. Next, I locate a suitable shot where the spaces inside the silverware drawer are visible. This occurs at 02:03 when the user opens the blue drawer. There are 7 vertical spaces visible. \n3. I then multiply 4x7=28.", "referenced_object_masks": [ "lid2-43b5e6905c3f7eb6" ] }, { "key": "P06-20240510-090328:35c4a1a572afe26a6b7d2bdd85819e70de414e90", "video_id": "P06-20240510-090328", "question": "What is the total weight of the blue mug, the water in the blue mug, and the empty spoon?", "answer_choice_0": "1.4", "answer_choice_1": "1.47", "answer_choice_2": "1.83", "answer_choice_3": "1.87", "answer_choice_4": "1.74", "answer_id": 2, "answer": "1.83", "question_type": [ "Action Recognition", "Cause and Effect", "Event Occurence", "Situational Awareness", "Object Recognition", "Reading", "State Changes", "Numerical Reasoning" ], "objects_of_interest": [ "mobile phone-855aa96f03ed8b37", "notebook-3b7d5b84d7793f67", "weighing balance-0c9e10067de95eb9", "pack of tomatoes-b8808d8f4a41e7f5", "nutella jar-75bca47ea7ba68a0", "plastic polythene-15d9ec35a3ccffbd", "few green chilies-4dc4fd8e43f4b497", "some coriander stems-115a7ad36b1e33bc", "milk bottle-de5c1d43f85e93ce", "cloth-a0492bf80a332788", "bottle of ketchup-0ca0a55e3d725cd3", "cardboard box-1c9700650988bae1", "wooden chopping board-70a379ce57ce937e", "coffee jar-4d7413d4358b0515", "knives block-515036253e8ca9ac", "tissue roll stand-0093fc19b7a138ae", "tomato1-f9ca8bb21f4f70c9", "onion-5f60b07c59a47a5e", "pen-3cca4fdb3c3e3082", "cap of pen-d2686cef2a10d635", "bowl-aa0b750fdd820ae5", "knife-7400790ebf897c4e", "onion half-612950ddb665573f", "onion quarter-23fab365cfbad83e", "single packet-202704534a0473c8", "plate-6557472d49f2e4e7", "tomato second half-94b905f99277b6e9", "bowl2-68ac555d465966f0", "cloth2-3a6be236b6ef5dda", "scissors-e26460a4d1899ba9", "bowl3-70f8148cf7affc4d", "spoon-1c8432b105124e9d", "frying pan-ffc6f243191a87a4", "cup-7f0a0aede4480977", "cup2-3464619ed9d09770", "cup3-ee9ee372ec02473d", "pack of salted butter-5ba09ad1ae087d01", "butter knife-02aa3a7167382bc5", "spoon2-a76edcc6a4b0886a", "cut pieces of butter-34108cb42ad7b0c7", "another piece of butter-2c6208d1bf5c6183", "piece of butter3-1be54c2d40768b2c", "spatula-fc4a2c33e67667d9", "remaining butter-d45ee8376c6d841c", "slice of butter-62e6b1fa5c0dc915", "two slices of butter-e2080d65c65236a9", "plate2-aa4415f19c6a3dcf", "plate3-1f5780017b055345", "spoon3-491c6e98882e7dc5", "butter-7d3df65e5ec9e4d4", "piece of tomato-936449f738f086a1", "cloth3-474d6724e51ae8f7", "pancake-9be88deac27ab4c1", "frying pan2-652942a83642c735", "frying pan3-59c27fb2305a8a48", "wooden spoon-fea719d3ea5092dd", "pancake2-abeee8397bde5a71", "broken piece of pancake-0171a87d5eb8447a" ], "reasoning": "1. I started by finding the weight of the cup (cup-7f0a0aede4480977) first, by homing on the weight of the cup (cup-7f0a0aede4480977) without water. At 14:46, the user lifts the cup (cup-7f0a0aede4480977) off of the scale (weighing balance-0c9e10067de95eb9) which shows us a pre-zeroed out reading of \"1.31\".\n2. The user then pours water from one cup (cup3-ee9ee372ec02473d) into the first, and then places the cup back on the scale (weighing balance-0c9e10067de95eb9), which reads out \".43\", revealing that's how much water is in the cup.\n3. At 16:46, the user places a spoon (spoon2-a76edcc6a4b0886a) onto the scale, and the readout shows \".09\".\n4. If we add the weight of the empty cup, the weight of the water in the cup, and the weight of the spoon, the equation would look like \"1.31 +.43 +.09\", which leaves us with \"1.83\" as our answer.", "referenced_object_masks": [ "cup3-ee9ee372ec02473d", "weighing balance-0c9e10067de95eb9", "spoon2-a76edcc6a4b0886a", "cup-7f0a0aede4480977" ] }, { "key": "P06-20240510-090328:736b22e0ac7e38c606377ebf9eab08361fe676e2", "video_id": "P06-20240510-090328", "question": "What would be the combined total if I added the scale's numerical values displayed each time I measure the powder in the bowl?", "answer_choice_0": "20", "answer_choice_1": "30", "answer_choice_2": "10", "answer_choice_3": "50", "answer_choice_4": "40", "answer_id": 4, "answer": "40", "question_type": [ "Object Recognition", "Action Recognition", "Reading", "Counting", "State Changes", "Numerical Reasoning", "Counterfactual" ], "objects_of_interest": [ "mobile phone-855aa96f03ed8b37", "notebook-3b7d5b84d7793f67", "weighing balance-0c9e10067de95eb9", "pack of tomatoes-b8808d8f4a41e7f5", "nutella jar-75bca47ea7ba68a0", "plastic polythene-15d9ec35a3ccffbd", "few green chilies-4dc4fd8e43f4b497", "some coriander stems-115a7ad36b1e33bc", "milk bottle-de5c1d43f85e93ce", "cloth-a0492bf80a332788", "bottle of ketchup-0ca0a55e3d725cd3", "cardboard box-1c9700650988bae1", "wooden chopping board-70a379ce57ce937e", "coffee jar-4d7413d4358b0515", "knives block-515036253e8ca9ac", "tissue roll stand-0093fc19b7a138ae", "tomato1-f9ca8bb21f4f70c9", "onion-5f60b07c59a47a5e", "pen-3cca4fdb3c3e3082", "cap of pen-d2686cef2a10d635", "bowl-aa0b750fdd820ae5", "knife-7400790ebf897c4e", "onion half-612950ddb665573f", "onion quarter-23fab365cfbad83e", "single packet-202704534a0473c8", "plate-6557472d49f2e4e7", "tomato second half-94b905f99277b6e9", "bowl2-68ac555d465966f0", "cloth2-3a6be236b6ef5dda", "scissors-e26460a4d1899ba9", "bowl3-70f8148cf7affc4d", "spoon-1c8432b105124e9d", "frying pan-ffc6f243191a87a4", "cup-7f0a0aede4480977", "cup2-3464619ed9d09770", "cup3-ee9ee372ec02473d", "pack of salted butter-5ba09ad1ae087d01", "butter knife-02aa3a7167382bc5", "spoon2-a76edcc6a4b0886a", "cut pieces of butter-34108cb42ad7b0c7", "another piece of butter-2c6208d1bf5c6183", "piece of butter3-1be54c2d40768b2c", "spatula-fc4a2c33e67667d9", "remaining butter-d45ee8376c6d841c", "slice of butter-62e6b1fa5c0dc915", "two slices of butter-e2080d65c65236a9", "plate2-aa4415f19c6a3dcf", "plate3-1f5780017b055345", "spoon3-491c6e98882e7dc5", "butter-7d3df65e5ec9e4d4", "piece of tomato-936449f738f086a1", "cloth3-474d6724e51ae8f7", "pancake-9be88deac27ab4c1", "frying pan2-652942a83642c735", "frying pan3-59c27fb2305a8a48", "wooden spoon-fea719d3ea5092dd", "pancake2-abeee8397bde5a71", "broken piece of pancake-0171a87d5eb8447a" ], "reasoning": "1. I watched the video and found the scale (weighing balance-0c9e10067de95eb9) at 00:15.\n2. I found the bowl (bowl-aa0b750fdd820ae5) at 03:43.\n3. I saw the user pour a pack of powder (single packet-202704534a0473c8) into the bowl (bowl-aa0b750fdd820ae5) on the scale (weighing balance-0c9e10067de95eb9) at 10:35-10:50.\n4. I saw the reading of the scale (weighing balance-0c9e10067de95eb9) at 10:50, which read \"20\".\n5. I saw the user pour the first bowl (bowl-aa0b750fdd820ae5) of powder into a second bowl (bowl3-70f8148cf7affc4d) at 11:44.\n6. I saw the user pour more from the pack of powder (single packet-202704534a0473c8) into the bowl (bowl-aa0b750fdd820ae5) on the scale (weighing balance-0c9e10067de95eb9) at 11:48-12:00.\n7. I saw the reading of the scale (weighing balance-0c9e10067de95eb9) at 12:00, which read \"20\".\n8. I calculated 20+20=40.\n9. Therefore, the numerical value of both measurements in the bowl (bowl-aa0b750fdd820ae5) is 40.", "referenced_object_masks": [ "bowl3-70f8148cf7affc4d", "single packet-202704534a0473c8", "bowl-aa0b750fdd820ae5", "weighing balance-0c9e10067de95eb9" ] }, { "key": "P06-20240510-090328:7fe973bc0ba01c6a7e61226bc982361105caeb1b", "video_id": "P06-20240510-090328", "question": "When I am cooking the 2nd omelet, approximately how many complete counterclockwise circles do I make around the pan with the butter on the knife?", "answer_choice_0": "5", "answer_choice_1": "7", "answer_choice_2": "2", "answer_choice_3": "9", "answer_choice_4": "4", "answer_id": 2, "answer": "2", "question_type": [ "Action Recognition", "Numerical Reasoning", "Event Occurence", "Temporal Reasoning", "Object Recognition" ], "objects_of_interest": [ "mobile phone-855aa96f03ed8b37", "notebook-3b7d5b84d7793f67", "weighing balance-0c9e10067de95eb9", "pack of tomatoes-b8808d8f4a41e7f5", "nutella jar-75bca47ea7ba68a0", "plastic polythene-15d9ec35a3ccffbd", "few green chilies-4dc4fd8e43f4b497", "some coriander stems-115a7ad36b1e33bc", "milk bottle-de5c1d43f85e93ce", "cloth-a0492bf80a332788", "bottle of ketchup-0ca0a55e3d725cd3", "cardboard box-1c9700650988bae1", "wooden chopping board-70a379ce57ce937e", "coffee jar-4d7413d4358b0515", "knives block-515036253e8ca9ac", "tissue roll stand-0093fc19b7a138ae", "tomato1-f9ca8bb21f4f70c9", "onion-5f60b07c59a47a5e", "pen-3cca4fdb3c3e3082", "cap of pen-d2686cef2a10d635", "bowl-aa0b750fdd820ae5", "knife-7400790ebf897c4e", "onion half-612950ddb665573f", "onion quarter-23fab365cfbad83e", "single packet-202704534a0473c8", "plate-6557472d49f2e4e7", "tomato second half-94b905f99277b6e9", "bowl2-68ac555d465966f0", "cloth2-3a6be236b6ef5dda", "scissors-e26460a4d1899ba9", "bowl3-70f8148cf7affc4d", "spoon-1c8432b105124e9d", "frying pan-ffc6f243191a87a4", "cup-7f0a0aede4480977", "cup2-3464619ed9d09770", "cup3-ee9ee372ec02473d", "pack of salted butter-5ba09ad1ae087d01", "butter knife-02aa3a7167382bc5", "spoon2-a76edcc6a4b0886a", "cut pieces of butter-34108cb42ad7b0c7", "another piece of butter-2c6208d1bf5c6183", "piece of butter3-1be54c2d40768b2c", "spatula-fc4a2c33e67667d9", "remaining butter-d45ee8376c6d841c", "slice of butter-62e6b1fa5c0dc915", "two slices of butter-e2080d65c65236a9", "plate2-aa4415f19c6a3dcf", "plate3-1f5780017b055345", "spoon3-491c6e98882e7dc5", "butter-7d3df65e5ec9e4d4", "piece of tomato-936449f738f086a1", "cloth3-474d6724e51ae8f7", "pancake-9be88deac27ab4c1", "frying pan2-652942a83642c735", "frying pan3-59c27fb2305a8a48", "wooden spoon-fea719d3ea5092dd", "pancake2-abeee8397bde5a71", "broken piece of pancake-0171a87d5eb8447a" ], "reasoning": "1. First, I identified when the user cooked the first pancake, which occured from 17:54 when they poured the mixture into the pan (frying pan-ffc6f243191a87a4) until they removed the pancacke (pancake-9be88deac27ab4c1) from the pan (frying pan-ffc6f243191a87a4) and placed it on the plate (plate3-1f5780017b055345) at 26:19. \n2. Next, at 27:35, I saw the user begin to cook the second pancake when they poured the mixture into the pan (frying pan-ffc6f243191a87a4). \n3. Then, at 27:59, I saw the user pick up a knife (butter knife-02aa3a7167382bc5) and scoop butter onto it at 28:03. \n4. At 28:05, I saw the user begin to make circles on the pan (frying pan-ffc6f243191a87a4) with the butter . They make nearly 3/4th a complete counterclockwise circle at 28:05-28:07, and nearly a full and 1/3rd a counterclockwise circle at 28:08-28:11, for a total of approximately 2 full counterclockwise circles.", "referenced_object_masks": [ "butter knife-02aa3a7167382bc5", "plate3-1f5780017b055345", "pancake-9be88deac27ab4c1", "frying pan-ffc6f243191a87a4" ] }, { "key": "P06-20240510-090328:8be7ef2b2d6b9bab39f4233f7ec022e19334fc2c", "video_id": "P06-20240510-090328", "question": "Where did I move the tomato to after washing it in the sink?", "answer_choice_0": "To a countertop", "answer_choice_1": "To the stovetop", "answer_choice_2": "To a tabletop", "answer_choice_3": "To a cutting board", "answer_choice_4": "To the kitchen sink", "answer_id": 3, "answer": "To a cutting board", "question_type": [ "Object Recognition", "Action Recognition", "Situational Awareness", "Temporal Reasoning", "State Changes", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "mobile phone-855aa96f03ed8b37", "notebook-3b7d5b84d7793f67", "weighing balance-0c9e10067de95eb9", "pack of tomatoes-b8808d8f4a41e7f5", "nutella jar-75bca47ea7ba68a0", "plastic polythene-15d9ec35a3ccffbd", "few green chilies-4dc4fd8e43f4b497", "some coriander stems-115a7ad36b1e33bc", "milk bottle-de5c1d43f85e93ce", "cloth-a0492bf80a332788", "bottle of ketchup-0ca0a55e3d725cd3", "cardboard box-1c9700650988bae1", "wooden chopping board-70a379ce57ce937e", "coffee jar-4d7413d4358b0515", "knives block-515036253e8ca9ac", "tissue roll stand-0093fc19b7a138ae", "tomato1-f9ca8bb21f4f70c9", "onion-5f60b07c59a47a5e", "pen-3cca4fdb3c3e3082", "cap of pen-d2686cef2a10d635", "bowl-aa0b750fdd820ae5", "knife-7400790ebf897c4e", "onion half-612950ddb665573f", "onion quarter-23fab365cfbad83e", "single packet-202704534a0473c8", "plate-6557472d49f2e4e7", "tomato second half-94b905f99277b6e9", "bowl2-68ac555d465966f0", "cloth2-3a6be236b6ef5dda", "scissors-e26460a4d1899ba9", "bowl3-70f8148cf7affc4d", "spoon-1c8432b105124e9d", "frying pan-ffc6f243191a87a4", "cup-7f0a0aede4480977", "cup2-3464619ed9d09770", "cup3-ee9ee372ec02473d", "pack of salted butter-5ba09ad1ae087d01", "butter knife-02aa3a7167382bc5", "spoon2-a76edcc6a4b0886a", "cut pieces of butter-34108cb42ad7b0c7", "another piece of butter-2c6208d1bf5c6183", "piece of butter3-1be54c2d40768b2c", "spatula-fc4a2c33e67667d9", "remaining butter-d45ee8376c6d841c", "slice of butter-62e6b1fa5c0dc915", "two slices of butter-e2080d65c65236a9", "plate2-aa4415f19c6a3dcf", "plate3-1f5780017b055345", "spoon3-491c6e98882e7dc5", "butter-7d3df65e5ec9e4d4", "piece of tomato-936449f738f086a1", "cloth3-474d6724e51ae8f7", "pancake-9be88deac27ab4c1", "frying pan2-652942a83642c735", "frying pan3-59c27fb2305a8a48", "wooden spoon-fea719d3ea5092dd", "pancake2-abeee8397bde5a71", "broken piece of pancake-0171a87d5eb8447a" ], "reasoning": "1. I watched the video and found the tomato (tomato1-f9ca8bb21f4f70c9) at 02:06.\n2. I saw the user wash the tomato (tomato1-f9ca8bb21f4f70c9) in the sink at 06:00-06:07.\n3. After 06:07, I saw the user move the tomato (tomato1-f9ca8bb21f4f70c9) to a cutting board (wooden chopping board-70a379ce57ce937e).\n4. Therefore, the user moved the tomato (tomato1-f9ca8bb21f4f70c9) to a cutting board (wooden chopping board-70a379ce57ce937e) after washing the tomato (tomato1-f9ca8bb21f4f70c9) in the sink.", "referenced_object_masks": [ "wooden chopping board-70a379ce57ce937e", "tomato1-f9ca8bb21f4f70c9" ] }, { "key": "P06-20240510-090328:935fc0c622abb81086cad5cf05413b9c8394b446", "video_id": "P06-20240510-090328", "question": "What type of actions do I use to spread the butter between 18:45 - 18:49, from the perspective of the paper towel roll in view?", "answer_choice_0": "Top to bottom vertical motions", "answer_choice_1": "Only counter-clockwise motions", "answer_choice_2": "Clockwise & counter-clockwise motions", "answer_choice_3": "Sprinkling motions", "answer_choice_4": "Left to right motions", "answer_id": 2, "answer": "Clockwise & counter-clockwise motions", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness", "Temporal Reasoning" ], "objects_of_interest": [ "mobile phone-855aa96f03ed8b37", "notebook-3b7d5b84d7793f67", "weighing balance-0c9e10067de95eb9", "pack of tomatoes-b8808d8f4a41e7f5", "nutella jar-75bca47ea7ba68a0", "plastic polythene-15d9ec35a3ccffbd", "few green chilies-4dc4fd8e43f4b497", "some coriander stems-115a7ad36b1e33bc", "milk bottle-de5c1d43f85e93ce", "cloth-a0492bf80a332788", "bottle of ketchup-0ca0a55e3d725cd3", "cardboard box-1c9700650988bae1", "wooden chopping board-70a379ce57ce937e", "coffee jar-4d7413d4358b0515", "knives block-515036253e8ca9ac", "tissue roll stand-0093fc19b7a138ae", "tomato1-f9ca8bb21f4f70c9", "onion-5f60b07c59a47a5e", "pen-3cca4fdb3c3e3082", "cap of pen-d2686cef2a10d635", "bowl-aa0b750fdd820ae5", "knife-7400790ebf897c4e", "onion half-612950ddb665573f", "onion quarter-23fab365cfbad83e", "single packet-202704534a0473c8", "plate-6557472d49f2e4e7", "tomato second half-94b905f99277b6e9", "bowl2-68ac555d465966f0", "cloth2-3a6be236b6ef5dda", "scissors-e26460a4d1899ba9", "bowl3-70f8148cf7affc4d", "spoon-1c8432b105124e9d", "frying pan-ffc6f243191a87a4", "cup-7f0a0aede4480977", "cup2-3464619ed9d09770", "cup3-ee9ee372ec02473d", "pack of salted butter-5ba09ad1ae087d01", "butter knife-02aa3a7167382bc5", "spoon2-a76edcc6a4b0886a", "cut pieces of butter-34108cb42ad7b0c7", "another piece of butter-2c6208d1bf5c6183", "piece of butter3-1be54c2d40768b2c", "spatula-fc4a2c33e67667d9", "remaining butter-d45ee8376c6d841c", "slice of butter-62e6b1fa5c0dc915", "two slices of butter-e2080d65c65236a9", "plate2-aa4415f19c6a3dcf", "plate3-1f5780017b055345", "spoon3-491c6e98882e7dc5", "butter-7d3df65e5ec9e4d4", "piece of tomato-936449f738f086a1", "cloth3-474d6724e51ae8f7", "pancake-9be88deac27ab4c1", "frying pan2-652942a83642c735", "frying pan3-59c27fb2305a8a48", "wooden spoon-fea719d3ea5092dd", "pancake2-abeee8397bde5a71", "broken piece of pancake-0171a87d5eb8447a" ], "reasoning": "1. I found the initial timestamp at 18:45, and saw that the user had the butter (butter-7d3df65e5ec9e4d4) in their right hand.\n2. I then found the roll of paper towels (tissue roll stand-0093fc19b7a138ae), and then roughly inverted whatever perspective I'd see from the user's perspective.\n3. I watched the user's hand spread butter (butter-7d3df65e5ec9e4d4) around the circular pan (frying pan-ffc6f243191a87a4) toward the left, before then quickly completing a full circle in the opposite direction.\n4. Because the user went in both directions, the answer doesn't change from either the user, or the roll of paper towel's (tissue roll stand-0093fc19b7a138ae) perspectives. It's both clockwise, and counter-clockwise.", "referenced_object_masks": [ "frying pan-ffc6f243191a87a4", "tissue roll stand-0093fc19b7a138ae", "butter-7d3df65e5ec9e4d4" ] }, { "key": "P06-20240510-090328:940bc855faf0b25ad6b4828e1e46f7337fd93013", "video_id": "P06-20240510-090328", "question": "What was the second item I put onto the first white plate I grab?", "answer_choice_0": "Diced coriander", "answer_choice_1": "Diced tomatoes", "answer_choice_2": "A silver knife", "answer_choice_3": "Diced onions", "answer_choice_4": "A flour tortilla", "answer_id": 1, "answer": "Diced tomatoes", "question_type": [ "Object Recognition", "Counting", "Action Recognition", "State Changes" ], "objects_of_interest": [ "mobile phone-855aa96f03ed8b37", "notebook-3b7d5b84d7793f67", "weighing balance-0c9e10067de95eb9", "pack of tomatoes-b8808d8f4a41e7f5", "nutella jar-75bca47ea7ba68a0", "plastic polythene-15d9ec35a3ccffbd", "few green chilies-4dc4fd8e43f4b497", "some coriander stems-115a7ad36b1e33bc", "milk bottle-de5c1d43f85e93ce", "cloth-a0492bf80a332788", "bottle of ketchup-0ca0a55e3d725cd3", "cardboard box-1c9700650988bae1", "wooden chopping board-70a379ce57ce937e", "coffee jar-4d7413d4358b0515", "knives block-515036253e8ca9ac", "tissue roll stand-0093fc19b7a138ae", "tomato1-f9ca8bb21f4f70c9", "onion-5f60b07c59a47a5e", "pen-3cca4fdb3c3e3082", "cap of pen-d2686cef2a10d635", "bowl-aa0b750fdd820ae5", "knife-7400790ebf897c4e", "onion half-612950ddb665573f", "onion quarter-23fab365cfbad83e", "single packet-202704534a0473c8", "plate-6557472d49f2e4e7", "tomato second half-94b905f99277b6e9", "bowl2-68ac555d465966f0", "cloth2-3a6be236b6ef5dda", "scissors-e26460a4d1899ba9", "bowl3-70f8148cf7affc4d", "spoon-1c8432b105124e9d", "frying pan-ffc6f243191a87a4", "cup-7f0a0aede4480977", "cup2-3464619ed9d09770", "cup3-ee9ee372ec02473d", "pack of salted butter-5ba09ad1ae087d01", "butter knife-02aa3a7167382bc5", "spoon2-a76edcc6a4b0886a", "cut pieces of butter-34108cb42ad7b0c7", "another piece of butter-2c6208d1bf5c6183", "piece of butter3-1be54c2d40768b2c", "spatula-fc4a2c33e67667d9", "remaining butter-d45ee8376c6d841c", "slice of butter-62e6b1fa5c0dc915", "two slices of butter-e2080d65c65236a9", "plate2-aa4415f19c6a3dcf", "plate3-1f5780017b055345", "spoon3-491c6e98882e7dc5", "butter-7d3df65e5ec9e4d4", "piece of tomato-936449f738f086a1", "cloth3-474d6724e51ae8f7", "pancake-9be88deac27ab4c1", "frying pan2-652942a83642c735", "frying pan3-59c27fb2305a8a48", "wooden spoon-fea719d3ea5092dd", "pancake2-abeee8397bde5a71", "broken piece of pancake-0171a87d5eb8447a" ], "reasoning": "1. I watched the video and found the white plate (plate-6557472d49f2e4e7) at 05:45.\n2. I counted each item the user put onto the white plate (plate-6557472d49f2e4e7).\n3. At 07:33, I saw the user put the first item onto the white plate (plate-6557472d49f2e4e7), which were diced onions .\n4. At 08:14, I saw the user put the second item onto the white plate (plate-6557472d49f2e4e7), which were diced tomatoes (piece of tomato-936449f738f086a1).\n5. Therefore, the second item the user put onto the white plate (plate-6557472d49f2e4e7) was diced tomatoes.", "referenced_object_masks": [ "plate-6557472d49f2e4e7", "piece of tomato-936449f738f086a1" ] }, { "key": "P06-20240510-090328:9fd5eb17dc4e24696b7a4d2124a5e77eb4b8f4ee", "video_id": "P06-20240510-090328", "question": "Not counting the pen, what is the last item that I touched before I touched the notebook for the third time?", "answer_choice_0": "Onion", "answer_choice_1": "Tomato", "answer_choice_2": "Cupboard", "answer_choice_3": "Weighing scale", "answer_choice_4": "Drawer", "answer_id": 2, "answer": "Cupboard", "question_type": [ "Object Recognition", "Egocentric & Spatial Perception", "Temporal Reasoning" ], "objects_of_interest": [ "mobile phone-855aa96f03ed8b37", "notebook-3b7d5b84d7793f67", "weighing balance-0c9e10067de95eb9", "pack of tomatoes-b8808d8f4a41e7f5", "nutella jar-75bca47ea7ba68a0", "plastic polythene-15d9ec35a3ccffbd", "few green chilies-4dc4fd8e43f4b497", "some coriander stems-115a7ad36b1e33bc", "milk bottle-de5c1d43f85e93ce", "cloth-a0492bf80a332788", "bottle of ketchup-0ca0a55e3d725cd3", "cardboard box-1c9700650988bae1", "wooden chopping board-70a379ce57ce937e", "coffee jar-4d7413d4358b0515", "knives block-515036253e8ca9ac", "tissue roll stand-0093fc19b7a138ae", "tomato1-f9ca8bb21f4f70c9", "onion-5f60b07c59a47a5e", "pen-3cca4fdb3c3e3082", "cap of pen-d2686cef2a10d635", "bowl-aa0b750fdd820ae5", "knife-7400790ebf897c4e", "onion half-612950ddb665573f", "onion quarter-23fab365cfbad83e", "single packet-202704534a0473c8", "plate-6557472d49f2e4e7", "tomato second half-94b905f99277b6e9", "bowl2-68ac555d465966f0", "cloth2-3a6be236b6ef5dda", "scissors-e26460a4d1899ba9", "bowl3-70f8148cf7affc4d", "spoon-1c8432b105124e9d", "frying pan-ffc6f243191a87a4", "cup-7f0a0aede4480977", "cup2-3464619ed9d09770", "cup3-ee9ee372ec02473d", "pack of salted butter-5ba09ad1ae087d01", "butter knife-02aa3a7167382bc5", "spoon2-a76edcc6a4b0886a", "cut pieces of butter-34108cb42ad7b0c7", "another piece of butter-2c6208d1bf5c6183", "piece of butter3-1be54c2d40768b2c", "spatula-fc4a2c33e67667d9", "remaining butter-d45ee8376c6d841c", "slice of butter-62e6b1fa5c0dc915", "two slices of butter-e2080d65c65236a9", "plate2-aa4415f19c6a3dcf", "plate3-1f5780017b055345", "spoon3-491c6e98882e7dc5", "butter-7d3df65e5ec9e4d4", "piece of tomato-936449f738f086a1", "cloth3-474d6724e51ae8f7", "pancake-9be88deac27ab4c1", "frying pan2-652942a83642c735", "frying pan3-59c27fb2305a8a48", "wooden spoon-fea719d3ea5092dd", "pancake2-abeee8397bde5a71", "broken piece of pancake-0171a87d5eb8447a" ], "reasoning": "1. I watched the video to determine when the user touched the notebook (notebook-3b7d5b84d7793f67). I found the first instance of this at 00:15, when the user places their hand on the notebook (notebook-3b7d5b84d7793f67) and adjusts its spot on the counter slightly.\n2. Then, I saw that the user moved the notebook (notebook-3b7d5b84d7793f67) for a 2nd time at 01:38. At this time, the user moved the notebook (notebook-3b7d5b84d7793f67) in order to create more room on the counter for the (weighing balance-0c9e10067de95eb9).\n3. Then, I saw that the user moved the notebook (notebook-3b7d5b84d7793f67) for a 3rd time at 02:46. At this time, the user moved the notebook (notebook-3b7d5b84d7793f67) in order to write on it with the pen (pen-3cca4fdb3c3e3082).\n4. Since this was the 3rd instance, I went back to see what the final object was that the user touched before 02:46, discounting the pen (pen-3cca4fdb3c3e3082). From 02:38-02:42, I saw that the user opened a cupboard below the counter. I determined that was the final answer.", "referenced_object_masks": [ "pen-3cca4fdb3c3e3082", "notebook-3b7d5b84d7793f67", "weighing balance-0c9e10067de95eb9" ] }, { "key": "P06-20240510-090328:acebc2b0b5bbb82fca7590b4ac142af0bc01c853", "video_id": "P06-20240510-090328", "question": "If it took me twice as long to dice a whole tomato from start to finish, approximately how long would it have taken me to dice 3 tomatoes?", "answer_choice_0": "3 minutes", "answer_choice_1": "2 minutes", "answer_choice_2": "4 minutes", "answer_choice_3": "6 minutes", "answer_choice_4": "1 minute", "answer_id": 3, "answer": "6 minutes", "question_type": [ "Object Recognition", "Action Recognition", "State Changes", "Temporal Reasoning", "Numerical Reasoning", "Counterfactual" ], "objects_of_interest": [ "mobile phone-855aa96f03ed8b37", "notebook-3b7d5b84d7793f67", "weighing balance-0c9e10067de95eb9", "pack of tomatoes-b8808d8f4a41e7f5", "nutella jar-75bca47ea7ba68a0", "plastic polythene-15d9ec35a3ccffbd", "few green chilies-4dc4fd8e43f4b497", "some coriander stems-115a7ad36b1e33bc", "milk bottle-de5c1d43f85e93ce", "cloth-a0492bf80a332788", "bottle of ketchup-0ca0a55e3d725cd3", "cardboard box-1c9700650988bae1", "wooden chopping board-70a379ce57ce937e", "coffee jar-4d7413d4358b0515", "knives block-515036253e8ca9ac", "tissue roll stand-0093fc19b7a138ae", "tomato1-f9ca8bb21f4f70c9", "onion-5f60b07c59a47a5e", "pen-3cca4fdb3c3e3082", "cap of pen-d2686cef2a10d635", "bowl-aa0b750fdd820ae5", "knife-7400790ebf897c4e", "onion half-612950ddb665573f", "onion quarter-23fab365cfbad83e", "single packet-202704534a0473c8", "plate-6557472d49f2e4e7", "tomato second half-94b905f99277b6e9", "bowl2-68ac555d465966f0", "cloth2-3a6be236b6ef5dda", "scissors-e26460a4d1899ba9", "bowl3-70f8148cf7affc4d", "spoon-1c8432b105124e9d", "frying pan-ffc6f243191a87a4", "cup-7f0a0aede4480977", "cup2-3464619ed9d09770", "cup3-ee9ee372ec02473d", "pack of salted butter-5ba09ad1ae087d01", "butter knife-02aa3a7167382bc5", "spoon2-a76edcc6a4b0886a", "cut pieces of butter-34108cb42ad7b0c7", "another piece of butter-2c6208d1bf5c6183", "piece of butter3-1be54c2d40768b2c", "spatula-fc4a2c33e67667d9", "remaining butter-d45ee8376c6d841c", "slice of butter-62e6b1fa5c0dc915", "two slices of butter-e2080d65c65236a9", "plate2-aa4415f19c6a3dcf", "plate3-1f5780017b055345", "spoon3-491c6e98882e7dc5", "butter-7d3df65e5ec9e4d4", "piece of tomato-936449f738f086a1", "cloth3-474d6724e51ae8f7", "pancake-9be88deac27ab4c1", "frying pan2-652942a83642c735", "frying pan3-59c27fb2305a8a48", "wooden spoon-fea719d3ea5092dd", "pancake2-abeee8397bde5a71", "broken piece of pancake-0171a87d5eb8447a" ], "reasoning": "1. I watched the video and found the tomato (tomato1-f9ca8bb21f4f70c9) at 02:06.\n2. I saw the user start dicing the whole tomato (tomato1-f9ca8bb21f4f70c9) at 07:44.\n3. I saw the user finish dicing the whole tomato (tomato1-f9ca8bb21f4f70c9) at 08:42.\n4. I calculated 08:42-07:44=00:58.\n5. I calculated 00:58*2=01:56.\n6. I calculated 01:56*3=05:48..\n7. Therefore, if it took the user twice as long to dice a whole tomato (tomato1-f9ca8bb21f4f70c9) from start to finish, it would have taken them 05:48, or approximately 6 minutes to dice 3 tomatoes.", "referenced_object_masks": [ "tomato1-f9ca8bb21f4f70c9" ] }, { "key": "P06-20240510-090328:dc426aa67a287aeab6986f18f80fc4349e923977", "video_id": "P06-20240510-090328", "question": "Where would the sink have been in relation to me if it was on the opposite side of the kitchen while I was dicing the onion?", "answer_choice_0": "Behind, to my left", "answer_choice_1": "Behind, to my right", "answer_choice_2": "In front, to my right", "answer_choice_3": "In front, to my left", "answer_choice_4": "Directly behind", "answer_id": 2, "answer": "In front, to my right", "question_type": [ "Object Recognition", "Action Recognition", "Situational Awareness", "Egocentric & Spatial Perception", "Counterfactual" ], "objects_of_interest": [ "mobile phone-855aa96f03ed8b37", "notebook-3b7d5b84d7793f67", "weighing balance-0c9e10067de95eb9", "pack of tomatoes-b8808d8f4a41e7f5", "nutella jar-75bca47ea7ba68a0", "plastic polythene-15d9ec35a3ccffbd", "few green chilies-4dc4fd8e43f4b497", "some coriander stems-115a7ad36b1e33bc", "milk bottle-de5c1d43f85e93ce", "cloth-a0492bf80a332788", "bottle of ketchup-0ca0a55e3d725cd3", "cardboard box-1c9700650988bae1", "wooden chopping board-70a379ce57ce937e", "coffee jar-4d7413d4358b0515", "knives block-515036253e8ca9ac", "tissue roll stand-0093fc19b7a138ae", "tomato1-f9ca8bb21f4f70c9", "onion-5f60b07c59a47a5e", "pen-3cca4fdb3c3e3082", "cap of pen-d2686cef2a10d635", "bowl-aa0b750fdd820ae5", "knife-7400790ebf897c4e", "onion half-612950ddb665573f", "onion quarter-23fab365cfbad83e", "single packet-202704534a0473c8", "plate-6557472d49f2e4e7", "tomato second half-94b905f99277b6e9", "bowl2-68ac555d465966f0", "cloth2-3a6be236b6ef5dda", "scissors-e26460a4d1899ba9", "bowl3-70f8148cf7affc4d", "spoon-1c8432b105124e9d", "frying pan-ffc6f243191a87a4", "cup-7f0a0aede4480977", "cup2-3464619ed9d09770", "cup3-ee9ee372ec02473d", "pack of salted butter-5ba09ad1ae087d01", "butter knife-02aa3a7167382bc5", "spoon2-a76edcc6a4b0886a", "cut pieces of butter-34108cb42ad7b0c7", "another piece of butter-2c6208d1bf5c6183", "piece of butter3-1be54c2d40768b2c", "spatula-fc4a2c33e67667d9", "remaining butter-d45ee8376c6d841c", "slice of butter-62e6b1fa5c0dc915", "two slices of butter-e2080d65c65236a9", "plate2-aa4415f19c6a3dcf", "plate3-1f5780017b055345", "spoon3-491c6e98882e7dc5", "butter-7d3df65e5ec9e4d4", "piece of tomato-936449f738f086a1", "cloth3-474d6724e51ae8f7", "pancake-9be88deac27ab4c1", "frying pan2-652942a83642c735", "frying pan3-59c27fb2305a8a48", "wooden spoon-fea719d3ea5092dd", "pancake2-abeee8397bde5a71", "broken piece of pancake-0171a87d5eb8447a" ], "reasoning": "1. I watched the video and found the onion (onion-5f60b07c59a47a5e) at 02:13.\n2. I saw the user dicing the onion (onion quarter-23fab365cfbad83e) at 06:55-07:30.\n3. Around 05:55, I saw the user turn around and bring a tomato (tomato1-f9ca8bb21f4f70c9) and some coriander (some coriander stems-115a7ad36b1e33bc) to the sink. The user turns around and moves slightly to the left to reach it, and they do this from the same place they dice the onion (onion quarter-23fab365cfbad83e).\n4. Given this informaton, I realize that, were the sink to be in the opposite side of the kitchen, it would be in front and to the right of the user.\n5. Therefore, in front and to the right is my answer.", "referenced_object_masks": [ "some coriander stems-115a7ad36b1e33bc", "onion quarter-23fab365cfbad83e", "onion-5f60b07c59a47a5e", "tomato1-f9ca8bb21f4f70c9" ] }, { "key": "P06-20240510-090328:e495318491e097e64a92c8aad4fea21c10af1d2c", "video_id": "P06-20240510-090328", "question": "If the object being cut was designated a nomenclature that had the formula of \"'object' 'polynomial'\", where the object directly correlated to the object being cut, and polynomial designation directly correlated to the number of cuts performed between 07:43 and 07:58, what would the nomenclature result in?", "answer_choice_0": "Butter heptic", "answer_choice_1": "Tomato quintic", "answer_choice_2": "Tomato nonic", "answer_choice_3": "Onion cubed", "answer_choice_4": "Onion squared", "answer_id": 1, "answer": "Tomato quintic", "question_type": [ "Action Recognition", "Counting", "Numerical Reasoning", "Object Recognition", "Situational Awareness", "Temporal Reasoning", "State Changes" ], "objects_of_interest": [ "mobile phone-855aa96f03ed8b37", "notebook-3b7d5b84d7793f67", "weighing balance-0c9e10067de95eb9", "pack of tomatoes-b8808d8f4a41e7f5", "nutella jar-75bca47ea7ba68a0", "plastic polythene-15d9ec35a3ccffbd", "few green chilies-4dc4fd8e43f4b497", "some coriander stems-115a7ad36b1e33bc", "milk bottle-de5c1d43f85e93ce", "cloth-a0492bf80a332788", "bottle of ketchup-0ca0a55e3d725cd3", "cardboard box-1c9700650988bae1", "wooden chopping board-70a379ce57ce937e", "coffee jar-4d7413d4358b0515", "knives block-515036253e8ca9ac", "tissue roll stand-0093fc19b7a138ae", "tomato1-f9ca8bb21f4f70c9", "onion-5f60b07c59a47a5e", "pen-3cca4fdb3c3e3082", "cap of pen-d2686cef2a10d635", "bowl-aa0b750fdd820ae5", "knife-7400790ebf897c4e", "onion half-612950ddb665573f", "onion quarter-23fab365cfbad83e", "single packet-202704534a0473c8", "plate-6557472d49f2e4e7", "tomato second half-94b905f99277b6e9", "bowl2-68ac555d465966f0", "cloth2-3a6be236b6ef5dda", "scissors-e26460a4d1899ba9", "bowl3-70f8148cf7affc4d", "spoon-1c8432b105124e9d", "frying pan-ffc6f243191a87a4", "cup-7f0a0aede4480977", "cup2-3464619ed9d09770", "cup3-ee9ee372ec02473d", "pack of salted butter-5ba09ad1ae087d01", "butter knife-02aa3a7167382bc5", "spoon2-a76edcc6a4b0886a", "cut pieces of butter-34108cb42ad7b0c7", "another piece of butter-2c6208d1bf5c6183", "piece of butter3-1be54c2d40768b2c", "spatula-fc4a2c33e67667d9", "remaining butter-d45ee8376c6d841c", "slice of butter-62e6b1fa5c0dc915", "two slices of butter-e2080d65c65236a9", "plate2-aa4415f19c6a3dcf", "plate3-1f5780017b055345", "spoon3-491c6e98882e7dc5", "butter-7d3df65e5ec9e4d4", "piece of tomato-936449f738f086a1", "cloth3-474d6724e51ae8f7", "pancake-9be88deac27ab4c1", "frying pan2-652942a83642c735", "frying pan3-59c27fb2305a8a48", "wooden spoon-fea719d3ea5092dd", "pancake2-abeee8397bde5a71", "broken piece of pancake-0171a87d5eb8447a" ], "reasoning": "1. I found the initial timestamp at 07:43, and understood that the object being cut at that marker was a tomato (tomato1-f9ca8bb21f4f70c9).\n2. I watched the user cut the tomato (tomato1-f9ca8bb21f4f70c9) on the cutting board (wooden chopping board-70a379ce57ce937e) 5 times between the stamps 07:43 and 07:58.\n3. I then translated the polynomial name that correlated to the number of cuts made. 5 cuts gives us the designation of \"quintic\", which means the final nomenclature would result in \"tomato quintic\".", "referenced_object_masks": [ "wooden chopping board-70a379ce57ce937e", "tomato1-f9ca8bb21f4f70c9" ] }, { "key": "P06-20240510-094438:087f596aa11329a659185bea61b706f7ec41841a", "video_id": "P06-20240510-094438", "question": "After I open the cabinet door and pull items out and put them back in, how many items touched by me are bottles?", "answer_choice_0": "4", "answer_choice_1": "1", "answer_choice_2": "2", "answer_choice_3": "3", "answer_choice_4": "5", "answer_id": 2, "answer": "2", "question_type": [ "Counting" ], "objects_of_interest": [ "spatula-c46dcb69664dfae7", "frying pan-142f6c6ee0d554c1", "phone-4f08a39f9a815ed0", "bottle-f8ebcee9cf407968", "bottle2-c006e8e7071fab5f", "plastic box-cb296bbf2e4caacf", "jar of coffee granules-58b462a11b0c404c", "broken piece of pancake-ce1e6b69fed93a18" ], "reasoning": "1. I noticed the user walk and open the cabinet at 02:07.\n2. I noticed the user pull out 2 bottles , 1 container , and one jar ending at 02:17.\n3. I noticed the user put 2 bottles back at 02:21.", "referenced_object_masks": [] }, { "key": "P06-20240510-094438:1569f0359b0e1c35b6536085fa1f32e1e3cdb2f5", "video_id": "P06-20240510-094438", "question": "After I pat the food in the pan with the spatula 13 times in one go, what knob on the stove do I touch?", "answer_choice_0": "Second-to-leftmost knob", "answer_choice_1": "Leftmost knob", "answer_choice_2": "Rightmost knob", "answer_choice_3": "Second-to-rightmost knob", "answer_choice_4": "Middle knob", "answer_id": 1, "answer": "Leftmost knob", "question_type": [ "Situational Awareness" ], "objects_of_interest": [ "spatula-c46dcb69664dfae7", "frying pan-142f6c6ee0d554c1", "phone-4f08a39f9a815ed0", "bottle-f8ebcee9cf407968", "bottle2-c006e8e7071fab5f", "plastic box-cb296bbf2e4caacf", "jar of coffee granules-58b462a11b0c404c", "broken piece of pancake-ce1e6b69fed93a18" ], "reasoning": "1. I saw the user grab the spatula (spatula-c46dcb69664dfae7) at 00:57.\n2. I saw the user pat the food in the pan with the spatula (spatula-c46dcb69664dfae7) from 00:57 to 01:15: at 00:58, 00:58. 00:59, 00:59-01:00, 01:00, 01:01, 01:01, 01:01-01:02, 01:02, 01:02-01:03, 01:03, 01:03, and 01:04. There are no interruptions in between these times, so the user pats the food 13 times in one go here.\n3. I saw the user grab the leftmost knob at 01:06.\n4. Therefore, the answer is leftmost knob.", "referenced_object_masks": [ "spatula-c46dcb69664dfae7" ] }, { "key": "P06-20240510-094438:299f99086e8a75fef184edb3e914d97c657cc8a4", "video_id": "P06-20240510-094438", "question": "What is the sum of the numbers on my phone when it displays the time?", "answer_choice_0": "18", "answer_choice_1": "20", "answer_choice_2": "19", "answer_choice_3": "21", "answer_choice_4": "17", "answer_id": 0, "answer": "18", "question_type": [ "Counting", "Numerical Reasoning" ], "objects_of_interest": [ "spatula-c46dcb69664dfae7", "frying pan-142f6c6ee0d554c1", "phone-4f08a39f9a815ed0", "bottle-f8ebcee9cf407968", "bottle2-c006e8e7071fab5f", "plastic box-cb296bbf2e4caacf", "jar of coffee granules-58b462a11b0c404c", "broken piece of pancake-ce1e6b69fed93a18" ], "reasoning": "1. I saw the user grab their phone at 00:39.\n2. I saw the user hold their phone out at 00:42. \n3. I saw the time say 09:45 on the phone at 00:43.\n4. I added 0+9+4+5 to reach a sum of 18.", "referenced_object_masks": [] }, { "key": "P06-20240510-094438:9417a2e7fc7979b81c6e50dc59865fc90d645405", "video_id": "P06-20240510-094438", "question": "Before I touch the food in the pan with my left hand for the first time, how many times is the spatula pressed on the food?", "answer_choice_0": "Five", "answer_choice_1": "Two", "answer_choice_2": "Three", "answer_choice_3": "One", "answer_choice_4": "Four", "answer_id": 2, "answer": "Three", "question_type": [ "Counting" ], "objects_of_interest": [ "spatula-c46dcb69664dfae7", "frying pan-142f6c6ee0d554c1", "phone-4f08a39f9a815ed0", "bottle-f8ebcee9cf407968", "bottle2-c006e8e7071fab5f", "plastic box-cb296bbf2e4caacf", "jar of coffee granules-58b462a11b0c404c", "broken piece of pancake-ce1e6b69fed93a18" ], "reasoning": "1. I watched for the user to touch food in the pan (frying pan-142f6c6ee0d554c1) with their left hand.\n2. I noticed the user touch the food in the pan (frying pan-142f6c6ee0d554c1) with their left hand at 00:20.\n3. I went back to see how many times the user patted the food with the spatula (spatula-c46dcb69664dfae7) before touching it with their hand and counted three times from 00:18-00:19.", "referenced_object_masks": [ "frying pan-142f6c6ee0d554c1", "spatula-c46dcb69664dfae7" ] }, { "key": "P06-20240510-094438:ed63f12927a5def979ba7343210eaa4b3d176762", "video_id": "P06-20240510-094438", "question": "How many more spoons are in the metal container to the left of the stove than in the drying rack to the right of the sink?", "answer_choice_0": "3", "answer_choice_1": "1", "answer_choice_2": "2", "answer_choice_3": "4", "answer_choice_4": "0", "answer_id": 2, "answer": "2", "question_type": [ "Counting", "Numerical Reasoning" ], "objects_of_interest": [ "spatula-c46dcb69664dfae7", "frying pan-142f6c6ee0d554c1", "phone-4f08a39f9a815ed0", "bottle-f8ebcee9cf407968", "bottle2-c006e8e7071fab5f", "plastic box-cb296bbf2e4caacf", "jar of coffee granules-58b462a11b0c404c", "broken piece of pancake-ce1e6b69fed93a18" ], "reasoning": "1. I saw the user approach the counter at 00:55 and located the metal container. It contains 3 visible spoons , two wooden ones and one metal one . \n2. At 00:51, the user turns towards the sink where the drying rack is visible to its right. I saw 1 spoon resting inside. \n4. I subtracted the 3 spoons in the metal container minus the 1 spoon in the drying rack , which totals 2.", "referenced_object_masks": [] }, { "key": "P06-20240510-100047:2a4d7f887f73d6466a5f4ec77fc7367aa24fc1e6", "video_id": "P06-20240510-100047", "question": "If I divide the total time to fill up the tea kettle with water by the weight of 1 teaspoon of brown sugar, what number do I get? Round to the nearest tenth if needed.", "answer_choice_0": "16", "answer_choice_1": "0.4", "answer_choice_2": "4", "answer_choice_3": "400", "answer_choice_4": "40", "answer_id": 3, "answer": "400", "question_type": [ "Action Recognition", "Event Occurence", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "spatula-82446961e238813a", "pan-63377fdc2ca0cf38", "pancake-596a86119069a8ce", "saucepan-e305cb0c04ea3395", "plate-7b17b03d6af35c5b", "electric kettle-4094d10d4bebd60a", "mobile phone-e5a1845d7d0af125", "spoon-a4f555f454eaab64", "jar1-08b27da5f0785d38", "jar2-72a55966f700324d", "coffee cup-c1bbaf14cbf426ba", "coffee cup2-97a07d79fbcc1bf9", "milk bottle-6277236de4df02f3", "lid of milk carton-33831cd9baafc45e", "pen-812f273fd902c6c4", "notebook-d6de4bcee841a486", "lid of jar-58fa775a69d5366c", "weighing balance-22c112cdca74fbc4", "cup3-1c73bc29120d5380", "milk frother whole set-565f3cb8b9e0dbca", "milk frother-1354436915ebbea5", "frother stand-fa86c0a2e464a9e4", "plate3-57f12424060e8757", "saucepan lid-d98c6c4fae0bc54b", "bottle-5101f2720ad33bbb", "cap of bottle-607415d98fb0a86f", "cup4-af8e8df9c3ddf887", "spoon2-17db2caaef05f6ad", "spoons-89531fe83419abbf", "spoons3-61bf077c47fba923", "cleaning sponge-9d4ca10a7785ea73", "bottle of liquid dishwasher-d032fa1bde6128bc", "plate4-5ba633d80d7d7f45", "spoon5-a8a6bb88328da21a", "knife-ded5cc9cbd6ce558", "wooden chopping board-1410ae8be62cbb7d", "scissors-0592371430ddf852", "bowl-23b5644604645e17", "tablecloth-d868914c1ff7c87e", "bowl2-7cfebabd7c5ee8fe", "cup5-90d00bfad8bef3ac", "cup6-7fbd7d242d4ab043", "cup7-e5e107708fd61642", "bowl3-bad4f5634c59df1e", "cup8-798c2789521d3d4f", "plate5-d2f04ab1fb9ec454", "plate6-ded79ca35d9f1720", "spoon6-c37577a4677248b6", "button knife-9046dd05409c797d", "spoons7-d8c32fa3aef16311", "white cup-fb6185ad6f70f440", "sponge2-f4d8e69ea10b8817", "cloth2-abc4fcdc88d92259", "plastic bottle-3b53ebd9fa6a7de3", "glass-a70c76be0685168a", "few plastic pieces-efed6243b0067019", "tablecloth3-d311f9bd277d7b7a", "ketchup bottle-925e94907b1d1fc1", "cap of ketchup bottle-eed63781cceecd7f", "cap of pen-a2cedd17060a3050" ], "reasoning": "1. I watched the whole video and saw the user start to fill up the tea kettle (electric kettle-4094d10d4bebd60a) with water at 00:32.\n2. At 00:40, the user finished filling up the tea kettle (electric kettle-4094d10d4bebd60a) with water.\n3. It took the user 8 seconds to fill the tea kettle (electric kettle-4094d10d4bebd60a).\n4. At 02:58, the user weighed a teaspoon (spoon-a4f555f454eaab64) of brown sugar (jar1-08b27da5f0785d38) which was 0.02 g.\n4. 8 divided by 0.02 is 400.", "referenced_object_masks": [ "spoon-a4f555f454eaab64", "electric kettle-4094d10d4bebd60a", "jar1-08b27da5f0785d38" ] }, { "key": "P06-20240510-100047:557d8fdcde65cf2d00ad6d8e6fee3c605d74e2f6", "video_id": "P06-20240510-100047", "question": "Combined, how long do I heat the milk in the microwave and froth it with the hand frother?", "answer_choice_0": "2 minutes and 21 seconds", "answer_choice_1": "2 minutes and 4 seconds", "answer_choice_2": "1 minute and 45 seconds", "answer_choice_3": "2 minutes and 11 seconds", "answer_choice_4": "1 minute and 59 seconds", "answer_id": 0, "answer": "2 minutes and 21 seconds", "question_type": [ "Numerical Reasoning", "Temporal Reasoning", "Action Recognition", "Object Recognition" ], "objects_of_interest": [ "spatula-82446961e238813a", "pan-63377fdc2ca0cf38", "pancake-596a86119069a8ce", "saucepan-e305cb0c04ea3395", "plate-7b17b03d6af35c5b", "electric kettle-4094d10d4bebd60a", "mobile phone-e5a1845d7d0af125", "spoon-a4f555f454eaab64", "jar1-08b27da5f0785d38", "jar2-72a55966f700324d", "coffee cup-c1bbaf14cbf426ba", "coffee cup2-97a07d79fbcc1bf9", "milk bottle-6277236de4df02f3", "lid of milk carton-33831cd9baafc45e", "pen-812f273fd902c6c4", "notebook-d6de4bcee841a486", "lid of jar-58fa775a69d5366c", "weighing balance-22c112cdca74fbc4", "cup3-1c73bc29120d5380", "milk frother whole set-565f3cb8b9e0dbca", "milk frother-1354436915ebbea5", "frother stand-fa86c0a2e464a9e4", "plate3-57f12424060e8757", "saucepan lid-d98c6c4fae0bc54b", "bottle-5101f2720ad33bbb", "cap of bottle-607415d98fb0a86f", "cup4-af8e8df9c3ddf887", "spoon2-17db2caaef05f6ad", "spoons-89531fe83419abbf", "spoons3-61bf077c47fba923", "cleaning sponge-9d4ca10a7785ea73", "bottle of liquid dishwasher-d032fa1bde6128bc", "plate4-5ba633d80d7d7f45", "spoon5-a8a6bb88328da21a", "knife-ded5cc9cbd6ce558", "wooden chopping board-1410ae8be62cbb7d", "scissors-0592371430ddf852", "bowl-23b5644604645e17", "tablecloth-d868914c1ff7c87e", "bowl2-7cfebabd7c5ee8fe", "cup5-90d00bfad8bef3ac", "cup6-7fbd7d242d4ab043", "cup7-e5e107708fd61642", "bowl3-bad4f5634c59df1e", "cup8-798c2789521d3d4f", "plate5-d2f04ab1fb9ec454", "plate6-ded79ca35d9f1720", "spoon6-c37577a4677248b6", "button knife-9046dd05409c797d", "spoons7-d8c32fa3aef16311", "white cup-fb6185ad6f70f440", "sponge2-f4d8e69ea10b8817", "cloth2-abc4fcdc88d92259", "plastic bottle-3b53ebd9fa6a7de3", "glass-a70c76be0685168a", "few plastic pieces-efed6243b0067019", "tablecloth3-d311f9bd277d7b7a", "ketchup bottle-925e94907b1d1fc1", "cap of ketchup bottle-eed63781cceecd7f", "cap of pen-a2cedd17060a3050" ], "reasoning": "1. I watched the video and identified the mug (cup3-1c73bc29120d5380) with the milk at 07:20.\n2. Then, I watched the user place the mug (cup3-1c73bc29120d5380) in the microwave and heat it in there from 07:24-09:38.\n3. I analyzed this span and calculated that the mug (cup3-1c73bc29120d5380) was in there for 2 minutes and 14 seconds.\n4. Then, I observed the user use the milk frother (milk frother whole set-565f3cb8b9e0dbca) to froth the milk in the mug (cup3-1c73bc29120d5380) from 09:44-09:51\n5. I calculated that the user froths the milk for about 7 seconds.\n6. Lastly, I did the final math to determine that 2 minutes and 14 seconds plus 7 seconds equals 2 minutes and 21 seconds.", "referenced_object_masks": [ "milk frother whole set-565f3cb8b9e0dbca", "cup3-1c73bc29120d5380" ] }, { "key": "P06-20240510-100047:5d3e6094044b86302dfb53da47df3fec7a70ba12", "video_id": "P06-20240510-100047", "question": "At what time of the video do I pour the milk into the second mug?", "answer_choice_0": "05:54-05:59", "answer_choice_1": "05:23-05:27", "answer_choice_2": "04:56-04:59", "answer_choice_3": "06:01-06:11", "answer_choice_4": "06:14-06:18", "answer_id": 4, "answer": "06:14-06:18", "question_type": [ "Action Recognition", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "spatula-82446961e238813a", "pan-63377fdc2ca0cf38", "pancake-596a86119069a8ce", "saucepan-e305cb0c04ea3395", "plate-7b17b03d6af35c5b", "electric kettle-4094d10d4bebd60a", "mobile phone-e5a1845d7d0af125", "spoon-a4f555f454eaab64", "jar1-08b27da5f0785d38", "jar2-72a55966f700324d", "coffee cup-c1bbaf14cbf426ba", "coffee cup2-97a07d79fbcc1bf9", "milk bottle-6277236de4df02f3", "lid of milk carton-33831cd9baafc45e", "pen-812f273fd902c6c4", "notebook-d6de4bcee841a486", "lid of jar-58fa775a69d5366c", "weighing balance-22c112cdca74fbc4", "cup3-1c73bc29120d5380", "milk frother whole set-565f3cb8b9e0dbca", "milk frother-1354436915ebbea5", "frother stand-fa86c0a2e464a9e4", "plate3-57f12424060e8757", "saucepan lid-d98c6c4fae0bc54b", "bottle-5101f2720ad33bbb", "cap of bottle-607415d98fb0a86f", "cup4-af8e8df9c3ddf887", "spoon2-17db2caaef05f6ad", "spoons-89531fe83419abbf", "spoons3-61bf077c47fba923", "cleaning sponge-9d4ca10a7785ea73", "bottle of liquid dishwasher-d032fa1bde6128bc", "plate4-5ba633d80d7d7f45", "spoon5-a8a6bb88328da21a", "knife-ded5cc9cbd6ce558", "wooden chopping board-1410ae8be62cbb7d", "scissors-0592371430ddf852", "bowl-23b5644604645e17", "tablecloth-d868914c1ff7c87e", "bowl2-7cfebabd7c5ee8fe", "cup5-90d00bfad8bef3ac", "cup6-7fbd7d242d4ab043", "cup7-e5e107708fd61642", "bowl3-bad4f5634c59df1e", "cup8-798c2789521d3d4f", "plate5-d2f04ab1fb9ec454", "plate6-ded79ca35d9f1720", "spoon6-c37577a4677248b6", "button knife-9046dd05409c797d", "spoons7-d8c32fa3aef16311", "white cup-fb6185ad6f70f440", "sponge2-f4d8e69ea10b8817", "cloth2-abc4fcdc88d92259", "plastic bottle-3b53ebd9fa6a7de3", "glass-a70c76be0685168a", "few plastic pieces-efed6243b0067019", "tablecloth3-d311f9bd277d7b7a", "ketchup bottle-925e94907b1d1fc1", "cap of ketchup bottle-eed63781cceecd7f", "cap of pen-a2cedd17060a3050" ], "reasoning": "1. I watched the video to first identify when the milk (milk bottle-6277236de4df02f3) is poured by the user into the first mug (coffee cup-c1bbaf14cbf426ba).\n2. I observed the user pour the milk (milk bottle-6277236de4df02f3) into the right mug (coffee cup-c1bbaf14cbf426ba) from 01:32-01:37.\n3. I saw the user grab another mug (cup3-1c73bc29120d5380) at 06:02, which they place on the scale (weighing balance-22c112cdca74fbc4) from 06:06-06:09. \n4. From 06:14-06:18, the user poured milk (milk bottle-6277236de4df02f3) into this mug (cup3-1c73bc29120d5380).", "referenced_object_masks": [ "weighing balance-22c112cdca74fbc4", "cup3-1c73bc29120d5380", "milk bottle-6277236de4df02f3", "coffee cup-c1bbaf14cbf426ba" ] }, { "key": "P06-20240510-100047:745d620c23a935805c83d28cebdc6fd6fc862d72", "video_id": "P06-20240510-100047", "question": "What different textures is the instant coffee in this video? Put the answers in chronological order.", "answer_choice_0": "Powder, paste, liquid", "answer_choice_1": "Powder, liquid, paste", "answer_choice_2": "Liquid, paste, powder", "answer_choice_3": "Powder, liquid, frothy", "answer_choice_4": "Liquid, powder, paste", "answer_id": 0, "answer": "Powder, paste, liquid", "question_type": [ "Action Recognition", "Cause and Effect", "Event Occurence", "Object Recognition", "State Changes", "Temporal Reasoning" ], "objects_of_interest": [ "spatula-82446961e238813a", "pan-63377fdc2ca0cf38", "pancake-596a86119069a8ce", "saucepan-e305cb0c04ea3395", "plate-7b17b03d6af35c5b", "electric kettle-4094d10d4bebd60a", "mobile phone-e5a1845d7d0af125", "spoon-a4f555f454eaab64", "jar1-08b27da5f0785d38", "jar2-72a55966f700324d", "coffee cup-c1bbaf14cbf426ba", "coffee cup2-97a07d79fbcc1bf9", "milk bottle-6277236de4df02f3", "lid of milk carton-33831cd9baafc45e", "pen-812f273fd902c6c4", "notebook-d6de4bcee841a486", "lid of jar-58fa775a69d5366c", "weighing balance-22c112cdca74fbc4", "cup3-1c73bc29120d5380", "milk frother whole set-565f3cb8b9e0dbca", "milk frother-1354436915ebbea5", "frother stand-fa86c0a2e464a9e4", "plate3-57f12424060e8757", "saucepan lid-d98c6c4fae0bc54b", "bottle-5101f2720ad33bbb", "cap of bottle-607415d98fb0a86f", "cup4-af8e8df9c3ddf887", "spoon2-17db2caaef05f6ad", "spoons-89531fe83419abbf", "spoons3-61bf077c47fba923", "cleaning sponge-9d4ca10a7785ea73", "bottle of liquid dishwasher-d032fa1bde6128bc", "plate4-5ba633d80d7d7f45", "spoon5-a8a6bb88328da21a", "knife-ded5cc9cbd6ce558", "wooden chopping board-1410ae8be62cbb7d", "scissors-0592371430ddf852", "bowl-23b5644604645e17", "tablecloth-d868914c1ff7c87e", "bowl2-7cfebabd7c5ee8fe", "cup5-90d00bfad8bef3ac", "cup6-7fbd7d242d4ab043", "cup7-e5e107708fd61642", "bowl3-bad4f5634c59df1e", "cup8-798c2789521d3d4f", "plate5-d2f04ab1fb9ec454", "plate6-ded79ca35d9f1720", "spoon6-c37577a4677248b6", "button knife-9046dd05409c797d", "spoons7-d8c32fa3aef16311", "white cup-fb6185ad6f70f440", "sponge2-f4d8e69ea10b8817", "cloth2-abc4fcdc88d92259", "plastic bottle-3b53ebd9fa6a7de3", "glass-a70c76be0685168a", "few plastic pieces-efed6243b0067019", "tablecloth3-d311f9bd277d7b7a", "ketchup bottle-925e94907b1d1fc1", "cap of ketchup bottle-eed63781cceecd7f", "cap of pen-a2cedd17060a3050" ], "reasoning": "1. I watched the whole video and saw the user put 2 teaspoons of instant coffee into a white mug (coffee cup-c1bbaf14cbf426ba) starting at 03:21.\n2. At 04:25, the user put a small amount of hot water into the white mug (coffee cup-c1bbaf14cbf426ba).\n3. From 4:49 to 5:59, the user vigorously stirred the instant coffee and hot water to make a paste .\n4. At 10:03, the user pours the hot milk into the instant coffee paste making it now into a liquid .\n5. The 3 textures in chronological order are powder , paste , liquid .", "referenced_object_masks": [ "coffee cup-c1bbaf14cbf426ba" ] }, { "key": "P06-20240510-100047:a31b605ada828b70a0a5982e5364cb50d4fcfd0a", "video_id": "P06-20240510-100047", "question": "How much longer does it take me to fill the kettle with water than it does for me to pour the milk into the white mug?", "answer_choice_0": "9 seconds", "answer_choice_1": "12 seconds", "answer_choice_2": "4 seconds", "answer_choice_3": "2 seconds", "answer_choice_4": "7 seconds", "answer_id": 2, "answer": "4 seconds", "question_type": [ "Temporal Reasoning", "Action Recognition", "Object Recognition", "Situational Awareness", "Numerical Reasoning" ], "objects_of_interest": [ "spatula-82446961e238813a", "pan-63377fdc2ca0cf38", "pancake-596a86119069a8ce", "saucepan-e305cb0c04ea3395", "plate-7b17b03d6af35c5b", "electric kettle-4094d10d4bebd60a", "mobile phone-e5a1845d7d0af125", "spoon-a4f555f454eaab64", "jar1-08b27da5f0785d38", "jar2-72a55966f700324d", "coffee cup-c1bbaf14cbf426ba", "coffee cup2-97a07d79fbcc1bf9", "milk bottle-6277236de4df02f3", "lid of milk carton-33831cd9baafc45e", "pen-812f273fd902c6c4", "notebook-d6de4bcee841a486", "lid of jar-58fa775a69d5366c", "weighing balance-22c112cdca74fbc4", "cup3-1c73bc29120d5380", "milk frother whole set-565f3cb8b9e0dbca", "milk frother-1354436915ebbea5", "frother stand-fa86c0a2e464a9e4", "plate3-57f12424060e8757", "saucepan lid-d98c6c4fae0bc54b", "bottle-5101f2720ad33bbb", "cap of bottle-607415d98fb0a86f", "cup4-af8e8df9c3ddf887", "spoon2-17db2caaef05f6ad", "spoons-89531fe83419abbf", "spoons3-61bf077c47fba923", "cleaning sponge-9d4ca10a7785ea73", "bottle of liquid dishwasher-d032fa1bde6128bc", "plate4-5ba633d80d7d7f45", "spoon5-a8a6bb88328da21a", "knife-ded5cc9cbd6ce558", "wooden chopping board-1410ae8be62cbb7d", "scissors-0592371430ddf852", "bowl-23b5644604645e17", "tablecloth-d868914c1ff7c87e", "bowl2-7cfebabd7c5ee8fe", "cup5-90d00bfad8bef3ac", "cup6-7fbd7d242d4ab043", "cup7-e5e107708fd61642", "bowl3-bad4f5634c59df1e", "cup8-798c2789521d3d4f", "plate5-d2f04ab1fb9ec454", "plate6-ded79ca35d9f1720", "spoon6-c37577a4677248b6", "button knife-9046dd05409c797d", "spoons7-d8c32fa3aef16311", "white cup-fb6185ad6f70f440", "sponge2-f4d8e69ea10b8817", "cloth2-abc4fcdc88d92259", "plastic bottle-3b53ebd9fa6a7de3", "glass-a70c76be0685168a", "few plastic pieces-efed6243b0067019", "tablecloth3-d311f9bd277d7b7a", "ketchup bottle-925e94907b1d1fc1", "cap of ketchup bottle-eed63781cceecd7f", "cap of pen-a2cedd17060a3050" ], "reasoning": "1. I watched the video to see which event comes first, observing that the user fills the kettle (electric kettle-4094d10d4bebd60a) with water from 00:32-00:41.\n2. I determined that the user takes 9 seconds to fill the kettle (electric kettle-4094d10d4bebd60a).\n3. I continued to watch the video and got to 01:26-01:28, when the user sets 2 mugs (coffee cup-c1bbaf14cbf426ba, coffee cup2-97a07d79fbcc1bf9) on the counter. From 01:32-01:37, the user pours milk into the right mug (coffee cup-c1bbaf14cbf426ba), which is white.\n4. I determined that pouring milk into the white mug (coffee cup-c1bbaf14cbf426ba) totaled 5 seconds.\n5. I determined that filling the kettle (electric kettle-4094d10d4bebd60a) took 4 seconds longer than it took to pour the milk into the mug (coffee cup-c1bbaf14cbf426ba).", "referenced_object_masks": [ "coffee cup-c1bbaf14cbf426ba", "electric kettle-4094d10d4bebd60a", "coffee cup2-97a07d79fbcc1bf9" ] }, { "key": "P06-20240510-100047:b2239423de399e7212695aaf57fb00b07b21ea59", "video_id": "P06-20240510-100047", "question": "In the first minute of the video, how many times do I touch the electric kettle with my left hand?", "answer_choice_0": "2 times", "answer_choice_1": "0 times", "answer_choice_2": "1 time", "answer_choice_3": "3 times", "answer_choice_4": "4 times", "answer_id": 3, "answer": "3 times", "question_type": [ "Object Recognition", "Event Occurence", "Action Recognition", "Egocentric & Spatial Perception", "Temporal Reasoning" ], "objects_of_interest": [ "spatula-82446961e238813a", "pan-63377fdc2ca0cf38", "pancake-596a86119069a8ce", "saucepan-e305cb0c04ea3395", "plate-7b17b03d6af35c5b", "electric kettle-4094d10d4bebd60a", "mobile phone-e5a1845d7d0af125", "spoon-a4f555f454eaab64", "jar1-08b27da5f0785d38", "jar2-72a55966f700324d", "coffee cup-c1bbaf14cbf426ba", "coffee cup2-97a07d79fbcc1bf9", "milk bottle-6277236de4df02f3", "lid of milk carton-33831cd9baafc45e", "pen-812f273fd902c6c4", "notebook-d6de4bcee841a486", "lid of jar-58fa775a69d5366c", "weighing balance-22c112cdca74fbc4", "cup3-1c73bc29120d5380", "milk frother whole set-565f3cb8b9e0dbca", "milk frother-1354436915ebbea5", "frother stand-fa86c0a2e464a9e4", "plate3-57f12424060e8757", "saucepan lid-d98c6c4fae0bc54b", "bottle-5101f2720ad33bbb", "cap of bottle-607415d98fb0a86f", "cup4-af8e8df9c3ddf887", "spoon2-17db2caaef05f6ad", "spoons-89531fe83419abbf", "spoons3-61bf077c47fba923", "cleaning sponge-9d4ca10a7785ea73", "bottle of liquid dishwasher-d032fa1bde6128bc", "plate4-5ba633d80d7d7f45", "spoon5-a8a6bb88328da21a", "knife-ded5cc9cbd6ce558", "wooden chopping board-1410ae8be62cbb7d", "scissors-0592371430ddf852", "bowl-23b5644604645e17", "tablecloth-d868914c1ff7c87e", "bowl2-7cfebabd7c5ee8fe", "cup5-90d00bfad8bef3ac", "cup6-7fbd7d242d4ab043", "cup7-e5e107708fd61642", "bowl3-bad4f5634c59df1e", "cup8-798c2789521d3d4f", "plate5-d2f04ab1fb9ec454", "plate6-ded79ca35d9f1720", "spoon6-c37577a4677248b6", "button knife-9046dd05409c797d", "spoons7-d8c32fa3aef16311", "white cup-fb6185ad6f70f440", "sponge2-f4d8e69ea10b8817", "cloth2-abc4fcdc88d92259", "plastic bottle-3b53ebd9fa6a7de3", "glass-a70c76be0685168a", "few plastic pieces-efed6243b0067019", "tablecloth3-d311f9bd277d7b7a", "ketchup bottle-925e94907b1d1fc1", "cap of ketchup bottle-eed63781cceecd7f", "cap of pen-a2cedd17060a3050" ], "reasoning": "1. I watched the first minute of the video to identify any instances in which the user touches the kettle (electric kettle-4094d10d4bebd60a) with the left hand.\n2. I first observed the user grab the kettle (electric kettle-4094d10d4bebd60a) at 00:23 with the left hand. This marks the first time the kettle (electric kettle-4094d10d4bebd60a) is touched with the left hand.\n3. As I continued to watch the video, from 00:27-00:28, the user switched the kettle (electric kettle-4094d10d4bebd60a) to the right hand, which doesn't qualify for the question.\n4. Next, from 00:30-00:31, the user switched the kettle (electric kettle-4094d10d4bebd60a) back to the left hand, marking the second time the left hand touches it in the first minute.\n5. Then, from 00:41-00:42, the user closes the kettle's lid (electric kettle-4094d10d4bebd60a) with the right hand, not satisfying the criteria.\n6. The user then sets the kettle (electric kettle-4094d10d4bebd60a) down from 00:45-00:46, clicking the kettle's power button at 00:46 with the left hand, marking the third time it is touched in the first minute.\n7. I continued watching the first minute of the video and did not observe any more instances in which the user touched the kettle, meaning the final total is 3 times.", "referenced_object_masks": [ "electric kettle-4094d10d4bebd60a" ] }, { "key": "P06-20240510-100047:cd786b8a9131dbf43cc329d5d0b6df4cb4e1ea62", "video_id": "P06-20240510-100047", "question": "How many degrees would I need to turn when facing the sink to face the stove.", "answer_choice_0": "100 degrees", "answer_choice_1": "180 degrees", "answer_choice_2": "90 degrees", "answer_choice_3": "135 degrees", "answer_choice_4": "45 degrees", "answer_id": 1, "answer": "180 degrees", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "spatula-82446961e238813a", "pan-63377fdc2ca0cf38", "pancake-596a86119069a8ce", "saucepan-e305cb0c04ea3395", "plate-7b17b03d6af35c5b", "electric kettle-4094d10d4bebd60a", "mobile phone-e5a1845d7d0af125", "spoon-a4f555f454eaab64", "jar1-08b27da5f0785d38", "jar2-72a55966f700324d", "coffee cup-c1bbaf14cbf426ba", "coffee cup2-97a07d79fbcc1bf9", "milk bottle-6277236de4df02f3", "lid of milk carton-33831cd9baafc45e", "pen-812f273fd902c6c4", "notebook-d6de4bcee841a486", "lid of jar-58fa775a69d5366c", "weighing balance-22c112cdca74fbc4", "cup3-1c73bc29120d5380", "milk frother whole set-565f3cb8b9e0dbca", "milk frother-1354436915ebbea5", "frother stand-fa86c0a2e464a9e4", "plate3-57f12424060e8757", "saucepan lid-d98c6c4fae0bc54b", "bottle-5101f2720ad33bbb", "cap of bottle-607415d98fb0a86f", "cup4-af8e8df9c3ddf887", "spoon2-17db2caaef05f6ad", "spoons-89531fe83419abbf", "spoons3-61bf077c47fba923", "cleaning sponge-9d4ca10a7785ea73", "bottle of liquid dishwasher-d032fa1bde6128bc", "plate4-5ba633d80d7d7f45", "spoon5-a8a6bb88328da21a", "knife-ded5cc9cbd6ce558", "wooden chopping board-1410ae8be62cbb7d", "scissors-0592371430ddf852", "bowl-23b5644604645e17", "tablecloth-d868914c1ff7c87e", "bowl2-7cfebabd7c5ee8fe", "cup5-90d00bfad8bef3ac", "cup6-7fbd7d242d4ab043", "cup7-e5e107708fd61642", "bowl3-bad4f5634c59df1e", "cup8-798c2789521d3d4f", "plate5-d2f04ab1fb9ec454", "plate6-ded79ca35d9f1720", "spoon6-c37577a4677248b6", "button knife-9046dd05409c797d", "spoons7-d8c32fa3aef16311", "white cup-fb6185ad6f70f440", "sponge2-f4d8e69ea10b8817", "cloth2-abc4fcdc88d92259", "plastic bottle-3b53ebd9fa6a7de3", "glass-a70c76be0685168a", "few plastic pieces-efed6243b0067019", "tablecloth3-d311f9bd277d7b7a", "ketchup bottle-925e94907b1d1fc1", "cap of ketchup bottle-eed63781cceecd7f", "cap of pen-a2cedd17060a3050" ], "reasoning": "1. I watched the whole video and I saw the user put the tea kettle (electric kettle-4094d10d4bebd60a) full of water back on its base at 0:44. \n2. The user then turned around at 00:50. \n3. The user could see the kitchen sink on their right and the stove on their left directly across from the kitchen sink .\n4. Therefore, if the user was at the kitchen sink , they would have to turn 180 degrees to get to the stove .", "referenced_object_masks": [ "electric kettle-4094d10d4bebd60a" ] }, { "key": "P06-20240510-100047:da000e1a227c2c5c15a6a496325fb425d948aec5", "video_id": "P06-20240510-100047", "question": "If I added both the time it takes to stir the instant coffee and hot water and the time it takes for the milk to warm up in the microwave, how much time would I get?\u00a0Write your answer in mm:ss form.", "answer_choice_0": "03:15", "answer_choice_1": "03:21", "answer_choice_2": "02:11", "answer_choice_3": "01:10", "answer_choice_4": "03:24", "answer_id": 1, "answer": "03:21", "question_type": [ "Action Recognition", "Event Occurence", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "spatula-82446961e238813a", "pan-63377fdc2ca0cf38", "pancake-596a86119069a8ce", "saucepan-e305cb0c04ea3395", "plate-7b17b03d6af35c5b", "electric kettle-4094d10d4bebd60a", "mobile phone-e5a1845d7d0af125", "spoon-a4f555f454eaab64", "jar1-08b27da5f0785d38", "jar2-72a55966f700324d", "coffee cup-c1bbaf14cbf426ba", "coffee cup2-97a07d79fbcc1bf9", "milk bottle-6277236de4df02f3", "lid of milk carton-33831cd9baafc45e", "pen-812f273fd902c6c4", "notebook-d6de4bcee841a486", "lid of jar-58fa775a69d5366c", "weighing balance-22c112cdca74fbc4", "cup3-1c73bc29120d5380", "milk frother whole set-565f3cb8b9e0dbca", "milk frother-1354436915ebbea5", "frother stand-fa86c0a2e464a9e4", "plate3-57f12424060e8757", "saucepan lid-d98c6c4fae0bc54b", "bottle-5101f2720ad33bbb", "cap of bottle-607415d98fb0a86f", "cup4-af8e8df9c3ddf887", "spoon2-17db2caaef05f6ad", "spoons-89531fe83419abbf", "spoons3-61bf077c47fba923", "cleaning sponge-9d4ca10a7785ea73", "bottle of liquid dishwasher-d032fa1bde6128bc", "plate4-5ba633d80d7d7f45", "spoon5-a8a6bb88328da21a", "knife-ded5cc9cbd6ce558", "wooden chopping board-1410ae8be62cbb7d", "scissors-0592371430ddf852", "bowl-23b5644604645e17", "tablecloth-d868914c1ff7c87e", "bowl2-7cfebabd7c5ee8fe", "cup5-90d00bfad8bef3ac", "cup6-7fbd7d242d4ab043", "cup7-e5e107708fd61642", "bowl3-bad4f5634c59df1e", "cup8-798c2789521d3d4f", "plate5-d2f04ab1fb9ec454", "plate6-ded79ca35d9f1720", "spoon6-c37577a4677248b6", "button knife-9046dd05409c797d", "spoons7-d8c32fa3aef16311", "white cup-fb6185ad6f70f440", "sponge2-f4d8e69ea10b8817", "cloth2-abc4fcdc88d92259", "plastic bottle-3b53ebd9fa6a7de3", "glass-a70c76be0685168a", "few plastic pieces-efed6243b0067019", "tablecloth3-d311f9bd277d7b7a", "ketchup bottle-925e94907b1d1fc1", "cap of ketchup bottle-eed63781cceecd7f", "cap of pen-a2cedd17060a3050" ], "reasoning": "1. I watched the whole video and saw the user start to stir the instant coffee and hot water at 04:49.\n2. The user stopped stirring the instant coffee and hot water at 05:59.\n3. The length of time stirring the instant coffee and hot water is 1 minute and 10 seconds.\n4. At 07:21, the user starts warming up the milk in the microwave .\n5. At 09:32, the user stops the microwave .\n6. The total amount of time spent warming up the milk is 2 minutes and 11 seconds.\n7. Adding 1 minute and 10 seconds to 2 minutes and 11 seconds is 3 minutes and 21 seconds.", "referenced_object_masks": [] }, { "key": "P06-20240510-100047:dd520e269617e272d228e772bcd2473366e8ee92", "video_id": "P06-20240510-100047", "question": "What item do I clean 2 items after I complete the initial clean on the first mug?", "answer_choice_0": "Bowl", "answer_choice_1": "Plate", "answer_choice_2": "Frying pan", "answer_choice_3": "Knife", "answer_choice_4": "Spatula", "answer_id": 2, "answer": "Frying pan", "question_type": [ "Object Recognition", "Temporal Reasoning", "Counting" ], "objects_of_interest": [ "spatula-82446961e238813a", "pan-63377fdc2ca0cf38", "pancake-596a86119069a8ce", "saucepan-e305cb0c04ea3395", "plate-7b17b03d6af35c5b", "electric kettle-4094d10d4bebd60a", "mobile phone-e5a1845d7d0af125", "spoon-a4f555f454eaab64", "jar1-08b27da5f0785d38", "jar2-72a55966f700324d", "coffee cup-c1bbaf14cbf426ba", "coffee cup2-97a07d79fbcc1bf9", "milk bottle-6277236de4df02f3", "lid of milk carton-33831cd9baafc45e", "pen-812f273fd902c6c4", "notebook-d6de4bcee841a486", "lid of jar-58fa775a69d5366c", "weighing balance-22c112cdca74fbc4", "cup3-1c73bc29120d5380", "milk frother whole set-565f3cb8b9e0dbca", "milk frother-1354436915ebbea5", "frother stand-fa86c0a2e464a9e4", "plate3-57f12424060e8757", "saucepan lid-d98c6c4fae0bc54b", "bottle-5101f2720ad33bbb", "cap of bottle-607415d98fb0a86f", "cup4-af8e8df9c3ddf887", "spoon2-17db2caaef05f6ad", "spoons-89531fe83419abbf", "spoons3-61bf077c47fba923", "cleaning sponge-9d4ca10a7785ea73", "bottle of liquid dishwasher-d032fa1bde6128bc", "plate4-5ba633d80d7d7f45", "spoon5-a8a6bb88328da21a", "knife-ded5cc9cbd6ce558", "wooden chopping board-1410ae8be62cbb7d", "scissors-0592371430ddf852", "bowl-23b5644604645e17", "tablecloth-d868914c1ff7c87e", "bowl2-7cfebabd7c5ee8fe", "cup5-90d00bfad8bef3ac", "cup6-7fbd7d242d4ab043", "cup7-e5e107708fd61642", "bowl3-bad4f5634c59df1e", "cup8-798c2789521d3d4f", "plate5-d2f04ab1fb9ec454", "plate6-ded79ca35d9f1720", "spoon6-c37577a4677248b6", "button knife-9046dd05409c797d", "spoons7-d8c32fa3aef16311", "white cup-fb6185ad6f70f440", "sponge2-f4d8e69ea10b8817", "cloth2-abc4fcdc88d92259", "plastic bottle-3b53ebd9fa6a7de3", "glass-a70c76be0685168a", "few plastic pieces-efed6243b0067019", "tablecloth3-d311f9bd277d7b7a", "ketchup bottle-925e94907b1d1fc1", "cap of ketchup bottle-eed63781cceecd7f", "cap of pen-a2cedd17060a3050" ], "reasoning": "1. I watched the video and determined that the user begins doing the dishes at 11:30.\n2. I observed the user clean the mug (cup3-1c73bc29120d5380) first, from 11:42-11:52.\n3. Next, I saw the user grab the spatula (spatula-82446961e238813a) and clean it from 11:58-12:17. This means that the next item cleaned is the answer, as it will be the second item after the mug.\n4. The user then put down the spatula and grabbed the frying pan (pan-63377fdc2ca0cf38), cleaning it from 12:17-13:15. Thus, the pan (pan-63377fdc2ca0cf38) is the answer.", "referenced_object_masks": [ "pan-63377fdc2ca0cf38", "spatula-82446961e238813a", "cup3-1c73bc29120d5380" ] }, { "key": "P06-20240510-100047:e3fcb3f08080d0b6dfd7dc11b9916f28ae744f5b", "video_id": "P06-20240510-100047", "question": "If I add together the weight of the blue mug and the weight of just the milk, multiply it by the number of times I shake the strainer up and down at the sink after soaking the rice, and then raise it to the 1/2 power, what number would I get? Round to the nearest hundredth if needed.", "answer_choice_0": "45", "answer_choice_1": "42.43", "answer_choice_2": "41.95", "answer_choice_3": "42.33", "answer_choice_4": "39.69", "answer_id": 1, "answer": "42.43", "question_type": [ "Action Recognition", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning", "State Changes", "Counting" ], "objects_of_interest": [ "spatula-82446961e238813a", "pan-63377fdc2ca0cf38", "pancake-596a86119069a8ce", "saucepan-e305cb0c04ea3395", "plate-7b17b03d6af35c5b", "electric kettle-4094d10d4bebd60a", "mobile phone-e5a1845d7d0af125", "spoon-a4f555f454eaab64", "jar1-08b27da5f0785d38", "jar2-72a55966f700324d", "coffee cup-c1bbaf14cbf426ba", "coffee cup2-97a07d79fbcc1bf9", "milk bottle-6277236de4df02f3", "lid of milk carton-33831cd9baafc45e", "pen-812f273fd902c6c4", "notebook-d6de4bcee841a486", "lid of jar-58fa775a69d5366c", "weighing balance-22c112cdca74fbc4", "cup3-1c73bc29120d5380", "milk frother whole set-565f3cb8b9e0dbca", "milk frother-1354436915ebbea5", "frother stand-fa86c0a2e464a9e4", "plate3-57f12424060e8757", "saucepan lid-d98c6c4fae0bc54b", "bottle-5101f2720ad33bbb", "cap of bottle-607415d98fb0a86f", "cup4-af8e8df9c3ddf887", "spoon2-17db2caaef05f6ad", "spoons-89531fe83419abbf", "spoons3-61bf077c47fba923", "cleaning sponge-9d4ca10a7785ea73", "bottle of liquid dishwasher-d032fa1bde6128bc", "plate4-5ba633d80d7d7f45", "spoon5-a8a6bb88328da21a", "knife-ded5cc9cbd6ce558", "wooden chopping board-1410ae8be62cbb7d", "scissors-0592371430ddf852", "bowl-23b5644604645e17", "tablecloth-d868914c1ff7c87e", "bowl2-7cfebabd7c5ee8fe", "cup5-90d00bfad8bef3ac", "cup6-7fbd7d242d4ab043", "cup7-e5e107708fd61642", "bowl3-bad4f5634c59df1e", "cup8-798c2789521d3d4f", "plate5-d2f04ab1fb9ec454", "plate6-ded79ca35d9f1720", "spoon6-c37577a4677248b6", "button knife-9046dd05409c797d", "spoons7-d8c32fa3aef16311", "white cup-fb6185ad6f70f440", "sponge2-f4d8e69ea10b8817", "cloth2-abc4fcdc88d92259", "plastic bottle-3b53ebd9fa6a7de3", "glass-a70c76be0685168a", "few plastic pieces-efed6243b0067019", "tablecloth3-d311f9bd277d7b7a", "ketchup bottle-925e94907b1d1fc1", "cap of ketchup bottle-eed63781cceecd7f", "cap of pen-a2cedd17060a3050" ], "reasoning": "1. I watched the whole video and saw the user weigh the blue mug (cup3-1c73bc29120d5380) at 06:10. \n2. It weighed 136 g.\n3. At 06:23, the user zeroed out the scale with the blue mug (cup3-1c73bc29120d5380) on it so they could just measure the weight of the milk which was 89 g.\n4. 136 plus 89 is 225.\n5. At 01:29, the user starts swirling around the rice in the strainer. \n6. At 02:13, the user starts shaking the strainer up and down at the sink after soaking the rice.\n7. The user does this 8 times.\n8. 225*8 is 1800.\n9. The square root (same as raising to the 1/2 power) of 1800 is 42.42641.\n10. Rounding to the nearest hundredth is 42.43.", "referenced_object_masks": [ "cup3-1c73bc29120d5380" ] }, { "key": "P06-20240510-104642:1a4b61a81a8420e964c445e6b5f1e02001a8cd93", "video_id": "P06-20240510-104642", "question": "How many more times do I open or close a door with my right hand than my left before I set something down on the counter?", "answer_choice_0": "4", "answer_choice_1": "2", "answer_choice_2": "3", "answer_choice_3": "5", "answer_choice_4": "1", "answer_id": 3, "answer": "5", "question_type": [ "Counting", "Numerical Reasoning", "Action Recognition", "Temporal Reasoning", "Object Recognition" ], "objects_of_interest": [ "jar of nutella-e7083e8e2f6ff432", "pack of cottage cheese-2bbe05c1601be543", "pack of butter-8f91f29047b4b66f", "few green chillies-c2d47e643c755b5d", "green chillies-b9cf3401a9486063", "jar of garlic paste-9cd71cc778f011be", "jar of ginger paste-e0c0d3f3994f4177", "pack of tomatoes-182dcd405c740f82", "tablecloth-3e1017c50540eb7a", "weighing balance-e99414eff03ab478", "pen-724d6acb13614581", "notepad-67ab8b9533419a06", "cap of pen-281d5f32aa6ebe20", "chopping board-22f101f4409570c1", "pan-47c866bca54de2dc", "knife-64be2533c09e7f74", "scissor-da199bfc1083df47", "milk frother-fa0d6e4f33a99471", "onion-eb068a1080b7d7fa", "another piece of onion-e82d52d5e7ae57ff", "remaining onions-f695cd21fc85737b", "plate-12913834e3cd1217", "piece of onion-d70320ae9629ee7a", "tomatoes-11a63aee46ca91d1", "jar-97e783a4f3b8d509", "lid-90f9e538a607e441", "sliced tomato-799f6ca7ccb5e43c", "pack of onions-61a62784bbc0b31a", "remaining bunch of garlic-b5a586439889fc4d", "garlic clove-5fd518e64a287ca0", "clove of garlic-d0534b1a9b0778f1", "plug-9c3c2dd7f7fb1410", "another plug-04d6a82d9687d521", "mixer-26a48ae4576cbf29", "knob of mixer-69eaf33a8ede06d3", "bowl-0c6a9a90771c7a27", "plastic pack-bdc65e7ceb5b63ed", "cottage cheese-558b2f5c84a5ce18", "tissue roll-a0457f67d79c4628", "cheese slice-4d339d02345f7ba6", "cheese slice2-f2530d6d8ed8a86f", "bowl2-b8070468d93e2292", "another jar-b27980ab01b646eb", "salt-803c41c4a9bc1781", "thin box with blue strip-12264bdc70a1788a", "plastic bottle-d38ca8ef406a3876", "pack of noodles-eea89c246a427671", "pack of kidney beans-a1532836e8f040c6", "packet of pulses-918cd0e5eee26783", "packet-e32486ce139a57ab", "box-34bbbfaba8997802", "bottle of vegetable oil-b505615b6b75c447", "piece of butter-d56e77a961fb677e", "spoon-31583c54dbe9e210", "spoon2-de02474f7071b8a5", "lid of jar-829a7cb85573bdb6", "spatula-4590e9182ecefcb5", "wooden spoon-d3fd4a3f5bbe4b78", "electric kettle-b6a869ed8a70db69", "pressure cooker-1f2ebe097c381fd4", "unknown-bd84e9de0bfbbef5", "spoon3-0e8f1ca7de86862b", "jarN-066f09071e7d4a10", "bowl3-2b1c6fad4ad357a3", "bowl4-6a61c123c3ddbfb6", "lid of pressure cooker-4953cdb961b32bd5", "cooker-5e9799f36e5d5c09", "cup-9071102743f7d85e", "mobile phone-66aae19eb208c2ee", "tomato-cde702b4b2a11e3a", "sponge-927b4613722829e5", "liquid dishwasher bottle-8bc7c63d57672452", "saucepan-3468c049121e40f4", "another cup-366b157e3b45cfbe", "spoons-3df60a3097216f31", "cleaning sponge-09759011e356e236", "teaspoon picked from bottom of sink then gets combined-9383ebda0dd1a6a8", "knife 2 picked up-9dc7889ae95cd3a1", "plastic box-a491b8dd4abb917b", "plastic box lid-96090eed4372250a" ], "reasoning": "1. I found the initial timestamp of 00:00 and saw that the user began their walk to an open cabinet door, and closed it with their right hand.\n2. At 00:05 after the user closed the cabinet door, they walked to their right and closed a door leading away from the kitchen with their right hand so they could access the fridge.\n3. The user then opened the door with their right hand at 00:10, moved a jar of nutella (jar of nutella-e7083e8e2f6ff432), and then closed the fridge door with their right hand at 00:45 after grabbing several other items from the fridge.\n4. At 00:47, the user opened the door back to the kitchen with their right hand again as they walked back, meaning that they opened or closed a door 5 times total with their right hand, and a total of 0 times with their left hand.", "referenced_object_masks": [ "jar of nutella-e7083e8e2f6ff432" ] }, { "key": "P06-20240510-104642:296eed106645157f08acab542ae41d396058e901", "video_id": "P06-20240510-104642", "question": "If I cut the cottage cheese diagonally, what two shapes would I get, and what type of those shapes would they be?", "answer_choice_0": "Two right triangles", "answer_choice_1": "Two isosceles triangles", "answer_choice_2": "Two irregular quadrilaterals", "answer_choice_3": "Two rectangular prisms", "answer_choice_4": "Two squares", "answer_id": 0, "answer": "Two right triangles", "question_type": [ "Object Recognition", "Counterfactual", "Action Recognition", "Cause and Effect" ], "objects_of_interest": [ "jar of nutella-e7083e8e2f6ff432", "pack of cottage cheese-2bbe05c1601be543", "pack of butter-8f91f29047b4b66f", "few green chillies-c2d47e643c755b5d", "green chillies-b9cf3401a9486063", "jar of garlic paste-9cd71cc778f011be", "jar of ginger paste-e0c0d3f3994f4177", "pack of tomatoes-182dcd405c740f82", "tablecloth-3e1017c50540eb7a", "weighing balance-e99414eff03ab478", "pen-724d6acb13614581", "notepad-67ab8b9533419a06", "cap of pen-281d5f32aa6ebe20", "chopping board-22f101f4409570c1", "pan-47c866bca54de2dc", "knife-64be2533c09e7f74", "scissor-da199bfc1083df47", "milk frother-fa0d6e4f33a99471", "onion-eb068a1080b7d7fa", "another piece of onion-e82d52d5e7ae57ff", "remaining onions-f695cd21fc85737b", "plate-12913834e3cd1217", "piece of onion-d70320ae9629ee7a", "tomatoes-11a63aee46ca91d1", "jar-97e783a4f3b8d509", "lid-90f9e538a607e441", "sliced tomato-799f6ca7ccb5e43c", "pack of onions-61a62784bbc0b31a", "remaining bunch of garlic-b5a586439889fc4d", "garlic clove-5fd518e64a287ca0", "clove of garlic-d0534b1a9b0778f1", "plug-9c3c2dd7f7fb1410", "another plug-04d6a82d9687d521", "mixer-26a48ae4576cbf29", "knob of mixer-69eaf33a8ede06d3", "bowl-0c6a9a90771c7a27", "plastic pack-bdc65e7ceb5b63ed", "cottage cheese-558b2f5c84a5ce18", "tissue roll-a0457f67d79c4628", "cheese slice-4d339d02345f7ba6", "cheese slice2-f2530d6d8ed8a86f", "bowl2-b8070468d93e2292", "another jar-b27980ab01b646eb", "salt-803c41c4a9bc1781", "thin box with blue strip-12264bdc70a1788a", "plastic bottle-d38ca8ef406a3876", "pack of noodles-eea89c246a427671", "pack of kidney beans-a1532836e8f040c6", "packet of pulses-918cd0e5eee26783", "packet-e32486ce139a57ab", "box-34bbbfaba8997802", "bottle of vegetable oil-b505615b6b75c447", "piece of butter-d56e77a961fb677e", "spoon-31583c54dbe9e210", "spoon2-de02474f7071b8a5", "lid of jar-829a7cb85573bdb6", "spatula-4590e9182ecefcb5", "wooden spoon-d3fd4a3f5bbe4b78", "electric kettle-b6a869ed8a70db69", "pressure cooker-1f2ebe097c381fd4", "unknown-bd84e9de0bfbbef5", "spoon3-0e8f1ca7de86862b", "jarN-066f09071e7d4a10", "bowl3-2b1c6fad4ad357a3", "bowl4-6a61c123c3ddbfb6", "lid of pressure cooker-4953cdb961b32bd5", "cooker-5e9799f36e5d5c09", "cup-9071102743f7d85e", "mobile phone-66aae19eb208c2ee", "tomato-cde702b4b2a11e3a", "sponge-927b4613722829e5", "liquid dishwasher bottle-8bc7c63d57672452", "saucepan-3468c049121e40f4", "another cup-366b157e3b45cfbe", "spoons-3df60a3097216f31", "cleaning sponge-09759011e356e236", "teaspoon picked from bottom of sink then gets combined-9383ebda0dd1a6a8", "knife 2 picked up-9dc7889ae95cd3a1", "plastic box-a491b8dd4abb917b", "plastic box lid-96090eed4372250a" ], "reasoning": "1. I watched the video to see what the original shape of the cottage cheese (cottage cheese-558b2f5c84a5ce18) was before it got cut up and saw at 14:46 that it is rectangular in shape.\n2. If I were to cut a rectangle diagonally, I would get two triangles.\n3. The types of these triangles would be right triangles because the cottage cheese (cottage cheese-558b2f5c84a5ce18) is rectangular in shape, which implies that it has 4 90 degree (right) angles.", "referenced_object_masks": [ "cottage cheese-558b2f5c84a5ce18" ] }, { "key": "P06-20240510-104642:2bb2ef020a95796257022632b445e8ae3fdfe35c", "video_id": "P06-20240510-104642", "question": "If I could turn into any visible object that isn't directly touching the stove at the moment before the cheese was put in the pan, but after the moment I weighed the spoon, and I wanted to be as warm as possible, what object would I turn into?", "answer_choice_0": "The oven towel", "answer_choice_1": "The bowl", "answer_choice_2": "The cutting board", "answer_choice_3": "The knife", "answer_choice_4": "The wooden spoon", "answer_id": 4, "answer": "The wooden spoon", "question_type": [ "Cause and Effect", "Counterfactual", "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness", "Temporal Reasoning", "State Changes" ], "objects_of_interest": [ "jar of nutella-e7083e8e2f6ff432", "pack of cottage cheese-2bbe05c1601be543", "pack of butter-8f91f29047b4b66f", "few green chillies-c2d47e643c755b5d", "green chillies-b9cf3401a9486063", "jar of garlic paste-9cd71cc778f011be", "jar of ginger paste-e0c0d3f3994f4177", "pack of tomatoes-182dcd405c740f82", "tablecloth-3e1017c50540eb7a", "weighing balance-e99414eff03ab478", "pen-724d6acb13614581", "notepad-67ab8b9533419a06", "cap of pen-281d5f32aa6ebe20", "chopping board-22f101f4409570c1", "pan-47c866bca54de2dc", "knife-64be2533c09e7f74", "scissor-da199bfc1083df47", "milk frother-fa0d6e4f33a99471", "onion-eb068a1080b7d7fa", "another piece of onion-e82d52d5e7ae57ff", "remaining onions-f695cd21fc85737b", "plate-12913834e3cd1217", "piece of onion-d70320ae9629ee7a", "tomatoes-11a63aee46ca91d1", "jar-97e783a4f3b8d509", "lid-90f9e538a607e441", "sliced tomato-799f6ca7ccb5e43c", "pack of onions-61a62784bbc0b31a", "remaining bunch of garlic-b5a586439889fc4d", "garlic clove-5fd518e64a287ca0", "clove of garlic-d0534b1a9b0778f1", "plug-9c3c2dd7f7fb1410", "another plug-04d6a82d9687d521", "mixer-26a48ae4576cbf29", "knob of mixer-69eaf33a8ede06d3", "bowl-0c6a9a90771c7a27", "plastic pack-bdc65e7ceb5b63ed", "cottage cheese-558b2f5c84a5ce18", "tissue roll-a0457f67d79c4628", "cheese slice-4d339d02345f7ba6", "cheese slice2-f2530d6d8ed8a86f", "bowl2-b8070468d93e2292", "another jar-b27980ab01b646eb", "salt-803c41c4a9bc1781", "thin box with blue strip-12264bdc70a1788a", "plastic bottle-d38ca8ef406a3876", "pack of noodles-eea89c246a427671", "pack of kidney beans-a1532836e8f040c6", "packet of pulses-918cd0e5eee26783", "packet-e32486ce139a57ab", "box-34bbbfaba8997802", "bottle of vegetable oil-b505615b6b75c447", "piece of butter-d56e77a961fb677e", "spoon-31583c54dbe9e210", "spoon2-de02474f7071b8a5", "lid of jar-829a7cb85573bdb6", "spatula-4590e9182ecefcb5", "wooden spoon-d3fd4a3f5bbe4b78", "electric kettle-b6a869ed8a70db69", "pressure cooker-1f2ebe097c381fd4", "unknown-bd84e9de0bfbbef5", "spoon3-0e8f1ca7de86862b", "jarN-066f09071e7d4a10", "bowl3-2b1c6fad4ad357a3", "bowl4-6a61c123c3ddbfb6", "lid of pressure cooker-4953cdb961b32bd5", "cooker-5e9799f36e5d5c09", "cup-9071102743f7d85e", "mobile phone-66aae19eb208c2ee", "tomato-cde702b4b2a11e3a", "sponge-927b4613722829e5", "liquid dishwasher bottle-8bc7c63d57672452", "saucepan-3468c049121e40f4", "another cup-366b157e3b45cfbe", "spoons-3df60a3097216f31", "cleaning sponge-09759011e356e236", "teaspoon picked from bottom of sink then gets combined-9383ebda0dd1a6a8", "knife 2 picked up-9dc7889ae95cd3a1", "plastic box-a491b8dd4abb917b", "plastic box lid-96090eed4372250a" ], "reasoning": "1. I looked for both instances where the cheese (cheese slice2-f2530d6d8ed8a86f) was put into a pan, and where the spoon (spoon-31583c54dbe9e210) was weighed, and found my range was 26:06 to 37:30.\n2. I then found the timestamp 26:44, and saw that the stove was on, with the pan (pan-47c866bca54de2dc) heating up elements as the liquid bubbled.\n3. If the stove is on, the object that's in closest proximity to its warmth would be the wooden spoon (wooden spoon-d3fd4a3f5bbe4b78) continually mixing the contents of the pan.\n4. Though the pan (pan-47c866bca54de2dc) touches the stove, the wooden spoon (wooden spoon-d3fd4a3f5bbe4b78) never does, meaning that the object to turn into if you wanted to be as warm as possible, would be the spoon.", "referenced_object_masks": [ "cheese slice2-f2530d6d8ed8a86f", "spoon-31583c54dbe9e210", "wooden spoon-d3fd4a3f5bbe4b78", "pan-47c866bca54de2dc" ] }, { "key": "P06-20240510-104642:2e9204c81783b020e42911e822058857ec822bf3", "video_id": "P06-20240510-104642", "question": "The second unique object I cut on the cutting board most closely resemble the color of what object in the frame?", "answer_choice_0": "The kitchen shears", "answer_choice_1": "The cutting board", "answer_choice_2": "My shirt", "answer_choice_3": "The plate", "answer_choice_4": "The blender", "answer_id": 2, "answer": "My shirt", "question_type": [ "Action Recognition", "Temporal Reasoning", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "jar of nutella-e7083e8e2f6ff432", "pack of cottage cheese-2bbe05c1601be543", "pack of butter-8f91f29047b4b66f", "few green chillies-c2d47e643c755b5d", "green chillies-b9cf3401a9486063", "jar of garlic paste-9cd71cc778f011be", "jar of ginger paste-e0c0d3f3994f4177", "pack of tomatoes-182dcd405c740f82", "tablecloth-3e1017c50540eb7a", "weighing balance-e99414eff03ab478", "pen-724d6acb13614581", "notepad-67ab8b9533419a06", "cap of pen-281d5f32aa6ebe20", "chopping board-22f101f4409570c1", "pan-47c866bca54de2dc", "knife-64be2533c09e7f74", "scissor-da199bfc1083df47", "milk frother-fa0d6e4f33a99471", "onion-eb068a1080b7d7fa", "another piece of onion-e82d52d5e7ae57ff", "remaining onions-f695cd21fc85737b", "plate-12913834e3cd1217", "piece of onion-d70320ae9629ee7a", "tomatoes-11a63aee46ca91d1", "jar-97e783a4f3b8d509", "lid-90f9e538a607e441", "sliced tomato-799f6ca7ccb5e43c", "pack of onions-61a62784bbc0b31a", "remaining bunch of garlic-b5a586439889fc4d", "garlic clove-5fd518e64a287ca0", "clove of garlic-d0534b1a9b0778f1", "plug-9c3c2dd7f7fb1410", "another plug-04d6a82d9687d521", "mixer-26a48ae4576cbf29", "knob of mixer-69eaf33a8ede06d3", "bowl-0c6a9a90771c7a27", "plastic pack-bdc65e7ceb5b63ed", "cottage cheese-558b2f5c84a5ce18", "tissue roll-a0457f67d79c4628", "cheese slice-4d339d02345f7ba6", "cheese slice2-f2530d6d8ed8a86f", "bowl2-b8070468d93e2292", "another jar-b27980ab01b646eb", "salt-803c41c4a9bc1781", "thin box with blue strip-12264bdc70a1788a", "plastic bottle-d38ca8ef406a3876", "pack of noodles-eea89c246a427671", "pack of kidney beans-a1532836e8f040c6", "packet of pulses-918cd0e5eee26783", "packet-e32486ce139a57ab", "box-34bbbfaba8997802", "bottle of vegetable oil-b505615b6b75c447", "piece of butter-d56e77a961fb677e", "spoon-31583c54dbe9e210", "spoon2-de02474f7071b8a5", "lid of jar-829a7cb85573bdb6", "spatula-4590e9182ecefcb5", "wooden spoon-d3fd4a3f5bbe4b78", "electric kettle-b6a869ed8a70db69", "pressure cooker-1f2ebe097c381fd4", "unknown-bd84e9de0bfbbef5", "spoon3-0e8f1ca7de86862b", "jarN-066f09071e7d4a10", "bowl3-2b1c6fad4ad357a3", "bowl4-6a61c123c3ddbfb6", "lid of pressure cooker-4953cdb961b32bd5", "cooker-5e9799f36e5d5c09", "cup-9071102743f7d85e", "mobile phone-66aae19eb208c2ee", "tomato-cde702b4b2a11e3a", "sponge-927b4613722829e5", "liquid dishwasher bottle-8bc7c63d57672452", "saucepan-3468c049121e40f4", "another cup-366b157e3b45cfbe", "spoons-3df60a3097216f31", "cleaning sponge-09759011e356e236", "teaspoon picked from bottom of sink then gets combined-9383ebda0dd1a6a8", "knife 2 picked up-9dc7889ae95cd3a1", "plastic box-a491b8dd4abb917b", "plastic box lid-96090eed4372250a" ], "reasoning": "1. I found the initial timestamp at 08:14, and watched the clip until 08:30 to determine what object the user was cutting.\n2. I recognized the objects being cut on the cutting board (chopping board-22f101f4409570c1) were tomatoes (tomatoes-11a63aee46ca91d1).\n3. As the tomatoes (tomatoes-11a63aee46ca91d1) are red, I then identified any objects in the specified frame to see what most closely matched the color of the fruit. I saw that the user's shirt was the closest in hue.", "referenced_object_masks": [ "chopping board-22f101f4409570c1", "tomatoes-11a63aee46ca91d1" ] }, { "key": "P06-20240510-104642:5104958451cb5e6ac30372dbba32aa13b3c1ce25", "video_id": "P06-20240510-104642", "question": "What did I do with the leftover tomatoes that were not initially placed in the blender?", "answer_choice_0": "I threw them in the trash", "answer_choice_1": "I snacked on them", "answer_choice_2": "I blended them in the blender", "answer_choice_3": "I put them back", "answer_choice_4": "I rewashed them", "answer_id": 2, "answer": "I blended them in the blender", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "State Changes", "Action Recognition" ], "objects_of_interest": [ "jar of nutella-e7083e8e2f6ff432", "pack of cottage cheese-2bbe05c1601be543", "pack of butter-8f91f29047b4b66f", "few green chillies-c2d47e643c755b5d", "green chillies-b9cf3401a9486063", "jar of garlic paste-9cd71cc778f011be", "jar of ginger paste-e0c0d3f3994f4177", "pack of tomatoes-182dcd405c740f82", "tablecloth-3e1017c50540eb7a", "weighing balance-e99414eff03ab478", "pen-724d6acb13614581", "notepad-67ab8b9533419a06", "cap of pen-281d5f32aa6ebe20", "chopping board-22f101f4409570c1", "pan-47c866bca54de2dc", "knife-64be2533c09e7f74", "scissor-da199bfc1083df47", "milk frother-fa0d6e4f33a99471", "onion-eb068a1080b7d7fa", "another piece of onion-e82d52d5e7ae57ff", "remaining onions-f695cd21fc85737b", "plate-12913834e3cd1217", "piece of onion-d70320ae9629ee7a", "tomatoes-11a63aee46ca91d1", "jar-97e783a4f3b8d509", "lid-90f9e538a607e441", "sliced tomato-799f6ca7ccb5e43c", "pack of onions-61a62784bbc0b31a", "remaining bunch of garlic-b5a586439889fc4d", "garlic clove-5fd518e64a287ca0", "clove of garlic-d0534b1a9b0778f1", "plug-9c3c2dd7f7fb1410", "another plug-04d6a82d9687d521", "mixer-26a48ae4576cbf29", "knob of mixer-69eaf33a8ede06d3", "bowl-0c6a9a90771c7a27", "plastic pack-bdc65e7ceb5b63ed", "cottage cheese-558b2f5c84a5ce18", "tissue roll-a0457f67d79c4628", "cheese slice-4d339d02345f7ba6", "cheese slice2-f2530d6d8ed8a86f", "bowl2-b8070468d93e2292", "another jar-b27980ab01b646eb", "salt-803c41c4a9bc1781", "thin box with blue strip-12264bdc70a1788a", "plastic bottle-d38ca8ef406a3876", "pack of noodles-eea89c246a427671", "pack of kidney beans-a1532836e8f040c6", "packet of pulses-918cd0e5eee26783", "packet-e32486ce139a57ab", "box-34bbbfaba8997802", "bottle of vegetable oil-b505615b6b75c447", "piece of butter-d56e77a961fb677e", "spoon-31583c54dbe9e210", "spoon2-de02474f7071b8a5", "lid of jar-829a7cb85573bdb6", "spatula-4590e9182ecefcb5", "wooden spoon-d3fd4a3f5bbe4b78", "electric kettle-b6a869ed8a70db69", "pressure cooker-1f2ebe097c381fd4", "unknown-bd84e9de0bfbbef5", "spoon3-0e8f1ca7de86862b", "jarN-066f09071e7d4a10", "bowl3-2b1c6fad4ad357a3", "bowl4-6a61c123c3ddbfb6", "lid of pressure cooker-4953cdb961b32bd5", "cooker-5e9799f36e5d5c09", "cup-9071102743f7d85e", "mobile phone-66aae19eb208c2ee", "tomato-cde702b4b2a11e3a", "sponge-927b4613722829e5", "liquid dishwasher bottle-8bc7c63d57672452", "saucepan-3468c049121e40f4", "another cup-366b157e3b45cfbe", "spoons-3df60a3097216f31", "cleaning sponge-09759011e356e236", "teaspoon picked from bottom of sink then gets combined-9383ebda0dd1a6a8", "knife 2 picked up-9dc7889ae95cd3a1", "plastic box-a491b8dd4abb917b", "plastic box lid-96090eed4372250a" ], "reasoning": "1. I watched the video to find what would be considered \"leftover\" tomatoes and saw that when the user finished putting the desired amount of tomatoes (sliced tomato-799f6ca7ccb5e43c) inside of the blender jar (jar-97e783a4f3b8d509), a sizeable amount of them were left over on the cutting board (chopping board-22f101f4409570c1) at 09:43.\n2. I then observed what the user did with these tomatoes (sliced tomato-799f6ca7ccb5e43c) and saw that after they blended the initial mixture, they put the rest of the tomatoes (sliced tomato-799f6ca7ccb5e43c) in the blender (jar-97e783a4f3b8d509) between 12:14-12:21 and proceeded to blend them as well at 12:30.\n3. Therefore, the user blended the leftover tomatoes (sliced tomato-799f6ca7ccb5e43c) in the blender (mixer-26a48ae4576cbf29).", "referenced_object_masks": [ "sliced tomato-799f6ca7ccb5e43c", "chopping board-22f101f4409570c1", "mixer-26a48ae4576cbf29", "jar-97e783a4f3b8d509" ] }, { "key": "P06-20240510-104642:51b1116fbe4be53ca35c1de6644b480f8830ae4e", "video_id": "P06-20240510-104642", "question": "What did I blend inside the blender?", "answer_choice_0": "Tomatoes, chillies, and garlic", "answer_choice_1": "Onions, tomatoes, and garlic", "answer_choice_2": "Cheese, garlic, and chillies", "answer_choice_3": "Onions, chillies, and tomatoes", "answer_choice_4": "Onions, cheese, and chillies", "answer_id": 0, "answer": "Tomatoes, chillies, and garlic", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "State Changes" ], "objects_of_interest": [ "jar of nutella-e7083e8e2f6ff432", "pack of cottage cheese-2bbe05c1601be543", "pack of butter-8f91f29047b4b66f", "few green chillies-c2d47e643c755b5d", "green chillies-b9cf3401a9486063", "jar of garlic paste-9cd71cc778f011be", "jar of ginger paste-e0c0d3f3994f4177", "pack of tomatoes-182dcd405c740f82", "tablecloth-3e1017c50540eb7a", "weighing balance-e99414eff03ab478", "pen-724d6acb13614581", "notepad-67ab8b9533419a06", "cap of pen-281d5f32aa6ebe20", "chopping board-22f101f4409570c1", "pan-47c866bca54de2dc", "knife-64be2533c09e7f74", "scissor-da199bfc1083df47", "milk frother-fa0d6e4f33a99471", "onion-eb068a1080b7d7fa", "another piece of onion-e82d52d5e7ae57ff", "remaining onions-f695cd21fc85737b", "plate-12913834e3cd1217", "piece of onion-d70320ae9629ee7a", "tomatoes-11a63aee46ca91d1", "jar-97e783a4f3b8d509", "lid-90f9e538a607e441", "sliced tomato-799f6ca7ccb5e43c", "pack of onions-61a62784bbc0b31a", "remaining bunch of garlic-b5a586439889fc4d", "garlic clove-5fd518e64a287ca0", "clove of garlic-d0534b1a9b0778f1", "plug-9c3c2dd7f7fb1410", "another plug-04d6a82d9687d521", "mixer-26a48ae4576cbf29", "knob of mixer-69eaf33a8ede06d3", "bowl-0c6a9a90771c7a27", "plastic pack-bdc65e7ceb5b63ed", "cottage cheese-558b2f5c84a5ce18", "tissue roll-a0457f67d79c4628", "cheese slice-4d339d02345f7ba6", "cheese slice2-f2530d6d8ed8a86f", "bowl2-b8070468d93e2292", "another jar-b27980ab01b646eb", "salt-803c41c4a9bc1781", "thin box with blue strip-12264bdc70a1788a", "plastic bottle-d38ca8ef406a3876", "pack of noodles-eea89c246a427671", "pack of kidney beans-a1532836e8f040c6", "packet of pulses-918cd0e5eee26783", "packet-e32486ce139a57ab", "box-34bbbfaba8997802", "bottle of vegetable oil-b505615b6b75c447", "piece of butter-d56e77a961fb677e", "spoon-31583c54dbe9e210", "spoon2-de02474f7071b8a5", "lid of jar-829a7cb85573bdb6", "spatula-4590e9182ecefcb5", "wooden spoon-d3fd4a3f5bbe4b78", "electric kettle-b6a869ed8a70db69", "pressure cooker-1f2ebe097c381fd4", "unknown-bd84e9de0bfbbef5", "spoon3-0e8f1ca7de86862b", "jarN-066f09071e7d4a10", "bowl3-2b1c6fad4ad357a3", "bowl4-6a61c123c3ddbfb6", "lid of pressure cooker-4953cdb961b32bd5", "cooker-5e9799f36e5d5c09", "cup-9071102743f7d85e", "mobile phone-66aae19eb208c2ee", "tomato-cde702b4b2a11e3a", "sponge-927b4613722829e5", "liquid dishwasher bottle-8bc7c63d57672452", "saucepan-3468c049121e40f4", "another cup-366b157e3b45cfbe", "spoons-3df60a3097216f31", "cleaning sponge-09759011e356e236", "teaspoon picked from bottom of sink then gets combined-9383ebda0dd1a6a8", "knife 2 picked up-9dc7889ae95cd3a1", "plastic box-a491b8dd4abb917b", "plastic box lid-96090eed4372250a" ], "reasoning": "1. I watched the video to find what was put in the blender (mixer-26a48ae4576cbf29) first and saw the sliced tomato (sliced tomato-799f6ca7ccb5e43c) being added to the blender jar (jar-97e783a4f3b8d509) between 08:03-09:45.\n2. I then saw the green chillies (green chillies-b9cf3401a9486063) being cut up and added to the blender jar (jar-97e783a4f3b8d509) between 10:25-10:35.\n3. I then saw some garlic being added to the blender jar (jar-97e783a4f3b8d509) at 11:19.\n4. So, the user blended tomatoes, chillies, and garlic inside the blender (mixer-26a48ae4576cbf29).", "referenced_object_masks": [ "sliced tomato-799f6ca7ccb5e43c", "green chillies-b9cf3401a9486063", "mixer-26a48ae4576cbf29", "jar-97e783a4f3b8d509" ] }, { "key": "P06-20240510-104642:621d75402bc87c0fb9c008043e67455becc101a0", "video_id": "P06-20240510-104642", "question": "Why did I put water in the blender jar after I blended the leftover tomatoes?", "answer_choice_0": "To rinse the jar out for future use", "answer_choice_1": "To let the jar soak", "answer_choice_2": "To check the jar for leaks", "answer_choice_3": "To pour the rest of the tomatoes out", "answer_choice_4": "To begin washing the jar", "answer_id": 3, "answer": "To pour the rest of the tomatoes out", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Goal Reasoning" ], "objects_of_interest": [ "jar of nutella-e7083e8e2f6ff432", "pack of cottage cheese-2bbe05c1601be543", "pack of butter-8f91f29047b4b66f", "few green chillies-c2d47e643c755b5d", "green chillies-b9cf3401a9486063", "jar of garlic paste-9cd71cc778f011be", "jar of ginger paste-e0c0d3f3994f4177", "pack of tomatoes-182dcd405c740f82", "tablecloth-3e1017c50540eb7a", "weighing balance-e99414eff03ab478", "pen-724d6acb13614581", "notepad-67ab8b9533419a06", "cap of pen-281d5f32aa6ebe20", "chopping board-22f101f4409570c1", "pan-47c866bca54de2dc", "knife-64be2533c09e7f74", "scissor-da199bfc1083df47", "milk frother-fa0d6e4f33a99471", "onion-eb068a1080b7d7fa", "another piece of onion-e82d52d5e7ae57ff", "remaining onions-f695cd21fc85737b", "plate-12913834e3cd1217", "piece of onion-d70320ae9629ee7a", "tomatoes-11a63aee46ca91d1", "jar-97e783a4f3b8d509", "lid-90f9e538a607e441", "sliced tomato-799f6ca7ccb5e43c", "pack of onions-61a62784bbc0b31a", "remaining bunch of garlic-b5a586439889fc4d", "garlic clove-5fd518e64a287ca0", "clove of garlic-d0534b1a9b0778f1", "plug-9c3c2dd7f7fb1410", "another plug-04d6a82d9687d521", "mixer-26a48ae4576cbf29", "knob of mixer-69eaf33a8ede06d3", "bowl-0c6a9a90771c7a27", "plastic pack-bdc65e7ceb5b63ed", "cottage cheese-558b2f5c84a5ce18", "tissue roll-a0457f67d79c4628", "cheese slice-4d339d02345f7ba6", "cheese slice2-f2530d6d8ed8a86f", "bowl2-b8070468d93e2292", "another jar-b27980ab01b646eb", "salt-803c41c4a9bc1781", "thin box with blue strip-12264bdc70a1788a", "plastic bottle-d38ca8ef406a3876", "pack of noodles-eea89c246a427671", "pack of kidney beans-a1532836e8f040c6", "packet of pulses-918cd0e5eee26783", "packet-e32486ce139a57ab", "box-34bbbfaba8997802", "bottle of vegetable oil-b505615b6b75c447", "piece of butter-d56e77a961fb677e", "spoon-31583c54dbe9e210", "spoon2-de02474f7071b8a5", "lid of jar-829a7cb85573bdb6", "spatula-4590e9182ecefcb5", "wooden spoon-d3fd4a3f5bbe4b78", "electric kettle-b6a869ed8a70db69", "pressure cooker-1f2ebe097c381fd4", "unknown-bd84e9de0bfbbef5", "spoon3-0e8f1ca7de86862b", "jarN-066f09071e7d4a10", "bowl3-2b1c6fad4ad357a3", "bowl4-6a61c123c3ddbfb6", "lid of pressure cooker-4953cdb961b32bd5", "cooker-5e9799f36e5d5c09", "cup-9071102743f7d85e", "mobile phone-66aae19eb208c2ee", "tomato-cde702b4b2a11e3a", "sponge-927b4613722829e5", "liquid dishwasher bottle-8bc7c63d57672452", "saucepan-3468c049121e40f4", "another cup-366b157e3b45cfbe", "spoons-3df60a3097216f31", "cleaning sponge-09759011e356e236", "teaspoon picked from bottom of sink then gets combined-9383ebda0dd1a6a8", "knife 2 picked up-9dc7889ae95cd3a1", "plastic box-a491b8dd4abb917b", "plastic box lid-96090eed4372250a" ], "reasoning": "1. I first watched the video to find what would be considered \"leftover\" tomatoes and saw that when the user finished putting the desired amount of tomatoes (sliced tomato-799f6ca7ccb5e43c) inside of the blender jar (jar-97e783a4f3b8d509), a sizeable amount of them were left over on the cutting board (chopping board-22f101f4409570c1) at 09:43.\n2. I then observed what the user did with these tomatoes (sliced tomato-799f6ca7ccb5e43c) and saw that after they blended the initial mixture, they put the rest of the tomatoes (sliced tomato-799f6ca7ccb5e43c) in the blender between 12:14-12:21 and proceeded to blend them as well at 12:30.\n3. I then observed the user pour out the leftover tomatoes (sliced tomato-799f6ca7ccb5e43c) in the bowl (bowl-0c6a9a90771c7a27), then proceed to the sink and put water in the blender jar (jar-97e783a4f3b8d509) at 13:01.\n4. The user then puts the lid (lid-90f9e538a607e441) back on the blender jar (jar-97e783a4f3b8d509) at 13:05 and shakes it with the water inside at 13:08.\n5. The user then pours out the water into the bowl (bowl-0c6a9a90771c7a27) at 13:12, and I observed the water being a mixture of water and the rest of the tomatoes (sliced tomato-799f6ca7ccb5e43c).\n6. I also observed the blender jar (jar-97e783a4f3b8d509) being fairly empty and clean after the pour at 13:12, as opposed to the pour at 12:52, where it is obvious that the inside of the jar (jar-97e783a4f3b8d509) is still covered with tomato residue.\n7. Therefore, the process of pouring of water into the jar (jar-97e783a4f3b8d509) at 13:01, shaking it at 13:08, then pouring it out at 13:12 appears to be a strategy to be able to quickly pour the rest of the tomatoes (sliced tomato-799f6ca7ccb5e43c) out, instead of scraping the remaining tomatoes (sliced tomato-799f6ca7ccb5e43c) out from inside the blender jar (jar-97e783a4f3b8d509) using a spoon, for example.", "referenced_object_masks": [ "sliced tomato-799f6ca7ccb5e43c", "lid-90f9e538a607e441", "jar-97e783a4f3b8d509", "bowl-0c6a9a90771c7a27", "chopping board-22f101f4409570c1" ] }, { "key": "P06-20240510-104642:64c640a9323c93a01b7b58799930143a1bfeb830", "video_id": "P06-20240510-104642", "question": "What number would I get if I multiplied the number of tomatoes I used by their combined weight?", "answer_choice_0": "436", "answer_choice_1": "95", "answer_choice_2": "323", "answer_choice_3": "652", "answer_choice_4": "202", "answer_id": 0, "answer": "436", "question_type": [ "Counting", "Numerical Reasoning", "Object Recognition", "Reading" ], "objects_of_interest": [ "jar of nutella-e7083e8e2f6ff432", "pack of cottage cheese-2bbe05c1601be543", "pack of butter-8f91f29047b4b66f", "few green chillies-c2d47e643c755b5d", "green chillies-b9cf3401a9486063", "jar of garlic paste-9cd71cc778f011be", "jar of ginger paste-e0c0d3f3994f4177", "pack of tomatoes-182dcd405c740f82", "tablecloth-3e1017c50540eb7a", "weighing balance-e99414eff03ab478", "pen-724d6acb13614581", "notepad-67ab8b9533419a06", "cap of pen-281d5f32aa6ebe20", "chopping board-22f101f4409570c1", "pan-47c866bca54de2dc", "knife-64be2533c09e7f74", "scissor-da199bfc1083df47", "milk frother-fa0d6e4f33a99471", "onion-eb068a1080b7d7fa", "another piece of onion-e82d52d5e7ae57ff", "remaining onions-f695cd21fc85737b", "plate-12913834e3cd1217", "piece of onion-d70320ae9629ee7a", "tomatoes-11a63aee46ca91d1", "jar-97e783a4f3b8d509", "lid-90f9e538a607e441", "sliced tomato-799f6ca7ccb5e43c", "pack of onions-61a62784bbc0b31a", "remaining bunch of garlic-b5a586439889fc4d", "garlic clove-5fd518e64a287ca0", "clove of garlic-d0534b1a9b0778f1", "plug-9c3c2dd7f7fb1410", "another plug-04d6a82d9687d521", "mixer-26a48ae4576cbf29", "knob of mixer-69eaf33a8ede06d3", "bowl-0c6a9a90771c7a27", "plastic pack-bdc65e7ceb5b63ed", "cottage cheese-558b2f5c84a5ce18", "tissue roll-a0457f67d79c4628", "cheese slice-4d339d02345f7ba6", "cheese slice2-f2530d6d8ed8a86f", "bowl2-b8070468d93e2292", "another jar-b27980ab01b646eb", "salt-803c41c4a9bc1781", "thin box with blue strip-12264bdc70a1788a", "plastic bottle-d38ca8ef406a3876", "pack of noodles-eea89c246a427671", "pack of kidney beans-a1532836e8f040c6", "packet of pulses-918cd0e5eee26783", "packet-e32486ce139a57ab", "box-34bbbfaba8997802", "bottle of vegetable oil-b505615b6b75c447", "piece of butter-d56e77a961fb677e", "spoon-31583c54dbe9e210", "spoon2-de02474f7071b8a5", "lid of jar-829a7cb85573bdb6", "spatula-4590e9182ecefcb5", "wooden spoon-d3fd4a3f5bbe4b78", "electric kettle-b6a869ed8a70db69", "pressure cooker-1f2ebe097c381fd4", "unknown-bd84e9de0bfbbef5", "spoon3-0e8f1ca7de86862b", "jarN-066f09071e7d4a10", "bowl3-2b1c6fad4ad357a3", "bowl4-6a61c123c3ddbfb6", "lid of pressure cooker-4953cdb961b32bd5", "cooker-5e9799f36e5d5c09", "cup-9071102743f7d85e", "mobile phone-66aae19eb208c2ee", "tomato-cde702b4b2a11e3a", "sponge-927b4613722829e5", "liquid dishwasher bottle-8bc7c63d57672452", "saucepan-3468c049121e40f4", "another cup-366b157e3b45cfbe", "spoons-3df60a3097216f31", "cleaning sponge-09759011e356e236", "teaspoon picked from bottom of sink then gets combined-9383ebda0dd1a6a8", "knife 2 picked up-9dc7889ae95cd3a1", "plastic box-a491b8dd4abb917b", "plastic box lid-96090eed4372250a" ], "reasoning": "1. I watched the video to find the amount of tomatoes the user used and saw four tomatoes (tomatoes-11a63aee46ca91d1) come out of the pack of tomatoes (pack of tomatoes-182dcd405c740f82) at 06:23.\n2. I then waited to see when the user would weigh the tomatoes (tomatoes-11a63aee46ca91d1) on the scale and saw this happen at 6:04, with a weight of 109 (presumably grams).\n3. I multiplied 4x109=436.", "referenced_object_masks": [ "pack of tomatoes-182dcd405c740f82", "tomatoes-11a63aee46ca91d1" ] }, { "key": "P06-20240510-104642:77fb43bc31bc64cf15d552b6d610254d0d19b9f3", "video_id": "P06-20240510-104642", "question": "If I were the water in the cup at the moment before I put the cheese into the pan on the stove, what would my name be if it were a combination of the action being taken by the user + the approximate amount left in the cup at the end of the clip, rounded to the nearest fifth?", "answer_choice_0": "Drinking + two fifths", "answer_choice_1": "Drinking + four fifths", "answer_choice_2": "Pouring + one fifth", "answer_choice_3": "Pouring + three fifths", "answer_choice_4": "Pouring + four fifths", "answer_id": 0, "answer": "Drinking + two fifths", "question_type": [ "Action Recognition", "Audio Reasoning", "Temporal Reasoning", "Situational Awareness", "Counterfactual", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "jar of nutella-e7083e8e2f6ff432", "pack of cottage cheese-2bbe05c1601be543", "pack of butter-8f91f29047b4b66f", "few green chillies-c2d47e643c755b5d", "green chillies-b9cf3401a9486063", "jar of garlic paste-9cd71cc778f011be", "jar of ginger paste-e0c0d3f3994f4177", "pack of tomatoes-182dcd405c740f82", "tablecloth-3e1017c50540eb7a", "weighing balance-e99414eff03ab478", "pen-724d6acb13614581", "notepad-67ab8b9533419a06", "cap of pen-281d5f32aa6ebe20", "chopping board-22f101f4409570c1", "pan-47c866bca54de2dc", "knife-64be2533c09e7f74", "scissor-da199bfc1083df47", "milk frother-fa0d6e4f33a99471", "onion-eb068a1080b7d7fa", "another piece of onion-e82d52d5e7ae57ff", "remaining onions-f695cd21fc85737b", "plate-12913834e3cd1217", "piece of onion-d70320ae9629ee7a", "tomatoes-11a63aee46ca91d1", "jar-97e783a4f3b8d509", "lid-90f9e538a607e441", "sliced tomato-799f6ca7ccb5e43c", "pack of onions-61a62784bbc0b31a", "remaining bunch of garlic-b5a586439889fc4d", "garlic clove-5fd518e64a287ca0", "clove of garlic-d0534b1a9b0778f1", "plug-9c3c2dd7f7fb1410", "another plug-04d6a82d9687d521", "mixer-26a48ae4576cbf29", "knob of mixer-69eaf33a8ede06d3", "bowl-0c6a9a90771c7a27", "plastic pack-bdc65e7ceb5b63ed", "cottage cheese-558b2f5c84a5ce18", "tissue roll-a0457f67d79c4628", "cheese slice-4d339d02345f7ba6", "cheese slice2-f2530d6d8ed8a86f", "bowl2-b8070468d93e2292", "another jar-b27980ab01b646eb", "salt-803c41c4a9bc1781", "thin box with blue strip-12264bdc70a1788a", "plastic bottle-d38ca8ef406a3876", "pack of noodles-eea89c246a427671", "pack of kidney beans-a1532836e8f040c6", "packet of pulses-918cd0e5eee26783", "packet-e32486ce139a57ab", "box-34bbbfaba8997802", "bottle of vegetable oil-b505615b6b75c447", "piece of butter-d56e77a961fb677e", "spoon-31583c54dbe9e210", "spoon2-de02474f7071b8a5", "lid of jar-829a7cb85573bdb6", "spatula-4590e9182ecefcb5", "wooden spoon-d3fd4a3f5bbe4b78", "electric kettle-b6a869ed8a70db69", "pressure cooker-1f2ebe097c381fd4", "unknown-bd84e9de0bfbbef5", "spoon3-0e8f1ca7de86862b", "jarN-066f09071e7d4a10", "bowl3-2b1c6fad4ad357a3", "bowl4-6a61c123c3ddbfb6", "lid of pressure cooker-4953cdb961b32bd5", "cooker-5e9799f36e5d5c09", "cup-9071102743f7d85e", "mobile phone-66aae19eb208c2ee", "tomato-cde702b4b2a11e3a", "sponge-927b4613722829e5", "liquid dishwasher bottle-8bc7c63d57672452", "saucepan-3468c049121e40f4", "another cup-366b157e3b45cfbe", "spoons-3df60a3097216f31", "cleaning sponge-09759011e356e236", "teaspoon picked from bottom of sink then gets combined-9383ebda0dd1a6a8", "knife 2 picked up-9dc7889ae95cd3a1", "plastic box-a491b8dd4abb917b", "plastic box lid-96090eed4372250a" ], "reasoning": "1. I located the instance where cheese (cheese slice2-f2530d6d8ed8a86f) was being placed into the pan at 37:30\n2. From there, I worked backwards to find where water was in a cup.\n3. I found the timestamp of 36:58, and took note of the nearly full cup (cup-9071102743f7d85e).\n4. The user raised the mug (cup-9071102743f7d85e) to their mouth, and began to drink deeply, until they finally move to put the object at rest at 37:11.\n5. I saw briefly that the mug was a bit less than half full, indicating that the nearest fifth would have been 2/5ths.", "referenced_object_masks": [ "cheese slice2-f2530d6d8ed8a86f", "cup-9071102743f7d85e" ] }, { "key": "P06-20240510-104642:943d9a5655358f87bc509bcf97db579f93d74138", "video_id": "P06-20240510-104642", "question": "If I were a pack of kidney beans, how much more quickly would I have seen the world outside the cupboard than I would if I were a bottle of vegetable oil?", "answer_choice_0": "40 seconds", "answer_choice_1": "16 seconds", "answer_choice_2": "13 seconds", "answer_choice_3": "28 seconds", "answer_choice_4": "12 seconds", "answer_id": 1, "answer": "16 seconds", "question_type": [ "Egocentric & Spatial Perception", "Temporal Reasoning", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "jar of nutella-e7083e8e2f6ff432", "pack of cottage cheese-2bbe05c1601be543", "pack of butter-8f91f29047b4b66f", "few green chillies-c2d47e643c755b5d", "green chillies-b9cf3401a9486063", "jar of garlic paste-9cd71cc778f011be", "jar of ginger paste-e0c0d3f3994f4177", "pack of tomatoes-182dcd405c740f82", "tablecloth-3e1017c50540eb7a", "weighing balance-e99414eff03ab478", "pen-724d6acb13614581", "notepad-67ab8b9533419a06", "cap of pen-281d5f32aa6ebe20", "chopping board-22f101f4409570c1", "pan-47c866bca54de2dc", "knife-64be2533c09e7f74", "scissor-da199bfc1083df47", "milk frother-fa0d6e4f33a99471", "onion-eb068a1080b7d7fa", "another piece of onion-e82d52d5e7ae57ff", "remaining onions-f695cd21fc85737b", "plate-12913834e3cd1217", "piece of onion-d70320ae9629ee7a", "tomatoes-11a63aee46ca91d1", "jar-97e783a4f3b8d509", "lid-90f9e538a607e441", "sliced tomato-799f6ca7ccb5e43c", "pack of onions-61a62784bbc0b31a", "remaining bunch of garlic-b5a586439889fc4d", "garlic clove-5fd518e64a287ca0", "clove of garlic-d0534b1a9b0778f1", "plug-9c3c2dd7f7fb1410", "another plug-04d6a82d9687d521", "mixer-26a48ae4576cbf29", "knob of mixer-69eaf33a8ede06d3", "bowl-0c6a9a90771c7a27", "plastic pack-bdc65e7ceb5b63ed", "cottage cheese-558b2f5c84a5ce18", "tissue roll-a0457f67d79c4628", "cheese slice-4d339d02345f7ba6", "cheese slice2-f2530d6d8ed8a86f", "bowl2-b8070468d93e2292", "another jar-b27980ab01b646eb", "salt-803c41c4a9bc1781", "thin box with blue strip-12264bdc70a1788a", "plastic bottle-d38ca8ef406a3876", "pack of noodles-eea89c246a427671", "pack of kidney beans-a1532836e8f040c6", "packet of pulses-918cd0e5eee26783", "packet-e32486ce139a57ab", "box-34bbbfaba8997802", "bottle of vegetable oil-b505615b6b75c447", "piece of butter-d56e77a961fb677e", "spoon-31583c54dbe9e210", "spoon2-de02474f7071b8a5", "lid of jar-829a7cb85573bdb6", "spatula-4590e9182ecefcb5", "wooden spoon-d3fd4a3f5bbe4b78", "electric kettle-b6a869ed8a70db69", "pressure cooker-1f2ebe097c381fd4", "unknown-bd84e9de0bfbbef5", "spoon3-0e8f1ca7de86862b", "jarN-066f09071e7d4a10", "bowl3-2b1c6fad4ad357a3", "bowl4-6a61c123c3ddbfb6", "lid of pressure cooker-4953cdb961b32bd5", "cooker-5e9799f36e5d5c09", "cup-9071102743f7d85e", "mobile phone-66aae19eb208c2ee", "tomato-cde702b4b2a11e3a", "sponge-927b4613722829e5", "liquid dishwasher bottle-8bc7c63d57672452", "saucepan-3468c049121e40f4", "another cup-366b157e3b45cfbe", "spoons-3df60a3097216f31", "cleaning sponge-09759011e356e236", "teaspoon picked from bottom of sink then gets combined-9383ebda0dd1a6a8", "knife 2 picked up-9dc7889ae95cd3a1", "plastic box-a491b8dd4abb917b", "plastic box lid-96090eed4372250a" ], "reasoning": "1. I looked for the moment where the user was in a cupboard, and found at 18:15 they were rooting around, handling noodles (pack of noodles-eea89c246a427671) on the bottom shelf before reaching to the top shelf.\n2. On the top shelf, they pulled out a pack of kidney beans (pack of kidney beans-a1532836e8f040c6) at 18:19, and placed them on the counter.\n3. Moments later at 18:35, the user pulled out a bottle of vegetable oil (bottle of vegetable oil-b505615b6b75c447). Meaning that if I were the pack of kidney beans, I would have seen the outside world 16 seconds faster than the bottle of vegetable oil.", "referenced_object_masks": [ "bottle of vegetable oil-b505615b6b75c447", "pack of kidney beans-a1532836e8f040c6", "pack of noodles-eea89c246a427671" ] }, { "key": "P06-20240510-115307:19ff8d36818da7991f465099d5489b1658cdb373", "video_id": "P06-20240510-115307", "question": "Before adding onions to the butter in the pan, how many times do I hit the pan with the spatula?", "answer_choice_0": "12", "answer_choice_1": "14", "answer_choice_2": "13", "answer_choice_3": "15", "answer_choice_4": "16", "answer_id": 2, "answer": "13", "question_type": [ "Object Recognition", "Action Recognition", "Temporal Reasoning", "Egocentric & Spatial Perception", "Counting" ], "objects_of_interest": [ "box-a6de80b503cc5037", "lid of box-aa0f480fc7bacc9b", "pen-221e3f8d3556fb53", "notebook-4e82bf03e50a266f", "pressure cooker-60ec85db02d1f95d", "pack of butter-14283f4ef00fa4ac", "jar-cf9db484995ec4b7", "pan-9951a921eb099d4e", "chopping board-58f394ec7e76e257", "plate-ad28b1a0815291a5", "onion-4c28adf7a2bce4d0", "tomato-4a998a49b51a9777", "pack of green chilies-bad2b5d6099bbd2f", "green chilli-cac5e711e935a636", "tablecloth-597841fde63191c2", "weighing balance-29b9d04cbcf533e4", "knife-0b28b0a6aad23a1f", "half onion-92af8855ec4842ee", "another half onion-6abfa11c1cc36aac", "onion peel-132f2d52f1019492", "bowl-4e623e8b432823a9", "garlic-ecd80400c6167c70", "plant pot-4e12c444332910a8", "spoon-793772e04af7e3db", "spatula-e860f32c1e233801", "piece of butter-22f00ca09c2aa0bc", "spoon2-4a4b04e49eb59d5f", "jar2-683f08ea0a01349e", "lid of jar-febcf406ff8f1319", "cumin seeds-ac51d3c780f84ec9", "lid of pressure cooker-220957a3f29e5eef", "table cloth-080bafb5ae83f0bb", "spoon3-9ebfb19c0db753bd", "jar of salt-fc870eb14a1929d7", "jar3-b18030aef14c72db", "lid of jar2-0de4f6a3e1d6bfd9", "jar4-c7f79a3543509c60", "lid of jar4-219ae62cc632a3ee", "jar5-87a2fec6d90547ed", "lid of jar5-6bee2bdec7ce3f5c", "electric kettle-f2ac8034c82f2aeb" ], "reasoning": "1. At 04:26, I observe the user put the bowl (bowl-4e623e8b432823a9) filled with the chopped up onion (onion-4c28adf7a2bce4d0) behind the chopping board (chopping board-58f394ec7e76e257) in front of them.\n2. Between 08:45 and 08:48, I observe the user begin to unwrap the pack of butter (pack of butter-14283f4ef00fa4ac) and place it on the chopping board (chopping board-58f394ec7e76e257) in front of them.\n3. From 09:03 to 09:07, I observe the user cut the butter (pack of butter-14283f4ef00fa4ac) with the knife (knife-0b28b0a6aad23a1f) to slice off a smaller piece of butter (piece of butter-22f00ca09c2aa0bc).\n4. At 09:19, I observe the user add the piece of butter (piece of butter-22f00ca09c2aa0bc) to the pan (pan-9951a921eb099d4e) via the spoon (spoon-793772e04af7e3db).\n5. At 09:58, I observe the user bring the spatula (spatula-e860f32c1e233801) to the pan (pan-9951a921eb099d4e).\n6. At 10:08, I observe the user add the chopped up onion (onion-4c28adf7a2bce4d0) to the pan (pan-9951a921eb099d4e).\n7. Between 09:58 and 10:08, I count the user tap the spatula (spatula-e860f32c1e233801) against the pan (pan-9951a921eb099d4e) 13 times.", "referenced_object_masks": [ "pack of butter-14283f4ef00fa4ac", "onion-4c28adf7a2bce4d0", "pan-9951a921eb099d4e", "knife-0b28b0a6aad23a1f", "chopping board-58f394ec7e76e257", "spoon-793772e04af7e3db", "spatula-e860f32c1e233801", "bowl-4e623e8b432823a9", "piece of butter-22f00ca09c2aa0bc" ] }, { "key": "P06-20240510-115307:2e1a8ee986791297f979b6c9a059f117862b9b70", "video_id": "P06-20240510-115307", "question": "When I break apart the leaves and stems in front of the microwave, how close to me is the potted plant that I rinsed in the sink compared to the other potted plants?", "answer_choice_0": "Closest", "answer_choice_1": "2nd closest", "answer_choice_2": "Furthest", "answer_choice_3": "2nd furthest", "answer_choice_4": "They're all equidistant", "answer_id": 2, "answer": "Furthest", "question_type": [ "Object Recognition", "Action Recognition", "Temporal Reasoning", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "box-a6de80b503cc5037", "lid of box-aa0f480fc7bacc9b", "pen-221e3f8d3556fb53", "notebook-4e82bf03e50a266f", "pressure cooker-60ec85db02d1f95d", "pack of butter-14283f4ef00fa4ac", "jar-cf9db484995ec4b7", "pan-9951a921eb099d4e", "chopping board-58f394ec7e76e257", "plate-ad28b1a0815291a5", "onion-4c28adf7a2bce4d0", "tomato-4a998a49b51a9777", "pack of green chilies-bad2b5d6099bbd2f", "green chilli-cac5e711e935a636", "tablecloth-597841fde63191c2", "weighing balance-29b9d04cbcf533e4", "knife-0b28b0a6aad23a1f", "half onion-92af8855ec4842ee", "another half onion-6abfa11c1cc36aac", "onion peel-132f2d52f1019492", "bowl-4e623e8b432823a9", "garlic-ecd80400c6167c70", "plant pot-4e12c444332910a8", "spoon-793772e04af7e3db", "spatula-e860f32c1e233801", "piece of butter-22f00ca09c2aa0bc", "spoon2-4a4b04e49eb59d5f", "jar2-683f08ea0a01349e", "lid of jar-febcf406ff8f1319", "cumin seeds-ac51d3c780f84ec9", "lid of pressure cooker-220957a3f29e5eef", "table cloth-080bafb5ae83f0bb", "spoon3-9ebfb19c0db753bd", "jar of salt-fc870eb14a1929d7", "jar3-b18030aef14c72db", "lid of jar2-0de4f6a3e1d6bfd9", "jar4-c7f79a3543509c60", "lid of jar4-219ae62cc632a3ee", "jar5-87a2fec6d90547ed", "lid of jar5-6bee2bdec7ce3f5c", "electric kettle-f2ac8034c82f2aeb" ], "reasoning": "1. At 07:48, I locate the plant (plant pot-4e12c444332910a8) to the right of the drying rack on the counter.\n2. At 07:54, I identify 3 potted plants on the window sill behind the sink to the left of the drying rack .\n3. At 07:57, I observe the user rinsing the plant (plant pot-4e12c444332910a8) in the sink .\n4. At 12:42, I observe the user breaking apart the leaves and stems in front of the microwave on the far right side of the counter across from the sink and drying rack .\n5. At 12:42, I determine that the rinsed plant (plant pot-4e12c444332910a8) to the right of the drying rack is furthest from the user in comparison to the plants on the window sill .", "referenced_object_masks": [ "plant pot-4e12c444332910a8" ] }, { "key": "P06-20240510-115307:374bf9ddf90beb69e92462c2ab400e9e35dee554", "video_id": "P06-20240510-115307", "question": "What food item did I chop after I gathered the onions off the cutting board?", "answer_choice_0": "Apple", "answer_choice_1": "Green Beans", "answer_choice_2": "Cilantro", "answer_choice_3": "Garlic", "answer_choice_4": "Tomato", "answer_id": 4, "answer": "Tomato", "question_type": [ "Object Recognition" ], "objects_of_interest": [ "box-a6de80b503cc5037", "lid of box-aa0f480fc7bacc9b", "pen-221e3f8d3556fb53", "notebook-4e82bf03e50a266f", "pressure cooker-60ec85db02d1f95d", "pack of butter-14283f4ef00fa4ac", "jar-cf9db484995ec4b7", "pan-9951a921eb099d4e", "chopping board-58f394ec7e76e257", "plate-ad28b1a0815291a5", "onion-4c28adf7a2bce4d0", "tomato-4a998a49b51a9777", "pack of green chilies-bad2b5d6099bbd2f", "green chilli-cac5e711e935a636", "tablecloth-597841fde63191c2", "weighing balance-29b9d04cbcf533e4", "knife-0b28b0a6aad23a1f", "half onion-92af8855ec4842ee", "another half onion-6abfa11c1cc36aac", "onion peel-132f2d52f1019492", "bowl-4e623e8b432823a9", "garlic-ecd80400c6167c70", "plant pot-4e12c444332910a8", "spoon-793772e04af7e3db", "spatula-e860f32c1e233801", "piece of butter-22f00ca09c2aa0bc", "spoon2-4a4b04e49eb59d5f", "jar2-683f08ea0a01349e", "lid of jar-febcf406ff8f1319", "cumin seeds-ac51d3c780f84ec9", "lid of pressure cooker-220957a3f29e5eef", "table cloth-080bafb5ae83f0bb", "spoon3-9ebfb19c0db753bd", "jar of salt-fc870eb14a1929d7", "jar3-b18030aef14c72db", "lid of jar2-0de4f6a3e1d6bfd9", "jar4-c7f79a3543509c60", "lid of jar4-219ae62cc632a3ee", "jar5-87a2fec6d90547ed", "lid of jar5-6bee2bdec7ce3f5c", "electric kettle-f2ac8034c82f2aeb" ], "reasoning": "1. At 04:10, I saw the user begin to scrape the onions (onion-4c28adf7a2bce4d0) off of the chopping board (chopping board-58f394ec7e76e257).\n2. At 04:24, the user finished scraping the onions (onion-4c28adf7a2bce4d0) off the cutting board (chopping board-58f394ec7e76e257).\n3. At 04:29, I saw the user grab the tomato (tomato-4a998a49b51a9777).\n4. At 04:32, I saw the user slicing the tomato (tomato-4a998a49b51a9777).", "referenced_object_masks": [ "chopping board-58f394ec7e76e257", "tomato-4a998a49b51a9777", "onion-4c28adf7a2bce4d0" ] }, { "key": "P06-20240510-115307:577e62e5d8691fbfe89baa67d148fd427a25af0e", "video_id": "P06-20240510-115307", "question": "What is the numerical expression on the scale when the tomato is being weighed?", "answer_choice_0": "0.50", "answer_choice_1": "0.30", "answer_choice_2": "0.60", "answer_choice_3": "1.0", "answer_choice_4": "0.90", "answer_id": 1, "answer": "0.30", "question_type": [ "Numerical Reasoning", "Object Recognition" ], "objects_of_interest": [ "box-a6de80b503cc5037", "lid of box-aa0f480fc7bacc9b", "pen-221e3f8d3556fb53", "notebook-4e82bf03e50a266f", "pressure cooker-60ec85db02d1f95d", "pack of butter-14283f4ef00fa4ac", "jar-cf9db484995ec4b7", "pan-9951a921eb099d4e", "chopping board-58f394ec7e76e257", "plate-ad28b1a0815291a5", "onion-4c28adf7a2bce4d0", "tomato-4a998a49b51a9777", "pack of green chilies-bad2b5d6099bbd2f", "green chilli-cac5e711e935a636", "tablecloth-597841fde63191c2", "weighing balance-29b9d04cbcf533e4", "knife-0b28b0a6aad23a1f", "half onion-92af8855ec4842ee", "another half onion-6abfa11c1cc36aac", "onion peel-132f2d52f1019492", "bowl-4e623e8b432823a9", "garlic-ecd80400c6167c70", "plant pot-4e12c444332910a8", "spoon-793772e04af7e3db", "spatula-e860f32c1e233801", "piece of butter-22f00ca09c2aa0bc", "spoon2-4a4b04e49eb59d5f", "jar2-683f08ea0a01349e", "lid of jar-febcf406ff8f1319", "cumin seeds-ac51d3c780f84ec9", "lid of pressure cooker-220957a3f29e5eef", "table cloth-080bafb5ae83f0bb", "spoon3-9ebfb19c0db753bd", "jar of salt-fc870eb14a1929d7", "jar3-b18030aef14c72db", "lid of jar2-0de4f6a3e1d6bfd9", "jar4-c7f79a3543509c60", "lid of jar4-219ae62cc632a3ee", "jar5-87a2fec6d90547ed", "lid of jar5-6bee2bdec7ce3f5c", "electric kettle-f2ac8034c82f2aeb" ], "reasoning": "1. At 01:55, I saw the user place the tomato on the scale (weighing balance-29b9d04cbcf533e4).\n2. At 02:02, I saw the user record the weight of 0.30.\n3. The numerical expression on the scale is 0.30.", "referenced_object_masks": [ "weighing balance-29b9d04cbcf533e4" ] }, { "key": "P06-20240510-115307:6ad77d0bac48cb3ade33dfbb13751c0f248a775f", "video_id": "P06-20240510-115307", "question": "If I added the vegetables to the pan in reverse order, what would it be?", "answer_choice_0": "Garlic, tomato, green chilli, onion", "answer_choice_1": "Garlic, green chilli, tomato, onion", "answer_choice_2": "Parsley, garlic, tomato, onion", "answer_choice_3": "Onion, garlic, tomato, green chilli", "answer_choice_4": "Green chilli, garlic, onion, tomato", "answer_id": 1, "answer": "Garlic, green chilli, tomato, onion", "question_type": [ "Object Recognition", "Action Recognition", "Temporal Reasoning", "Egocentric & Spatial Perception", "Counting" ], "objects_of_interest": [ "box-a6de80b503cc5037", "lid of box-aa0f480fc7bacc9b", "pen-221e3f8d3556fb53", "notebook-4e82bf03e50a266f", "pressure cooker-60ec85db02d1f95d", "pack of butter-14283f4ef00fa4ac", "jar-cf9db484995ec4b7", "pan-9951a921eb099d4e", "chopping board-58f394ec7e76e257", "plate-ad28b1a0815291a5", "onion-4c28adf7a2bce4d0", "tomato-4a998a49b51a9777", "pack of green chilies-bad2b5d6099bbd2f", "green chilli-cac5e711e935a636", "tablecloth-597841fde63191c2", "weighing balance-29b9d04cbcf533e4", "knife-0b28b0a6aad23a1f", "half onion-92af8855ec4842ee", "another half onion-6abfa11c1cc36aac", "onion peel-132f2d52f1019492", "bowl-4e623e8b432823a9", "garlic-ecd80400c6167c70", "plant pot-4e12c444332910a8", "spoon-793772e04af7e3db", "spatula-e860f32c1e233801", "piece of butter-22f00ca09c2aa0bc", "spoon2-4a4b04e49eb59d5f", "jar2-683f08ea0a01349e", "lid of jar-febcf406ff8f1319", "cumin seeds-ac51d3c780f84ec9", "lid of pressure cooker-220957a3f29e5eef", "table cloth-080bafb5ae83f0bb", "spoon3-9ebfb19c0db753bd", "jar of salt-fc870eb14a1929d7", "jar3-b18030aef14c72db", "lid of jar2-0de4f6a3e1d6bfd9", "jar4-c7f79a3543509c60", "lid of jar4-219ae62cc632a3ee", "jar5-87a2fec6d90547ed", "lid of jar5-6bee2bdec7ce3f5c", "electric kettle-f2ac8034c82f2aeb" ], "reasoning": "1. I locate the user holding the spatula (spatula-e860f32c1e233801) with the pan (pan-9951a921eb099d4e) in front of them.\n2. At 10:19, I observe the user add the chopped up onion (onion-4c28adf7a2bce4d0) to the pan (pan-9951a921eb099d4e).\n3. At 14:25, I identify the vegetables (tomato-4a998a49b51a9777, green chilli-cac5e711e935a636, garlic-ecd80400c6167c70) placed on the chopping board (chopping board-58f394ec7e76e257) in the following order, each further from the center than the last: garlic, green chilli, tomato. \n4. Between 14:25 and 14:32, I observe the user begin to use the knife (knife-0b28b0a6aad23a1f) to scrape the vegetables (tomato-4a998a49b51a9777, green chilli-cac5e711e935a636, garlic-ecd80400c6167c70) from the chopping board (chopping board-58f394ec7e76e257) into the pan (pan-9951a921eb099d4e).\n5. I determine the order in which the vegetables (onion-4c28adf7a2bce4d0, tomato-4a998a49b51a9777, green chilli-cac5e711e935a636, garlic-ecd80400c6167c70) are added to the pan (pan-9951a921eb099d4e): onion, tomato, green chilli, garlic.\n6. I determine the reverse order would be garlic, green chilli, tomato, onion.", "referenced_object_masks": [ "onion-4c28adf7a2bce4d0", "tomato-4a998a49b51a9777", "green chilli-cac5e711e935a636", "garlic-ecd80400c6167c70", "pan-9951a921eb099d4e", "knife-0b28b0a6aad23a1f", "chopping board-58f394ec7e76e257", "spatula-e860f32c1e233801" ] }, { "key": "P06-20240510-115307:6f9c9f4898147ccaa22efea9430457407b33b17b", "video_id": "P06-20240510-115307", "question": "After watering the plant, what did I do with the food item I grabbed after drying my hands on the towel?", "answer_choice_0": "Mixed it into a dish", "answer_choice_1": "Cooked it", "answer_choice_2": "Sliced and weighed it", "answer_choice_3": "Put it in the sink", "answer_choice_4": "Wiped it off", "answer_id": 2, "answer": "Sliced and weighed it", "question_type": [ "Action Recognition", "Event Occurence", "Situational Awareness" ], "objects_of_interest": [ "box-a6de80b503cc5037", "lid of box-aa0f480fc7bacc9b", "pen-221e3f8d3556fb53", "notebook-4e82bf03e50a266f", "pressure cooker-60ec85db02d1f95d", "pack of butter-14283f4ef00fa4ac", "jar-cf9db484995ec4b7", "pan-9951a921eb099d4e", "chopping board-58f394ec7e76e257", "plate-ad28b1a0815291a5", "onion-4c28adf7a2bce4d0", "tomato-4a998a49b51a9777", "pack of green chilies-bad2b5d6099bbd2f", "green chilli-cac5e711e935a636", "tablecloth-597841fde63191c2", "weighing balance-29b9d04cbcf533e4", "knife-0b28b0a6aad23a1f", "half onion-92af8855ec4842ee", "another half onion-6abfa11c1cc36aac", "onion peel-132f2d52f1019492", "bowl-4e623e8b432823a9", "garlic-ecd80400c6167c70", "plant pot-4e12c444332910a8", "spoon-793772e04af7e3db", "spatula-e860f32c1e233801", "piece of butter-22f00ca09c2aa0bc", "spoon2-4a4b04e49eb59d5f", "jar2-683f08ea0a01349e", "lid of jar-febcf406ff8f1319", "cumin seeds-ac51d3c780f84ec9", "lid of pressure cooker-220957a3f29e5eef", "table cloth-080bafb5ae83f0bb", "spoon3-9ebfb19c0db753bd", "jar of salt-fc870eb14a1929d7", "jar3-b18030aef14c72db", "lid of jar2-0de4f6a3e1d6bfd9", "jar4-c7f79a3543509c60", "lid of jar4-219ae62cc632a3ee", "jar5-87a2fec6d90547ed", "lid of jar5-6bee2bdec7ce3f5c", "electric kettle-f2ac8034c82f2aeb" ], "reasoning": "1. At 08:35, I noticed the user picking up the towel (table cloth-080bafb5ae83f0bb) to dry their hands.\n2. At 08:46, I saw the user walk over to the counter and grab the butter (pack of butter-14283f4ef00fa4ac).\n3. At 09:05, I saw the user slice the butter (pack of butter-14283f4ef00fa4ac.\n4. At 09:09, I saw the user place the butter (pack of butter-14283f4ef00fa4ac) on the scale (weighing balance-29b9d04cbcf533e4) to be weighed.\n5. The butter (pack of butter-14283f4ef00fa4ac) was sliced and weighed.", "referenced_object_masks": [ "table cloth-080bafb5ae83f0bb", "weighing balance-29b9d04cbcf533e4", "pack of butter-14283f4ef00fa4ac" ] }, { "key": "P06-20240510-115307:834e876b218db79bde11bd5bb4c7899330a3493c", "video_id": "P06-20240510-115307", "question": "How much time lasped between the last time I added water to the dish until the end of the video?", "answer_choice_0": "45 seconds", "answer_choice_1": "30 seconds", "answer_choice_2": "3 minutes and 5 seconds", "answer_choice_3": "1 minute and 11 seconds", "answer_choice_4": "2 minutes", "answer_id": 3, "answer": "1 minute and 11 seconds", "question_type": [ "Situational Awareness", "Temporal Reasoning", "Event Occurence", "Action Recognition" ], "objects_of_interest": [ "box-a6de80b503cc5037", "lid of box-aa0f480fc7bacc9b", "pen-221e3f8d3556fb53", "notebook-4e82bf03e50a266f", "pressure cooker-60ec85db02d1f95d", "pack of butter-14283f4ef00fa4ac", "jar-cf9db484995ec4b7", "pan-9951a921eb099d4e", "chopping board-58f394ec7e76e257", "plate-ad28b1a0815291a5", "onion-4c28adf7a2bce4d0", "tomato-4a998a49b51a9777", "pack of green chilies-bad2b5d6099bbd2f", "green chilli-cac5e711e935a636", "tablecloth-597841fde63191c2", "weighing balance-29b9d04cbcf533e4", "knife-0b28b0a6aad23a1f", "half onion-92af8855ec4842ee", "another half onion-6abfa11c1cc36aac", "onion peel-132f2d52f1019492", "bowl-4e623e8b432823a9", "garlic-ecd80400c6167c70", "plant pot-4e12c444332910a8", "spoon-793772e04af7e3db", "spatula-e860f32c1e233801", "piece of butter-22f00ca09c2aa0bc", "spoon2-4a4b04e49eb59d5f", "jar2-683f08ea0a01349e", "lid of jar-febcf406ff8f1319", "cumin seeds-ac51d3c780f84ec9", "lid of pressure cooker-220957a3f29e5eef", "table cloth-080bafb5ae83f0bb", "spoon3-9ebfb19c0db753bd", "jar of salt-fc870eb14a1929d7", "jar3-b18030aef14c72db", "lid of jar2-0de4f6a3e1d6bfd9", "jar4-c7f79a3543509c60", "lid of jar4-219ae62cc632a3ee", "jar5-87a2fec6d90547ed", "lid of jar5-6bee2bdec7ce3f5c", "electric kettle-f2ac8034c82f2aeb" ], "reasoning": "1. At 16:46, I saw the user the pouring water into in the skillet (pan-9951a921eb099d4e).\n2. At 17:04, I saw the user mix in the water and other contents.\n3. At 17:22, I saw the user pour more water into the skillet (pan-9951a921eb099d4e).\n4. At 18:07, I saw the user turn off the eye of the stove .\n5. The video ended at 18:33.\n6. There was 1 minute and 11 seconds lasped between the time of the last bit of water being poured into the skillet (pan-9951a921eb099d4e) and the end of the video.", "referenced_object_masks": [ "pan-9951a921eb099d4e" ] }, { "key": "P06-20240510-115307:8bfa21385bc94bd9655edc336438315b2134471b", "video_id": "P06-20240510-115307", "question": "How many times do I slice the onion before manually removing the peel?", "answer_choice_0": "16", "answer_choice_1": "13", "answer_choice_2": "14", "answer_choice_3": "12", "answer_choice_4": "15", "answer_id": 2, "answer": "14", "question_type": [ "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counting" ], "objects_of_interest": [ "box-a6de80b503cc5037", "lid of box-aa0f480fc7bacc9b", "pen-221e3f8d3556fb53", "notebook-4e82bf03e50a266f", "pressure cooker-60ec85db02d1f95d", "pack of butter-14283f4ef00fa4ac", "jar-cf9db484995ec4b7", "pan-9951a921eb099d4e", "chopping board-58f394ec7e76e257", "plate-ad28b1a0815291a5", "onion-4c28adf7a2bce4d0", "tomato-4a998a49b51a9777", "pack of green chilies-bad2b5d6099bbd2f", "green chilli-cac5e711e935a636", "tablecloth-597841fde63191c2", "weighing balance-29b9d04cbcf533e4", "knife-0b28b0a6aad23a1f", "half onion-92af8855ec4842ee", "another half onion-6abfa11c1cc36aac", "onion peel-132f2d52f1019492", "bowl-4e623e8b432823a9", "garlic-ecd80400c6167c70", "plant pot-4e12c444332910a8", "spoon-793772e04af7e3db", "spatula-e860f32c1e233801", "piece of butter-22f00ca09c2aa0bc", "spoon2-4a4b04e49eb59d5f", "jar2-683f08ea0a01349e", "lid of jar-febcf406ff8f1319", "cumin seeds-ac51d3c780f84ec9", "lid of pressure cooker-220957a3f29e5eef", "table cloth-080bafb5ae83f0bb", "spoon3-9ebfb19c0db753bd", "jar of salt-fc870eb14a1929d7", "jar3-b18030aef14c72db", "lid of jar2-0de4f6a3e1d6bfd9", "jar4-c7f79a3543509c60", "lid of jar4-219ae62cc632a3ee", "jar5-87a2fec6d90547ed", "lid of jar5-6bee2bdec7ce3f5c", "electric kettle-f2ac8034c82f2aeb" ], "reasoning": "1. From 02:16 to 02:21, I observe the user slice the onion 12 times (onion-4c28adf7a2bce4d0) with the knife (knife-0b28b0a6aad23a1f).\n2. At 02:24, I observe the user move the first onion half (half onion-92af8855ec4842ee) in front of the paper towels on the counter.\n3. From 02:25 to 02:27, I observe the user make 2 slices to the second onion half (another half onion-6abfa11c1cc36aac) with the knife (knife-0b28b0a6aad23a1f).\n4. At 02:30 I observe the user begin to remove the large peel from the onion (another half onion-6abfa11c1cc36aac) with their hand.\n5. I determined the user sliced the onion 14 times before manually peeling the onion (another half onion-6abfa11c1cc36aac).", "referenced_object_masks": [ "another half onion-6abfa11c1cc36aac", "onion-4c28adf7a2bce4d0", "knife-0b28b0a6aad23a1f", "half onion-92af8855ec4842ee" ] }, { "key": "P06-20240510-115307:98235b691209ebd882c6de0dc03be81dfcc5489f", "video_id": "P06-20240510-115307", "question": "After rinsing off the vegetables, how many times did I fling off the excess water?", "answer_choice_0": "1", "answer_choice_1": "9", "answer_choice_2": "6", "answer_choice_3": "5", "answer_choice_4": "4", "answer_id": 2, "answer": "6", "question_type": [ "Action Recognition", "Counting", "Event Occurence", "Numerical Reasoning", "Situational Awareness" ], "objects_of_interest": [ "box-a6de80b503cc5037", "lid of box-aa0f480fc7bacc9b", "pen-221e3f8d3556fb53", "notebook-4e82bf03e50a266f", "pressure cooker-60ec85db02d1f95d", "pack of butter-14283f4ef00fa4ac", "jar-cf9db484995ec4b7", "pan-9951a921eb099d4e", "chopping board-58f394ec7e76e257", "plate-ad28b1a0815291a5", "onion-4c28adf7a2bce4d0", "tomato-4a998a49b51a9777", "pack of green chilies-bad2b5d6099bbd2f", "green chilli-cac5e711e935a636", "tablecloth-597841fde63191c2", "weighing balance-29b9d04cbcf533e4", "knife-0b28b0a6aad23a1f", "half onion-92af8855ec4842ee", "another half onion-6abfa11c1cc36aac", "onion peel-132f2d52f1019492", "bowl-4e623e8b432823a9", "garlic-ecd80400c6167c70", "plant pot-4e12c444332910a8", "spoon-793772e04af7e3db", "spatula-e860f32c1e233801", "piece of butter-22f00ca09c2aa0bc", "spoon2-4a4b04e49eb59d5f", "jar2-683f08ea0a01349e", "lid of jar-febcf406ff8f1319", "cumin seeds-ac51d3c780f84ec9", "lid of pressure cooker-220957a3f29e5eef", "table cloth-080bafb5ae83f0bb", "spoon3-9ebfb19c0db753bd", "jar of salt-fc870eb14a1929d7", "jar3-b18030aef14c72db", "lid of jar2-0de4f6a3e1d6bfd9", "jar4-c7f79a3543509c60", "lid of jar4-219ae62cc632a3ee", "jar5-87a2fec6d90547ed", "lid of jar5-6bee2bdec7ce3f5c", "electric kettle-f2ac8034c82f2aeb" ], "reasoning": "1. At 00:50, I saw the user gathering the vegetables on a plate (plate-ad28b1a0815291a5).\n2. At 00:56, I saw the user put a tomato (tomato-4a998a49b51a9777) on the plate (plate-ad28b1a0815291a5) on the counter.\n3. At 01:02, I watched the user take a fresh green vegetable out of a plastic bag.\n4. At 01:16, I watched the user take the green chilli (green chilli-cac5e711e935a636) and tomato (tomato-4a998a49b51a9777) to the sink .\n5. At 01:20, I watched the user rinse the tomato (tomato-4a998a49b51a9777) and green beans .\n6. At 01:23, I saw the user turn off the water.\n7. At 01:25, I saw the user start to fling off the excess water.\n8. The user flung off the excess water 6 times.", "referenced_object_masks": [ "plate-ad28b1a0815291a5", "tomato-4a998a49b51a9777", "green chilli-cac5e711e935a636" ] }, { "key": "P06-20240510-115307:e9220489ac2811d6a4203b9c2c982edca24dac94", "video_id": "P06-20240510-115307", "question": "When I grab the knife from the drying rack, where is the paper towel roll further from the stovetop located in relation to me?", "answer_choice_0": "Directly ahead", "answer_choice_1": "By my feet", "answer_choice_2": "Back left", "answer_choice_3": "Back right", "answer_choice_4": "Above", "answer_id": 3, "answer": "Back right", "question_type": [ "Object Recognition", "Action Recognition", "Temporal Reasoning", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "box-a6de80b503cc5037", "lid of box-aa0f480fc7bacc9b", "pen-221e3f8d3556fb53", "notebook-4e82bf03e50a266f", "pressure cooker-60ec85db02d1f95d", "pack of butter-14283f4ef00fa4ac", "jar-cf9db484995ec4b7", "pan-9951a921eb099d4e", "chopping board-58f394ec7e76e257", "plate-ad28b1a0815291a5", "onion-4c28adf7a2bce4d0", "tomato-4a998a49b51a9777", "pack of green chilies-bad2b5d6099bbd2f", "green chilli-cac5e711e935a636", "tablecloth-597841fde63191c2", "weighing balance-29b9d04cbcf533e4", "knife-0b28b0a6aad23a1f", "half onion-92af8855ec4842ee", "another half onion-6abfa11c1cc36aac", "onion peel-132f2d52f1019492", "bowl-4e623e8b432823a9", "garlic-ecd80400c6167c70", "plant pot-4e12c444332910a8", "spoon-793772e04af7e3db", "spatula-e860f32c1e233801", "piece of butter-22f00ca09c2aa0bc", "spoon2-4a4b04e49eb59d5f", "jar2-683f08ea0a01349e", "lid of jar-febcf406ff8f1319", "cumin seeds-ac51d3c780f84ec9", "lid of pressure cooker-220957a3f29e5eef", "table cloth-080bafb5ae83f0bb", "spoon3-9ebfb19c0db753bd", "jar of salt-fc870eb14a1929d7", "jar3-b18030aef14c72db", "lid of jar2-0de4f6a3e1d6bfd9", "jar4-c7f79a3543509c60", "lid of jar4-219ae62cc632a3ee", "jar5-87a2fec6d90547ed", "lid of jar5-6bee2bdec7ce3f5c", "electric kettle-f2ac8034c82f2aeb" ], "reasoning": "1. At 0:15, I locate the close paper towel roll to the immediate left of the stove in front of the user and the far paper towel roll on the far left end of the counter.\n2. At 1:47, I locate the close paper towel roll to the user's right and the far paper towel roll to the user's left.\n3. At 02:08, I observe the user turn around to face the knife (knife-0b28b0a6aad23a1f) in the drying rack in the right sink .\n4. At 02:11, I observe the user grab the knife (knife-0b28b0a6aad23a1f).\n5. I determine that the user would have to turn to their back right to see the far paper towel roll at 02:11.", "referenced_object_masks": [ "knife-0b28b0a6aad23a1f" ] }, { "key": "P06-20240510-121619:25844edceb44426eb9ab5985833546c8471027cb", "video_id": "P06-20240510-121619", "question": "What is behind and slightly to the left of the item I place in the drawer after immediately throwing away the onion skin?", "answer_choice_0": "Clear plastic bag", "answer_choice_1": "Silver spoon", "answer_choice_2": "Black scissors", "answer_choice_3": "Pair of glasses", "answer_choice_4": "Wooden spoon", "answer_id": 2, "answer": "Black scissors", "question_type": [ "Object Recognition", "Egocentric & Spatial Perception", "Action Recognition" ], "objects_of_interest": [ "pressure cooker-c7291b6669d0b764", "spatula-8b46dfa81b5e4725", "spoon-54be405bb65e20c6", "bowl-e72cf25be22cf49a", "lid of pressure cooker-3952265c5c7cf8b0", "pack of butter-f6cd01da37f5a5f3", "spoon3-b57e38a4e9b10408", "jar-5e714692d6a297e2", "lid of jar-8b4f637577fdb1c6", "pen-8f0c8d589228c3f8", "spoon4-f59ca078f3e3f8ed", "knife-cdc841b0da7680a5", "piece of butter-bc492bce55f1bd59", "chopping board-697e69f256a8ff22", "plate-6db2d4b2a586c11c", "spoon2-f83b2262005ab946", "onion-f40de62097bd87c7", "box-b61b1fcf0d61d5b2", "lid of box-708a7c6309a9cd99", "onion peel-0dfb1de9d4f8f2f4", "jar2-a2d57ffb658ae029", "salt-fd683237f238913d", "jar3-d4eac3294fbab1b3", "jar4-c68f0f293110bebb", "new jar-92f2c76c216ac496", "jar5-cf4cf20340497289", "spoon5-8d666bfa72ab4f8f", "jar6-092d0b62ff1ead30", "pack of green chillies-3d5a171c796c9dc1", "jar7-94b289e651b03c77", "jar8-9bd828c8379e014e", "plate2-73578c6d4a37373a", "bowl2-8b088fde36f982a6", "bowl3-0b777b1f3e40786b", "tablecloth-2a7ba0a85822bd52", "jar9-2223b8f8d1af1687", "jar lid-49d739410e181c4b", "cup-f974528413bb1e60", "sponge-2bd4b188a0513bd2", "bottle of dishwashing liquid-cf3e674883d4e9f2", "some spoons-8ba1a55e9a48d5a1", "spoons2-1a03670c4b457e48", "spoons3-7b544c81fb76c481", "cloth2-03c83a8cb3ed9148", "saucepan-0505d8b4a3d58914", "lid2-33bbe47170fb7bee", "lid3-623e13525ad1e2da", "lid4-9dba6760f94ad48c", "frying pan-86c0b136080112a7", "pack of wipes-fbc584de7bbbefff", "wipe-638dd3157a6703ab", "another wipe-4d82c60a9045dd3e" ], "reasoning": "1. First, I determine when the user discards an onion skin (onion peel-0dfb1de9d4f8f2f4). This occurs at 07:52.\n2. Next, I rewind the video to determine what item the user places in a drawer after this. I noticed at 07:11, the user picks up half a piece of red onion (onion-f40de62097bd87c7) from the counter and takes it to the fridge to store in a container (box-b61b1fcf0d61d5b2) at 07:41 but decides against it. The user removes the onion half (onion-f40de62097bd87c7) from the container (box-b61b1fcf0d61d5b2) at 07:43, discards the skin (onion peel-0dfb1de9d4f8f2f4), and walks it back to the counter to store in the drawer instead. \n3. As the user places the onion half (onion-f40de62097bd87c7) in the drawer at 07:57, the onion (onion-f40de62097bd87c7) touches the black handles of a pair of scissors , which are positioned behind and slightly left of the onion half (onion-f40de62097bd87c7).", "referenced_object_masks": [ "box-b61b1fcf0d61d5b2", "onion-f40de62097bd87c7", "onion peel-0dfb1de9d4f8f2f4" ] }, { "key": "P06-20240510-121619:29609d8ef0d9f0e5166e6bcaa44aa02abd575b8c", "video_id": "P06-20240510-121619", "question": "What will eventually happen to the item I placed in the drawer in front of me after I walked away from the fridge if I don't remove it soon?", "answer_choice_0": "It will collect dust", "answer_choice_1": "It will decompose", "answer_choice_2": "It will become stale", "answer_choice_3": "It will remain for later use", "answer_choice_4": "It will harden", "answer_id": 1, "answer": "It will decompose", "question_type": [ "Counterfactual", "Action Recognition", "Object Recognition", "Egocentric & Spatial Perception", "Cause and Effect" ], "objects_of_interest": [ "pressure cooker-c7291b6669d0b764", "spatula-8b46dfa81b5e4725", "spoon-54be405bb65e20c6", "bowl-e72cf25be22cf49a", "lid of pressure cooker-3952265c5c7cf8b0", "pack of butter-f6cd01da37f5a5f3", "spoon3-b57e38a4e9b10408", "jar-5e714692d6a297e2", "lid of jar-8b4f637577fdb1c6", "pen-8f0c8d589228c3f8", "spoon4-f59ca078f3e3f8ed", "knife-cdc841b0da7680a5", "piece of butter-bc492bce55f1bd59", "chopping board-697e69f256a8ff22", "plate-6db2d4b2a586c11c", "spoon2-f83b2262005ab946", "onion-f40de62097bd87c7", "box-b61b1fcf0d61d5b2", "lid of box-708a7c6309a9cd99", "onion peel-0dfb1de9d4f8f2f4", "jar2-a2d57ffb658ae029", "salt-fd683237f238913d", "jar3-d4eac3294fbab1b3", "jar4-c68f0f293110bebb", "new jar-92f2c76c216ac496", "jar5-cf4cf20340497289", "spoon5-8d666bfa72ab4f8f", "jar6-092d0b62ff1ead30", "pack of green chillies-3d5a171c796c9dc1", "jar7-94b289e651b03c77", "jar8-9bd828c8379e014e", "plate2-73578c6d4a37373a", "bowl2-8b088fde36f982a6", "bowl3-0b777b1f3e40786b", "tablecloth-2a7ba0a85822bd52", "jar9-2223b8f8d1af1687", "jar lid-49d739410e181c4b", "cup-f974528413bb1e60", "sponge-2bd4b188a0513bd2", "bottle of dishwashing liquid-cf3e674883d4e9f2", "some spoons-8ba1a55e9a48d5a1", "spoons2-1a03670c4b457e48", "spoons3-7b544c81fb76c481", "cloth2-03c83a8cb3ed9148", "saucepan-0505d8b4a3d58914", "lid2-33bbe47170fb7bee", "lid3-623e13525ad1e2da", "lid4-9dba6760f94ad48c", "frying pan-86c0b136080112a7", "pack of wipes-fbc584de7bbbefff", "wipe-638dd3157a6703ab", "another wipe-4d82c60a9045dd3e" ], "reasoning": "1. First, I locate the timestamp when the user places an item in a drawer in front of them after walking away from the fridge. This occurs at 07:56.\n2. Next, I rewind the video to determine what the item is. I noticed at 07:11, the user picks up half a piece of red onion (onion-f40de62097bd87c7) from the counter and takes it to the fridge to store in a container (box-b61b1fcf0d61d5b2) at 07:41 but decides against it. The user removes the onion half (onion-f40de62097bd87c7) from the container (box-b61b1fcf0d61d5b2) at 07:43, discards the skin (onion peel-0dfb1de9d4f8f2f4), and walks it back to the counter to store in the drawer instead. \n3. Since the stored item is a cut piece of red onion (onion-f40de62097bd87c7), if the user doesn't remove it from the drawer soon, the onion half (onion-f40de62097bd87c7) will smell and eventually decompose.", "referenced_object_masks": [ "box-b61b1fcf0d61d5b2", "onion-f40de62097bd87c7", "onion peel-0dfb1de9d4f8f2f4" ] }, { "key": "P06-20240510-121619:4ca686788b917427e3cd222d9571099741b094f4", "video_id": "P06-20240510-121619", "question": "When I'm cleaning the pressure cooker lid, where is the full bottle of oil in relation to me?", "answer_choice_0": "In the cabinet behind me", "answer_choice_1": "On the counter to my left", "answer_choice_2": "On the counter behind me", "answer_choice_3": "On the counter to my right", "answer_choice_4": "On the windowsill in front of me", "answer_id": 0, "answer": "In the cabinet behind me", "question_type": [ "Action Recognition", "Object Recognition", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "pressure cooker-c7291b6669d0b764", "spatula-8b46dfa81b5e4725", "spoon-54be405bb65e20c6", "bowl-e72cf25be22cf49a", "lid of pressure cooker-3952265c5c7cf8b0", "pack of butter-f6cd01da37f5a5f3", "spoon3-b57e38a4e9b10408", "jar-5e714692d6a297e2", "lid of jar-8b4f637577fdb1c6", "pen-8f0c8d589228c3f8", "spoon4-f59ca078f3e3f8ed", "knife-cdc841b0da7680a5", "piece of butter-bc492bce55f1bd59", "chopping board-697e69f256a8ff22", "plate-6db2d4b2a586c11c", "spoon2-f83b2262005ab946", "onion-f40de62097bd87c7", "box-b61b1fcf0d61d5b2", "lid of box-708a7c6309a9cd99", "onion peel-0dfb1de9d4f8f2f4", "jar2-a2d57ffb658ae029", "salt-fd683237f238913d", "jar3-d4eac3294fbab1b3", "jar4-c68f0f293110bebb", "new jar-92f2c76c216ac496", "jar5-cf4cf20340497289", "spoon5-8d666bfa72ab4f8f", "jar6-092d0b62ff1ead30", "pack of green chillies-3d5a171c796c9dc1", "jar7-94b289e651b03c77", "jar8-9bd828c8379e014e", "plate2-73578c6d4a37373a", "bowl2-8b088fde36f982a6", "bowl3-0b777b1f3e40786b", "tablecloth-2a7ba0a85822bd52", "jar9-2223b8f8d1af1687", "jar lid-49d739410e181c4b", "cup-f974528413bb1e60", "sponge-2bd4b188a0513bd2", "bottle of dishwashing liquid-cf3e674883d4e9f2", "some spoons-8ba1a55e9a48d5a1", "spoons2-1a03670c4b457e48", "spoons3-7b544c81fb76c481", "cloth2-03c83a8cb3ed9148", "saucepan-0505d8b4a3d58914", "lid2-33bbe47170fb7bee", "lid3-623e13525ad1e2da", "lid4-9dba6760f94ad48c", "frying pan-86c0b136080112a7", "pack of wipes-fbc584de7bbbefff", "wipe-638dd3157a6703ab", "another wipe-4d82c60a9045dd3e" ], "reasoning": "1. First I determine when the user is cleaning the pressure cooker lid (lid of pressure cooker-3952265c5c7cf8b0). This occurs at 10:51 while the user is standing at the sink. \n2. Next, I locate the full bottle of oil . I found it at 08:23, situated inside the upper cabinet to the right of the stove on the left side. The bottle is visible on the first shelf on the right. \n3. Studying the kitchen's layout, I determined that the sink is on the left while the cabinet with the oil is on the right. Since the user is standing at the sink while cleaning the pressure cooker lid (lid of pressure cooker-3952265c5c7cf8b0), the cabinet with the full bottle of oil is positioned behind the user.", "referenced_object_masks": [ "lid of pressure cooker-3952265c5c7cf8b0" ] }, { "key": "P06-20240510-121619:4e781593a201d97abbde43d4b6f185dac2f668b2", "video_id": "P06-20240510-121619", "question": "Immediately after tapping the wooden spoon against the skillet 6 times and then putting the spoon down, where is my left thumb in relation to my right thumb?", "answer_choice_0": "Underneath and in front", "answer_choice_1": "Covered by right hand", "answer_choice_2": "On top and behind", "answer_choice_3": "Interlocked together", "answer_choice_4": "Side by side", "answer_id": 2, "answer": "On top and behind", "question_type": [ "Action Recognition", "Counting", "Egocentric & Spatial Perception", "Object Recognition" ], "objects_of_interest": [ "pressure cooker-c7291b6669d0b764", "spatula-8b46dfa81b5e4725", "spoon-54be405bb65e20c6", "bowl-e72cf25be22cf49a", "lid of pressure cooker-3952265c5c7cf8b0", "pack of butter-f6cd01da37f5a5f3", "spoon3-b57e38a4e9b10408", "jar-5e714692d6a297e2", "lid of jar-8b4f637577fdb1c6", "pen-8f0c8d589228c3f8", "spoon4-f59ca078f3e3f8ed", "knife-cdc841b0da7680a5", "piece of butter-bc492bce55f1bd59", "chopping board-697e69f256a8ff22", "plate-6db2d4b2a586c11c", "spoon2-f83b2262005ab946", "onion-f40de62097bd87c7", "box-b61b1fcf0d61d5b2", "lid of box-708a7c6309a9cd99", "onion peel-0dfb1de9d4f8f2f4", "jar2-a2d57ffb658ae029", "salt-fd683237f238913d", "jar3-d4eac3294fbab1b3", "jar4-c68f0f293110bebb", "new jar-92f2c76c216ac496", "jar5-cf4cf20340497289", "spoon5-8d666bfa72ab4f8f", "jar6-092d0b62ff1ead30", "pack of green chillies-3d5a171c796c9dc1", "jar7-94b289e651b03c77", "jar8-9bd828c8379e014e", "plate2-73578c6d4a37373a", "bowl2-8b088fde36f982a6", "bowl3-0b777b1f3e40786b", "tablecloth-2a7ba0a85822bd52", "jar9-2223b8f8d1af1687", "jar lid-49d739410e181c4b", "cup-f974528413bb1e60", "sponge-2bd4b188a0513bd2", "bottle of dishwashing liquid-cf3e674883d4e9f2", "some spoons-8ba1a55e9a48d5a1", "spoons2-1a03670c4b457e48", "spoons3-7b544c81fb76c481", "cloth2-03c83a8cb3ed9148", "saucepan-0505d8b4a3d58914", "lid2-33bbe47170fb7bee", "lid3-623e13525ad1e2da", "lid4-9dba6760f94ad48c", "frying pan-86c0b136080112a7", "pack of wipes-fbc584de7bbbefff", "wipe-638dd3157a6703ab", "another wipe-4d82c60a9045dd3e" ], "reasoning": "1. First, I locate the timestamp when the user taps the wooden spoon (spatula-8b46dfa81b5e4725) against the skillet (frying pan-86c0b136080112a7) 6 times and then puts the spoon (spatula-8b46dfa81b5e4725) down. This occurs at 03:03. \n2. Immediately following those 2 actions, the user clasps their hands together in front of them at 03:07. I notice the positioning of their hands, with the left hand on top of the right. \n3. Next, I pay attention to the positioning of the user's thumbs. I then conclude that the user's left thumb is on top and behind the right thumb.", "referenced_object_masks": [ "spatula-8b46dfa81b5e4725", "frying pan-86c0b136080112a7" ] }, { "key": "P06-20240510-121619:538ab85a6f9310397d0453bcb9cb32b73e03e9ca", "video_id": "P06-20240510-121619", "question": "After I close the door, what is the fifth object I touch with my right hand?", "answer_choice_0": "Pressure cooker", "answer_choice_1": "Spatula", "answer_choice_2": "Spoon", "answer_choice_3": "Burner knob", "answer_choice_4": "Bowl", "answer_id": 4, "answer": "Bowl", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counting" ], "objects_of_interest": [ "pressure cooker-c7291b6669d0b764", "spatula-8b46dfa81b5e4725", "spoon-54be405bb65e20c6", "bowl-e72cf25be22cf49a", "lid of pressure cooker-3952265c5c7cf8b0", "pack of butter-f6cd01da37f5a5f3", "spoon3-b57e38a4e9b10408", "jar-5e714692d6a297e2", "lid of jar-8b4f637577fdb1c6", "pen-8f0c8d589228c3f8", "spoon4-f59ca078f3e3f8ed", "knife-cdc841b0da7680a5", "piece of butter-bc492bce55f1bd59", "chopping board-697e69f256a8ff22", "plate-6db2d4b2a586c11c", "spoon2-f83b2262005ab946", "onion-f40de62097bd87c7", "box-b61b1fcf0d61d5b2", "lid of box-708a7c6309a9cd99", "onion peel-0dfb1de9d4f8f2f4", "jar2-a2d57ffb658ae029", "salt-fd683237f238913d", "jar3-d4eac3294fbab1b3", "jar4-c68f0f293110bebb", "new jar-92f2c76c216ac496", "jar5-cf4cf20340497289", "spoon5-8d666bfa72ab4f8f", "jar6-092d0b62ff1ead30", "pack of green chillies-3d5a171c796c9dc1", "jar7-94b289e651b03c77", "jar8-9bd828c8379e014e", "plate2-73578c6d4a37373a", "bowl2-8b088fde36f982a6", "bowl3-0b777b1f3e40786b", "tablecloth-2a7ba0a85822bd52", "jar9-2223b8f8d1af1687", "jar lid-49d739410e181c4b", "cup-f974528413bb1e60", "sponge-2bd4b188a0513bd2", "bottle of dishwashing liquid-cf3e674883d4e9f2", "some spoons-8ba1a55e9a48d5a1", "spoons2-1a03670c4b457e48", "spoons3-7b544c81fb76c481", "cloth2-03c83a8cb3ed9148", "saucepan-0505d8b4a3d58914", "lid2-33bbe47170fb7bee", "lid3-623e13525ad1e2da", "lid4-9dba6760f94ad48c", "frying pan-86c0b136080112a7", "pack of wipes-fbc584de7bbbefff", "wipe-638dd3157a6703ab", "another wipe-4d82c60a9045dd3e" ], "reasoning": "1. I locate the opened door at 00:05 in front of the user to the right.\n2. I observe the user close the door from 00:10 to 00:14.\n3. From here, I begin to note the objects the user touches with their right hand. I observe the user picking up the pressure cooker (pressure cooker-c7291b6669d0b764) with their right hand from the stove at 00:21. This is the first object.\n4. I observe the user placing down the pressure cooker (pressure cooker-c7291b6669d0b764) next to the sink at 00:24.\n5. At 00:26, I observe the user adjust the burner knob on the stove with their right hand. This is the second object that the user touches with their right hand.\n6. At 00:39, the user touches the spatula (spatula-8b46dfa81b5e4725) with their right hand. This is the third object.\n7. At 00:55, the user touches the spoon (spoon-54be405bb65e20c6) with their right hand. This is the fourth object.\n8. At 00:59, the user touches the bowl (bowl-e72cf25be22cf49a) with their right hand. This is the fifth object.", "referenced_object_masks": [ "bowl-e72cf25be22cf49a", "pressure cooker-c7291b6669d0b764", "spoon-54be405bb65e20c6", "spatula-8b46dfa81b5e4725" ] }, { "key": "P06-20240510-121619:615bea001c552131cee7a4dbddd31e637d0e3a25", "video_id": "P06-20240510-121619", "question": "What number would I get if I added the number of plates in the drying rack the first time I walk past the sink and the number of buttons on the vent above the stove?", "answer_choice_0": "6", "answer_choice_1": "4", "answer_choice_2": "2", "answer_choice_3": "5", "answer_choice_4": "3", "answer_id": 0, "answer": "6", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning", "Numerical Reasoning", "Counting" ], "objects_of_interest": [ "pressure cooker-c7291b6669d0b764", "spatula-8b46dfa81b5e4725", "spoon-54be405bb65e20c6", "bowl-e72cf25be22cf49a", "lid of pressure cooker-3952265c5c7cf8b0", "pack of butter-f6cd01da37f5a5f3", "spoon3-b57e38a4e9b10408", "jar-5e714692d6a297e2", "lid of jar-8b4f637577fdb1c6", "pen-8f0c8d589228c3f8", "spoon4-f59ca078f3e3f8ed", "knife-cdc841b0da7680a5", "piece of butter-bc492bce55f1bd59", "chopping board-697e69f256a8ff22", "plate-6db2d4b2a586c11c", "spoon2-f83b2262005ab946", "onion-f40de62097bd87c7", "box-b61b1fcf0d61d5b2", "lid of box-708a7c6309a9cd99", "onion peel-0dfb1de9d4f8f2f4", "jar2-a2d57ffb658ae029", "salt-fd683237f238913d", "jar3-d4eac3294fbab1b3", "jar4-c68f0f293110bebb", "new jar-92f2c76c216ac496", "jar5-cf4cf20340497289", "spoon5-8d666bfa72ab4f8f", "jar6-092d0b62ff1ead30", "pack of green chillies-3d5a171c796c9dc1", "jar7-94b289e651b03c77", "jar8-9bd828c8379e014e", "plate2-73578c6d4a37373a", "bowl2-8b088fde36f982a6", "bowl3-0b777b1f3e40786b", "tablecloth-2a7ba0a85822bd52", "jar9-2223b8f8d1af1687", "jar lid-49d739410e181c4b", "cup-f974528413bb1e60", "sponge-2bd4b188a0513bd2", "bottle of dishwashing liquid-cf3e674883d4e9f2", "some spoons-8ba1a55e9a48d5a1", "spoons2-1a03670c4b457e48", "spoons3-7b544c81fb76c481", "cloth2-03c83a8cb3ed9148", "saucepan-0505d8b4a3d58914", "lid2-33bbe47170fb7bee", "lid3-623e13525ad1e2da", "lid4-9dba6760f94ad48c", "frying pan-86c0b136080112a7", "pack of wipes-fbc584de7bbbefff", "wipe-638dd3157a6703ab", "another wipe-4d82c60a9045dd3e" ], "reasoning": "1. I locate the drying rack in front of the sink to the right of the user at 00:00.\n2. I observe the user walk past the sink towards the door frame on the other end of the kitchen from 00:01 to 00:09.\n3. At 00:02, I locate one plate (plate2-73578c6d4a37373a) in the drying rack to the left of the user.\n4. At 00:05, I locate the vent above the stove to the left of the user and count 5 buttons .\n5. I add 1 and 5 to get 6.", "referenced_object_masks": [ "plate2-73578c6d4a37373a" ] }, { "key": "P06-20240510-121619:76ee50dda5e1f3724ecb0e1a00f282d98ed2dfb2", "video_id": "P06-20240510-121619", "question": "After placing the pressure cooker lid down to the left of the sink, what is the second object I touch with my left hand?", "answer_choice_0": "Bowl", "answer_choice_1": "Cutting board", "answer_choice_2": "Pressure cooker", "answer_choice_3": "Door handle", "answer_choice_4": "Spatula", "answer_id": 4, "answer": "Spatula", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counting" ], "objects_of_interest": [ "pressure cooker-c7291b6669d0b764", "spatula-8b46dfa81b5e4725", "spoon-54be405bb65e20c6", "bowl-e72cf25be22cf49a", "lid of pressure cooker-3952265c5c7cf8b0", "pack of butter-f6cd01da37f5a5f3", "spoon3-b57e38a4e9b10408", "jar-5e714692d6a297e2", "lid of jar-8b4f637577fdb1c6", "pen-8f0c8d589228c3f8", "spoon4-f59ca078f3e3f8ed", "knife-cdc841b0da7680a5", "piece of butter-bc492bce55f1bd59", "chopping board-697e69f256a8ff22", "plate-6db2d4b2a586c11c", "spoon2-f83b2262005ab946", "onion-f40de62097bd87c7", "box-b61b1fcf0d61d5b2", "lid of box-708a7c6309a9cd99", "onion peel-0dfb1de9d4f8f2f4", "jar2-a2d57ffb658ae029", "salt-fd683237f238913d", "jar3-d4eac3294fbab1b3", "jar4-c68f0f293110bebb", "new jar-92f2c76c216ac496", "jar5-cf4cf20340497289", "spoon5-8d666bfa72ab4f8f", "jar6-092d0b62ff1ead30", "pack of green chillies-3d5a171c796c9dc1", "jar7-94b289e651b03c77", "jar8-9bd828c8379e014e", "plate2-73578c6d4a37373a", "bowl2-8b088fde36f982a6", "bowl3-0b777b1f3e40786b", "tablecloth-2a7ba0a85822bd52", "jar9-2223b8f8d1af1687", "jar lid-49d739410e181c4b", "cup-f974528413bb1e60", "sponge-2bd4b188a0513bd2", "bottle of dishwashing liquid-cf3e674883d4e9f2", "some spoons-8ba1a55e9a48d5a1", "spoons2-1a03670c4b457e48", "spoons3-7b544c81fb76c481", "cloth2-03c83a8cb3ed9148", "saucepan-0505d8b4a3d58914", "lid2-33bbe47170fb7bee", "lid3-623e13525ad1e2da", "lid4-9dba6760f94ad48c", "frying pan-86c0b136080112a7", "pack of wipes-fbc584de7bbbefff", "wipe-638dd3157a6703ab", "another wipe-4d82c60a9045dd3e" ], "reasoning": "1. I locate the pressure cooker lid (lid of pressure cooker-3952265c5c7cf8b0) at 00:06 on the stove to the left of the user.\n2. I observe the user remove the pressure cooker lid (lid of pressure cooker-3952265c5c7cf8b0) from the pressure cooker (pressure cooker-c7291b6669d0b764) and placing it to the left of the sink from 1:38 to 1:41.\n3. At 01:44, I observe the user touch the door handle with their left hand.\n4. At 01:56, I observe the user push the spatula (spatula-8b46dfa81b5e4725) very gently with their left hand into their right hand.\n5. I determine the second object the user touches with their left hand is the spatula (spatula-8b46dfa81b5e4725).", "referenced_object_masks": [ "spatula-8b46dfa81b5e4725", "pressure cooker-c7291b6669d0b764", "lid of pressure cooker-3952265c5c7cf8b0" ] }, { "key": "P06-20240510-121619:cc47215257cd1681aff7ba37cb78024682f3c8e3", "video_id": "P06-20240510-121619", "question": "After I first touch the butter, where do I place the spatula down the second time?", "answer_choice_0": "Plate", "answer_choice_1": "Sink", "answer_choice_2": "Stove", "answer_choice_3": "Cutting board", "answer_choice_4": "Frying Pan", "answer_id": 3, "answer": "Cutting board", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counting" ], "objects_of_interest": [ "pressure cooker-c7291b6669d0b764", "spatula-8b46dfa81b5e4725", "spoon-54be405bb65e20c6", "bowl-e72cf25be22cf49a", "lid of pressure cooker-3952265c5c7cf8b0", "pack of butter-f6cd01da37f5a5f3", "spoon3-b57e38a4e9b10408", "jar-5e714692d6a297e2", "lid of jar-8b4f637577fdb1c6", "pen-8f0c8d589228c3f8", "spoon4-f59ca078f3e3f8ed", "knife-cdc841b0da7680a5", "piece of butter-bc492bce55f1bd59", "chopping board-697e69f256a8ff22", "plate-6db2d4b2a586c11c", "spoon2-f83b2262005ab946", "onion-f40de62097bd87c7", "box-b61b1fcf0d61d5b2", "lid of box-708a7c6309a9cd99", "onion peel-0dfb1de9d4f8f2f4", "jar2-a2d57ffb658ae029", "salt-fd683237f238913d", "jar3-d4eac3294fbab1b3", "jar4-c68f0f293110bebb", "new jar-92f2c76c216ac496", "jar5-cf4cf20340497289", "spoon5-8d666bfa72ab4f8f", "jar6-092d0b62ff1ead30", "pack of green chillies-3d5a171c796c9dc1", "jar7-94b289e651b03c77", "jar8-9bd828c8379e014e", "plate2-73578c6d4a37373a", "bowl2-8b088fde36f982a6", "bowl3-0b777b1f3e40786b", "tablecloth-2a7ba0a85822bd52", "jar9-2223b8f8d1af1687", "jar lid-49d739410e181c4b", "cup-f974528413bb1e60", "sponge-2bd4b188a0513bd2", "bottle of dishwashing liquid-cf3e674883d4e9f2", "some spoons-8ba1a55e9a48d5a1", "spoons2-1a03670c4b457e48", "spoons3-7b544c81fb76c481", "cloth2-03c83a8cb3ed9148", "saucepan-0505d8b4a3d58914", "lid2-33bbe47170fb7bee", "lid3-623e13525ad1e2da", "lid4-9dba6760f94ad48c", "frying pan-86c0b136080112a7", "pack of wipes-fbc584de7bbbefff", "wipe-638dd3157a6703ab", "another wipe-4d82c60a9045dd3e" ], "reasoning": "1. I locate the spatula (spatula-8b46dfa81b5e4725) at 00:06 on the frying pan (frying pan-86c0b136080112a7) on top of the stove .\n2. I locate the butter (pack of butter-f6cd01da37f5a5f3) at 00:18 to the left of the stove .\n3. I observe the user pick up the butter (pack of butter-f6cd01da37f5a5f3) and place it to the left of the plate (plate-6db2d4b2a586c11c) from 02:03 to 02:04.\n4. I observe the user place the spatula (spatula-8b46dfa81b5e4725) down on the frying pan (frying pan-86c0b136080112a7) at 02:14.\n5. I observe the user place the spatula (spatula-8b46dfa81b5e4725) down on the cutting board (chopping board-697e69f256a8ff22) at 03:06.\n6. I determine the user places the spatula (spatula-8b46dfa81b5e4725) down the second time on the cutting board (chopping board-697e69f256a8ff22).", "referenced_object_masks": [ "pack of butter-f6cd01da37f5a5f3", "plate-6db2d4b2a586c11c", "chopping board-697e69f256a8ff22", "frying pan-86c0b136080112a7", "spatula-8b46dfa81b5e4725" ] }, { "key": "P06-20240510-121619:e402f38bbff4d4fc26c7a7cdf63b78cb5b40b9c9", "video_id": "P06-20240510-121619", "question": "When I'm facing away from the clock towards the door, where is the washing machine in relation to me the next to the last time I face the door before wiping out the white bowl?", "answer_choice_0": "In front of me", "answer_choice_1": "Behind me", "answer_choice_2": "On my left", "answer_choice_3": "Behind the door", "answer_choice_4": "On my right", "answer_id": 2, "answer": "On my left", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Counting" ], "objects_of_interest": [ "pressure cooker-c7291b6669d0b764", "spatula-8b46dfa81b5e4725", "spoon-54be405bb65e20c6", "bowl-e72cf25be22cf49a", "lid of pressure cooker-3952265c5c7cf8b0", "pack of butter-f6cd01da37f5a5f3", "spoon3-b57e38a4e9b10408", "jar-5e714692d6a297e2", "lid of jar-8b4f637577fdb1c6", "pen-8f0c8d589228c3f8", "spoon4-f59ca078f3e3f8ed", "knife-cdc841b0da7680a5", "piece of butter-bc492bce55f1bd59", "chopping board-697e69f256a8ff22", "plate-6db2d4b2a586c11c", "spoon2-f83b2262005ab946", "onion-f40de62097bd87c7", "box-b61b1fcf0d61d5b2", "lid of box-708a7c6309a9cd99", "onion peel-0dfb1de9d4f8f2f4", "jar2-a2d57ffb658ae029", "salt-fd683237f238913d", "jar3-d4eac3294fbab1b3", "jar4-c68f0f293110bebb", "new jar-92f2c76c216ac496", "jar5-cf4cf20340497289", "spoon5-8d666bfa72ab4f8f", "jar6-092d0b62ff1ead30", "pack of green chillies-3d5a171c796c9dc1", "jar7-94b289e651b03c77", "jar8-9bd828c8379e014e", "plate2-73578c6d4a37373a", "bowl2-8b088fde36f982a6", "bowl3-0b777b1f3e40786b", "tablecloth-2a7ba0a85822bd52", "jar9-2223b8f8d1af1687", "jar lid-49d739410e181c4b", "cup-f974528413bb1e60", "sponge-2bd4b188a0513bd2", "bottle of dishwashing liquid-cf3e674883d4e9f2", "some spoons-8ba1a55e9a48d5a1", "spoons2-1a03670c4b457e48", "spoons3-7b544c81fb76c481", "cloth2-03c83a8cb3ed9148", "saucepan-0505d8b4a3d58914", "lid2-33bbe47170fb7bee", "lid3-623e13525ad1e2da", "lid4-9dba6760f94ad48c", "frying pan-86c0b136080112a7", "pack of wipes-fbc584de7bbbefff", "wipe-638dd3157a6703ab", "another wipe-4d82c60a9045dd3e" ], "reasoning": "1. First, I locate a timestamp where the clock is in clear view. I noticed the clock for the first time at 00:08, on the kitchen wall above the doorframe. \n2. Next, I determine when the user wipes out the white bowl . This occurs at 09:26. This timestamp is my cutoff. \n3. Next, I locate all the instances, before 09:26, when the user faces away from the clock towards the white door. The first time occurs at 00:17, followed by 06:54, 07:58, 08:15, 08:29, 08:56, 09:09, and 09:14. 09:09 is the next to last time the user faces the white door before wiping out the white bowl . \n4. I searched for the washing machine . In every instance, including 09:09, the appliance is situated on the user's left.", "referenced_object_masks": [] }, { "key": "P06-20240510-121619:ebc7c369f7305ead4a144f1e7f9dc1a328d37f47", "video_id": "P06-20240510-121619", "question": "What number would I get if I added the number of spaces in the drawer's utensil tray with the number of plant pots on the window sill above the sink?", "answer_choice_0": "6", "answer_choice_1": "8", "answer_choice_2": "10", "answer_choice_3": "9", "answer_choice_4": "7", "answer_id": 1, "answer": "8", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning", "Numerical Reasoning", "Counting" ], "objects_of_interest": [ "pressure cooker-c7291b6669d0b764", "spatula-8b46dfa81b5e4725", "spoon-54be405bb65e20c6", "bowl-e72cf25be22cf49a", "lid of pressure cooker-3952265c5c7cf8b0", "pack of butter-f6cd01da37f5a5f3", "spoon3-b57e38a4e9b10408", "jar-5e714692d6a297e2", "lid of jar-8b4f637577fdb1c6", "pen-8f0c8d589228c3f8", "spoon4-f59ca078f3e3f8ed", "knife-cdc841b0da7680a5", "piece of butter-bc492bce55f1bd59", "chopping board-697e69f256a8ff22", "plate-6db2d4b2a586c11c", "spoon2-f83b2262005ab946", "onion-f40de62097bd87c7", "box-b61b1fcf0d61d5b2", "lid of box-708a7c6309a9cd99", "onion peel-0dfb1de9d4f8f2f4", "jar2-a2d57ffb658ae029", "salt-fd683237f238913d", "jar3-d4eac3294fbab1b3", "jar4-c68f0f293110bebb", "new jar-92f2c76c216ac496", "jar5-cf4cf20340497289", "spoon5-8d666bfa72ab4f8f", "jar6-092d0b62ff1ead30", "pack of green chillies-3d5a171c796c9dc1", "jar7-94b289e651b03c77", "jar8-9bd828c8379e014e", "plate2-73578c6d4a37373a", "bowl2-8b088fde36f982a6", "bowl3-0b777b1f3e40786b", "tablecloth-2a7ba0a85822bd52", "jar9-2223b8f8d1af1687", "jar lid-49d739410e181c4b", "cup-f974528413bb1e60", "sponge-2bd4b188a0513bd2", "bottle of dishwashing liquid-cf3e674883d4e9f2", "some spoons-8ba1a55e9a48d5a1", "spoons2-1a03670c4b457e48", "spoons3-7b544c81fb76c481", "cloth2-03c83a8cb3ed9148", "saucepan-0505d8b4a3d58914", "lid2-33bbe47170fb7bee", "lid3-623e13525ad1e2da", "lid4-9dba6760f94ad48c", "frying pan-86c0b136080112a7", "pack of wipes-fbc584de7bbbefff", "wipe-638dd3157a6703ab", "another wipe-4d82c60a9045dd3e" ], "reasoning": "1. I locate the drawer under the counter to the left of the user at 00:00.\n2. I locate the window sill above the sink at 00:03 to the left of the user and count 3 plant pots .\n3. From 03:15 to 03:16, I observe the user open the drawer to reveal the utensil tray .\n4. I count 5 spaces in the utensil tray .\n5. I add 3 and 5 to get 8.", "referenced_object_masks": [] }, { "key": "P06-20240510-135423:3777fb0a88deb8bb659a667942a24afba56f872a", "video_id": "P06-20240510-135423", "question": "When I picked up the 3rd item that I dried with a towel after drying the strainer, approximately how many degrees counterclockwise would I have to turn to face the shelf with ramen?", "answer_choice_0": "180", "answer_choice_1": "90", "answer_choice_2": "45", "answer_choice_3": "135", "answer_choice_4": "215", "answer_id": 3, "answer": "135", "question_type": [ "Egocentric & Spatial Perception", "Action Recognition", "Counting", "Counterfactual", "Numerical Reasoning", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "saucepan-6847059654db27a8", "strainer-f525741b3726bcd2", "chopping board-a4019ce959b3a4c3", "frying pan-7c9d021408ae686f", "plate-dc678fd29f05eba8", "plate2-a8656821d3b409a5", "pressure cooker-ea2e30e57392d665", "pressure cooker lid-002e52e267828925", "lid-2df9add5ac124de5", "tablecloth-2eb8a1c2f145386b", "cup-ae5eb8409995928a", "glass-ce11abd1aff30b90", "bowl-c8bd52865f0a1738", "knife-e7385bb35bbef160", "spatula-7238bc75891e1262", "packet-9304fa199fb30872", "chopping board2-70e2c6fb199f6824", "cardboard box-9adabef281e611fb", "fallen packet of noodles round shape-f8e376ac30ca975a", "packet2-c804fcd34ee99ccc", "packet3-a05e25be04cde69f", "few packets-9c1e3226876d51b7", "notebook-5c27032b444821be", "pen-b873490f9815c984", "weighing balance-663fb4b0960d8b2a", "glass2-c92998ae7728e427" ], "reasoning": "1. I watched the video and saw that the user dried the strainer (strainer-f525741b3726bcd2) with the towel (tablecloth-2eb8a1c2f145386b) from 01:06-01:10. \n2. I identified the first 3 items dried with a towel (tablecloth-2eb8a1c2f145386b) after the strainer (strainer-f525741b3726bcd2) as the saucepan (saucepan-6847059654db27a8) at 01:12, the blue mug (cup-ae5eb8409995928a) at 01:39, and the bowl (bowl-c8bd52865f0a1738) at 01:58. \n3. I looked for the first time the user picked up the bowl (bowl-c8bd52865f0a1738) and saw that this occurred at 01:54. \n4. I observed the surroundings at 01:54 and saw that the user was standing in front of the sink and dish drying rack . \n5. I looked for the first time the user interacted with a packet of ramen (fallen packet of noodles round shape-f8e376ac30ca975a). At 03:01, a packet of ramen (fallen packet of noodles round shape-f8e376ac30ca975a) fell on the user from a box on the shelf . The user picked up the packet of ramen (fallen packet of noodles round shape-f8e376ac30ca975a) at 03:07.\n6. I studied the surroundings between when the packet of ramen (fallen packet of noodles round shape-f8e376ac30ca975a) fell and when it (fallen packet of noodles round shape-f8e376ac30ca975a) was picked up, from 03:01-03:07. I saw that when the packet of ramen (fallen packet of noodles round shape-f8e376ac30ca975a) fell, the user was looking up toward a cabinet with 2 shelves. I observed that there was a wall perpendicularly abutting the right edge of the cabinet. From 03:02-03:06 I saw the user turn about 90 degrees counterclockwise to face the packet of ramen (fallen packet of noodles round shape-f8e376ac30ca975a) on the floor. I observed that the user was in between parallel counters to the left and right. I saw the edge of the dish drying rack a few steps ahead and to the left. \n7. When the user was standing in front of the dish drying rack with the bowl (bowl-c8bd52865f0a1738), if they had wanted to face the shelf with the packet of ramen (fallen packet of noodles round shape-f8e376ac30ca975a) they would first turn 90 degrees counterclockwise to face the wall perpendicular to the cabinet. They would have to turn further counterclockwise, about 45 degrees, to face the cabinet itself. Therefore, I added 90+45=135 degrees.", "referenced_object_masks": [ "strainer-f525741b3726bcd2", "bowl-c8bd52865f0a1738", "tablecloth-2eb8a1c2f145386b", "fallen packet of noodles round shape-f8e376ac30ca975a", "saucepan-6847059654db27a8", "cup-ae5eb8409995928a" ] }, { "key": "P06-20240510-135423:3bed6a8825164b68d55fa581d6d37c966aba1e4e", "video_id": "P06-20240510-135423", "question": "Rounding to the nearest 5 seconds, how much time passed between taking out the first glass and completing the task of filling this glass with powder?", "answer_choice_0": "1 minute and 50 seconds", "answer_choice_1": "1 minute and 55 seconds", "answer_choice_2": "1 minute and 45 seconds", "answer_choice_3": "2 minutes and 5 seconds", "answer_choice_4": "2 minutes", "answer_id": 4, "answer": "2 minutes", "question_type": [ "Object Recognition", "Action Recognition", "Counting", "Numerical Reasoning", "Temporal Reasoning" ], "objects_of_interest": [ "saucepan-6847059654db27a8", "strainer-f525741b3726bcd2", "chopping board-a4019ce959b3a4c3", "frying pan-7c9d021408ae686f", "plate-dc678fd29f05eba8", "plate2-a8656821d3b409a5", "pressure cooker-ea2e30e57392d665", "pressure cooker lid-002e52e267828925", "lid-2df9add5ac124de5", "tablecloth-2eb8a1c2f145386b", "cup-ae5eb8409995928a", "glass-ce11abd1aff30b90", "bowl-c8bd52865f0a1738", "knife-e7385bb35bbef160", "spatula-7238bc75891e1262", "packet-9304fa199fb30872", "chopping board2-70e2c6fb199f6824", "cardboard box-9adabef281e611fb", "fallen packet of noodles round shape-f8e376ac30ca975a", "packet2-c804fcd34ee99ccc", "packet3-a05e25be04cde69f", "few packets-9c1e3226876d51b7", "notebook-5c27032b444821be", "pen-b873490f9815c984", "weighing balance-663fb4b0960d8b2a", "glass2-c92998ae7728e427" ], "reasoning": "1. I watched the video and saw the user take out a glass (glass-ce11abd1aff30b90) for the first time at 01:47. \n2. I saw that the user filled this glass (glass-ce11abd1aff30b90) with powder from the packet of powder (packet-9304fa199fb30872) from 03:41-03:46. \n3. I subtracted the time the glass (glass-ce11abd1aff30b90) was taken out from the end of the segment in which it (glass-ce11abd1aff30b90) was filled: 03:46-01:47=01:59. I rounded to the nearest 5 seconds: 02:00.", "referenced_object_masks": [ "glass-ce11abd1aff30b90", "packet-9304fa199fb30872" ] }, { "key": "P06-20240510-135423:4ca318da404eb02b2d36a8f0bcd734c1fc17ade4", "video_id": "P06-20240510-135423", "question": "What is the difference between the number of items I removed from the dish drying rack in the first minute and the amount of time (in seconds) I left the living room door open?", "answer_choice_0": "0", "answer_choice_1": "1", "answer_choice_2": "2", "answer_choice_3": "3", "answer_choice_4": "4", "answer_id": 3, "answer": "3", "question_type": [ "Action Recognition", "Object Recognition", "Counting", "Egocentric & Spatial Perception", "Numerical Reasoning", "Situational Awareness", "Temporal Reasoning" ], "objects_of_interest": [ "saucepan-6847059654db27a8", "strainer-f525741b3726bcd2", "chopping board-a4019ce959b3a4c3", "frying pan-7c9d021408ae686f", "plate-dc678fd29f05eba8", "plate2-a8656821d3b409a5", "pressure cooker-ea2e30e57392d665", "pressure cooker lid-002e52e267828925", "lid-2df9add5ac124de5", "tablecloth-2eb8a1c2f145386b", "cup-ae5eb8409995928a", "glass-ce11abd1aff30b90", "bowl-c8bd52865f0a1738", "knife-e7385bb35bbef160", "spatula-7238bc75891e1262", "packet-9304fa199fb30872", "chopping board2-70e2c6fb199f6824", "cardboard box-9adabef281e611fb", "fallen packet of noodles round shape-f8e376ac30ca975a", "packet2-c804fcd34ee99ccc", "packet3-a05e25be04cde69f", "few packets-9c1e3226876d51b7", "notebook-5c27032b444821be", "pen-b873490f9815c984", "weighing balance-663fb4b0960d8b2a", "glass2-c92998ae7728e427" ], "reasoning": "1. I watched the video and counted each item removed from the dish drying rack in the first minute. At 00:09 the user removed a saucepan (saucepan-6847059654db27a8). At 00:13 the user removed a strainer (strainer-f525741b3726bcd2). At 00:20 the user removed a chopping board (chopping board-a4019ce959b3a4c3). At 00:30 the user removed a frying pan (frying pan-7c9d021408ae686f). At 00:34 the user removed 2 plates (plate-dc678fd29f05eba8, plate2-a8656821d3b409a5). At 00:40, the user removed a pressure cooker (pressure cooker-ea2e30e57392d665) and the pressure cooker lid (pressure cooker lid-002e52e267828925). At 00:47, the user removed a lid (lid-2df9add5ac124de5). This makes a total of 9 items removed in the first minute.\n2. I observed the user open a door that led to another room at 04:15. I saw the user walk through the doorway and enter the other room at 04:17. At this time, I saw that the room was decorated with furniture such as armchairs , a coffee table , couch , and television and identified it as the living room.\n3. I saw the user walk back through the door frame into the kitchen at 04:25 and heard them close the door at 04:27. \n3. I found the duration of time the living room door was open: 04:27-04:15=:12. \n4. I found the difference between the number of items (saucepan-6847059654db27a8, strainer-f525741b3726bcd2, chopping board-a4019ce959b3a4c3, frying pan-7c9d021408ae686f, plate-dc678fd29f05eba8, plate2-a8656821d3b409a5, pressure cooker-ea2e30e57392d665, pressure cooker lid-002e52e267828925, lid-2df9add5ac124de5) removed from the dish drying rack and the seconds spent outside the kitchen: 12-9=3.", "referenced_object_masks": [ "strainer-f525741b3726bcd2", "chopping board-a4019ce959b3a4c3", "plate-dc678fd29f05eba8", "plate2-a8656821d3b409a5", "pressure cooker lid-002e52e267828925", "frying pan-7c9d021408ae686f", "saucepan-6847059654db27a8", "pressure cooker-ea2e30e57392d665", "lid-2df9add5ac124de5" ] }, { "key": "P06-20240510-135423:8464a4f0cad3707681be0274752ff3c62841be18", "video_id": "P06-20240510-135423", "question": "When I picked up the pack of ramen, approximately how many degrees clockwise would I turn if I wanted to face the area where I first grabbed the towel?", "answer_choice_0": "45", "answer_choice_1": "180", "answer_choice_2": "215", "answer_choice_3": "90", "answer_choice_4": "135", "answer_id": 1, "answer": "180", "question_type": [ "Egocentric & Spatial Perception", "Action Recognition", "Numerical Reasoning", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "saucepan-6847059654db27a8", "strainer-f525741b3726bcd2", "chopping board-a4019ce959b3a4c3", "frying pan-7c9d021408ae686f", "plate-dc678fd29f05eba8", "plate2-a8656821d3b409a5", "pressure cooker-ea2e30e57392d665", "pressure cooker lid-002e52e267828925", "lid-2df9add5ac124de5", "tablecloth-2eb8a1c2f145386b", "cup-ae5eb8409995928a", "glass-ce11abd1aff30b90", "bowl-c8bd52865f0a1738", "knife-e7385bb35bbef160", "spatula-7238bc75891e1262", "packet-9304fa199fb30872", "chopping board2-70e2c6fb199f6824", "cardboard box-9adabef281e611fb", "fallen packet of noodles round shape-f8e376ac30ca975a", "packet2-c804fcd34ee99ccc", "packet3-a05e25be04cde69f", "few packets-9c1e3226876d51b7", "notebook-5c27032b444821be", "pen-b873490f9815c984", "weighing balance-663fb4b0960d8b2a", "glass2-c92998ae7728e427" ], "reasoning": "1. I watched the video and saw that the user picked up a packet of ramen (fallen packet of noodles round shape-f8e376ac30ca975a) off the floor at 03:07-03:08. \n2. I observed the surroundings from 03:07-03:08 and saw that there was a counter to the right of the user, and above the counter, a row of cabinets. I also observed a hutch with dishes on it ahead and slightly left of the user. \n3. I looked for the first time that the user grabbed the towel (tablecloth-2eb8a1c2f145386b) and saw that this occurred at 00:52. \n4. I observed the surroundings at 00:52 and saw that there was a counter between the user and a wall with windows. I saw that the sink was in front and just to the right of the user, and a refrigerator was in front and just to the left. \n5. I looked for the next time the row of cabinets was visible after 00:52 to get a better sense of the space. From 00:52-01:13, the user stepped toward the right while facing the wall with windows, then at 01:13, turned about 90 degrees left to face the far end of the kitchen with the refrigerator . I noticed the wall of cabinets to the left of the user now, and determined that they were parallel to the wall with windows.\n6. To determine how far the user would turn at 03:08, I first approximated that to face the cabinets parallel to the windows, the user would turn 45 degrees. Then, turning another 90 degrees would have the user face the edge of the kitchen between the two parallel counters. The user would need to turn slightly clockwise, about 45 degrees, to face the drawer where they pulled out the towel (tablecloth-2eb8a1c2f145386b). I added these amounts to find a total: 45+90+45=180.", "referenced_object_masks": [ "fallen packet of noodles round shape-f8e376ac30ca975a", "tablecloth-2eb8a1c2f145386b" ] }, { "key": "P06-20240510-135423:f26ee2150be569a20bcec085e0cd6b4709e01ae1", "video_id": "P06-20240510-135423", "question": "If I added the number of plates I stacked to the number of times I filled a glass, and divided that sum by the number of items dried with a towel, what would the simplified fraction be?", "answer_choice_0": "3/4", "answer_choice_1": "1/4", "answer_choice_2": "1/2", "answer_choice_3": "1/3", "answer_choice_4": "2/3", "answer_id": 4, "answer": "2/3", "question_type": [ "Object Recognition", "Action Recognition", "Counting", "Numerical Reasoning" ], "objects_of_interest": [ "saucepan-6847059654db27a8", "strainer-f525741b3726bcd2", "chopping board-a4019ce959b3a4c3", "frying pan-7c9d021408ae686f", "plate-dc678fd29f05eba8", "plate2-a8656821d3b409a5", "pressure cooker-ea2e30e57392d665", "pressure cooker lid-002e52e267828925", "lid-2df9add5ac124de5", "tablecloth-2eb8a1c2f145386b", "cup-ae5eb8409995928a", "glass-ce11abd1aff30b90", "bowl-c8bd52865f0a1738", "knife-e7385bb35bbef160", "spatula-7238bc75891e1262", "packet-9304fa199fb30872", "chopping board2-70e2c6fb199f6824", "cardboard box-9adabef281e611fb", "fallen packet of noodles round shape-f8e376ac30ca975a", "packet2-c804fcd34ee99ccc", "packet3-a05e25be04cde69f", "few packets-9c1e3226876d51b7", "notebook-5c27032b444821be", "pen-b873490f9815c984", "weighing balance-663fb4b0960d8b2a", "glass2-c92998ae7728e427" ], "reasoning": "1. I watched the video and counted each time the user stacked a plate. I saw the user take a plate (plate2-a8656821d3b409a5) out of the dish drying rack at 00:34, but this was placed on the counter, not stacked, so I didn't count it. At 00:37, I saw the user stack another plate (plate-dc678fd29f05eba8) on top of others, and counted this instance. There were no other plates that the user stacked. This makes a total of 1. \n2. I counted each time the user filled a glass. From 03:41-03:46 I saw the user fill a glass (glass-ce11abd1aff30b90) from a packet of powder (packet-9304fa199fb30872). From 06:17-06:20 I saw the user fill a glass (glass-ce11abd1aff30b90) with water . From 06:49-06:50 I saw the user fill a glass (glass-ce11abd1aff30b90) with water again. There were no other instances of the user filling up a glass. This makes a total of 3.\n3. I counted each time the user dried an item with a towel (tablecloth-2eb8a1c2f145386b). At 00:56 the user began to dry the lid (lid-2df9add5ac124de5). At 01:00 the user began to dry the frying pan (frying pan-7c9d021408ae686f). At 01:06 the user began to dry the strainer (strainer-f525741b3726bcd2). At 01:12 the user began to dry the saucepan (saucepan-6847059654db27a8). At 01:39 the user began to dry a blue mug (cup-ae5eb8409995928a). At 01:58 the user began to dry a bowl (bowl-c8bd52865f0a1738). This makes a total of 6, as there were no other times when the user dried an item with a towel (tablecloth-2eb8a1c2f145386b).\n4. I added the number of plates stacked (plate-dc678fd29f05eba8) to the number of times a glass (glass-ce11abd1aff30b90, glass2-c92998ae7728e427) was filled: 1+3=4.\n5. I divided the previous sum by the number of items dried (lid-2df9add5ac124de5, frying pan-7c9d021408ae686f, strainer-f525741b3726bcd2, saucepan-6847059654db27a8, cup-ae5eb8409995928a, bowl-c8bd52865f0a1738): 4/6 and represented it as a simplified fraction: 2/3.", "referenced_object_masks": [ "strainer-f525741b3726bcd2", "bowl-c8bd52865f0a1738", "tablecloth-2eb8a1c2f145386b", "frying pan-7c9d021408ae686f", "plate-dc678fd29f05eba8", "lid-2df9add5ac124de5", "saucepan-6847059654db27a8", "cup-ae5eb8409995928a", "glass-ce11abd1aff30b90", "glass2-c92998ae7728e427", "plate2-a8656821d3b409a5", "packet-9304fa199fb30872" ] }, { "key": "P06-20240510-140459:1ed2f0bebf5ce7942bff2187958ff0e5a830a981", "video_id": "P06-20240510-140459", "question": "What number would I get if I multiplied the largest digit from the third measurement and the number of potted plants on the window sill above the sink?", "answer_choice_0": "24", "answer_choice_1": "3", "answer_choice_2": "12", "answer_choice_3": "15", "answer_choice_4": "18", "answer_id": 4, "answer": "18", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning", "Numerical Reasoning", "Counting" ], "objects_of_interest": [ "outer plastic packaging of packet-655ef824f77ae4d6", "saucepan-8eebf5d83b84ef0b", "glass-e08f5e0176246774", "second glass-1cee33a9db671045", "pen-3ba6055a1b650d1c", "bottle of vegetable oil-e4628dd7f2603a09", "cap of vegetable oil bottle-c78dd56eeeb4b271", "spoon first-eca5e5111ccd402f", "chopping board-e341e1f5cd647520", "tablecloth-8bba3349418a6941", "notebook-7c11ad8d39252294", "translucent plate-3d101df93945a08c", "orange packet-85a33c3522b0b3dc", "saucepan with curry-2efc9fd3bf9cf271", "pressure cooker-8f57c2e0c741b6a7", "rolling pin-c83c2c5391e13ec0", "second translucent plate-c8be4725fbbf6120", "yellow packet-ce733daf63efa7bc", "pack of butter-8eb8574ad3113fe1", "blue box-83cad2b4827645a7", "bowl of mashed potatoes-002d269d89eedde4", "spoon with black plastic-e6e8ceecf358d42f", "bowl white first-c2e0407519ee7422", "tablecloth second-45f2865c6be39ac8", "butter knife-53bbd99953a05cb6", "tong-e7370639c35b1b04", "plate from fridge-ecd0a2153426fac1", "yellow lid-7ac45c583477930d", "plastic box-70391c347c91a747", "white lid-e431f95e1b126d73", "cup-4afd3f8b8ee42a28", "box of juice-5c2bef829fcbd77a", "cleaning sponge-be589be085c055bd", "cleaning up liquid-5df5aed82d89db57", "the other rarely used sponge-b110567c14bcfec3", "some spoons-83d2c3b240890f42", "bottle-b407866279dbd617", "problematic track both sponges moved-3366bfb5ca0837c6", "lid of bottle-81b76427030c72d4", "second bottle-5a7bc705331948a8", "second bottle lid-86444162e8387f03", "green glass bottle-898faacaa1799e3d", "red steel bottle-ed8e06acdd9af392", "second green glass bottle-7f114b79c49c2087", "tablecloth third-7a144af344191a77", "frying pan-9e23ac56c59b36cf", "transparent bottle-e88dc4f9936f5268", "transparent bottle cap-5c39da8bad5aa3b2", "container with yellow lid-64cab09e0480f540", "weighing balance-8fd437b5090abd8c", "phone-96d82abe02abc89f", "bottle green-11b949afb6b981e7" ], "reasoning": "1. I locate the digital scale (weighing balance-8fd437b5090abd8c) to the left of the user on the counter at 00:02.\n2. I locate the window sill above the sink at 01:46 to the left of the user and count 3 plant pots .\n3. I observe the digital scale (weighing balance-8fd437b5090abd8c) showing measurements for one object at 00:29 and another object at 00:45.\n4. I observe the digital scale (weighing balance-8fd437b5090abd8c) showing 0, 3, and 6 for the glass (second glass-1cee33a9db671045) filled with the powdery substance from the bag (outer plastic packaging of packet-655ef824f77ae4d6) at 01:09.\n5. I determine the largest digit from the digital scale (weighing balance-8fd437b5090abd8c)'s third measurement is 6.\n6. I multiply 6 by 3 to get 18.", "referenced_object_masks": [ "weighing balance-8fd437b5090abd8c", "outer plastic packaging of packet-655ef824f77ae4d6", "second glass-1cee33a9db671045" ] }, { "key": "P06-20240510-140459:32b72067d761a0626141a99cd98e9f7983c7dd9b", "video_id": "P06-20240510-140459", "question": "How many times do I flip over the rolled-out dough after going to get more flour, if I'd flipped it one less time?", "answer_choice_0": "1", "answer_choice_1": "3", "answer_choice_2": "5", "answer_choice_3": "6", "answer_choice_4": "2", "answer_id": 2, "answer": "5", "question_type": [ "Action Recognition", "Numerical Reasoning", "Counting" ], "objects_of_interest": [ "outer plastic packaging of packet-655ef824f77ae4d6", "saucepan-8eebf5d83b84ef0b", "glass-e08f5e0176246774", "second glass-1cee33a9db671045", "pen-3ba6055a1b650d1c", "bottle of vegetable oil-e4628dd7f2603a09", "cap of vegetable oil bottle-c78dd56eeeb4b271", "spoon first-eca5e5111ccd402f", "chopping board-e341e1f5cd647520", "tablecloth-8bba3349418a6941", "notebook-7c11ad8d39252294", "translucent plate-3d101df93945a08c", "orange packet-85a33c3522b0b3dc", "saucepan with curry-2efc9fd3bf9cf271", "pressure cooker-8f57c2e0c741b6a7", "rolling pin-c83c2c5391e13ec0", "second translucent plate-c8be4725fbbf6120", "yellow packet-ce733daf63efa7bc", "pack of butter-8eb8574ad3113fe1", "blue box-83cad2b4827645a7", "bowl of mashed potatoes-002d269d89eedde4", "spoon with black plastic-e6e8ceecf358d42f", "bowl white first-c2e0407519ee7422", "tablecloth second-45f2865c6be39ac8", "butter knife-53bbd99953a05cb6", "tong-e7370639c35b1b04", "plate from fridge-ecd0a2153426fac1", "yellow lid-7ac45c583477930d", "plastic box-70391c347c91a747", "white lid-e431f95e1b126d73", "cup-4afd3f8b8ee42a28", "box of juice-5c2bef829fcbd77a", "cleaning sponge-be589be085c055bd", "cleaning up liquid-5df5aed82d89db57", "the other rarely used sponge-b110567c14bcfec3", "some spoons-83d2c3b240890f42", "bottle-b407866279dbd617", "problematic track both sponges moved-3366bfb5ca0837c6", "lid of bottle-81b76427030c72d4", "second bottle-5a7bc705331948a8", "second bottle lid-86444162e8387f03", "green glass bottle-898faacaa1799e3d", "red steel bottle-ed8e06acdd9af392", "second green glass bottle-7f114b79c49c2087", "tablecloth third-7a144af344191a77", "frying pan-9e23ac56c59b36cf", "transparent bottle-e88dc4f9936f5268", "transparent bottle cap-5c39da8bad5aa3b2", "container with yellow lid-64cab09e0480f540", "weighing balance-8fd437b5090abd8c", "phone-96d82abe02abc89f", "bottle green-11b949afb6b981e7" ], "reasoning": "1. I watch for the user to roll out dough with the rolling pin (rolling pin-c83c2c5391e13ec0) for the first time, which begins at 07:50.\n2. I then watch for the user to get out more flour , which they retrieve from the pantry at 08:18.\n3. I watch for the user to flip the rolled-out dough, which occurs at 08:25, 08:35, 08:37, 08:41, 09:00, and 09:15.\n4. The user flips the dough six times, and one less would be five.", "referenced_object_masks": [ "rolling pin-c83c2c5391e13ec0" ] }, { "key": "P06-20240510-140459:5db0f20f1e7b31f150f939a8bb1fb6d6f34c462e", "video_id": "P06-20240510-140459", "question": "After I take the vegetable oil from the cabinet, what is the second object I touch with my right hand after placing the vegetable oil bottle cap down?", "answer_choice_0": "Cutting board", "answer_choice_1": "Sink", "answer_choice_2": "Stove", "answer_choice_3": "Plate", "answer_choice_4": "Spoon", "answer_id": 4, "answer": "Spoon", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counting" ], "objects_of_interest": [ "outer plastic packaging of packet-655ef824f77ae4d6", "saucepan-8eebf5d83b84ef0b", "glass-e08f5e0176246774", "second glass-1cee33a9db671045", "pen-3ba6055a1b650d1c", "bottle of vegetable oil-e4628dd7f2603a09", "cap of vegetable oil bottle-c78dd56eeeb4b271", "spoon first-eca5e5111ccd402f", "chopping board-e341e1f5cd647520", "tablecloth-8bba3349418a6941", "notebook-7c11ad8d39252294", "translucent plate-3d101df93945a08c", "orange packet-85a33c3522b0b3dc", "saucepan with curry-2efc9fd3bf9cf271", "pressure cooker-8f57c2e0c741b6a7", "rolling pin-c83c2c5391e13ec0", "second translucent plate-c8be4725fbbf6120", "yellow packet-ce733daf63efa7bc", "pack of butter-8eb8574ad3113fe1", "blue box-83cad2b4827645a7", "bowl of mashed potatoes-002d269d89eedde4", "spoon with black plastic-e6e8ceecf358d42f", "bowl white first-c2e0407519ee7422", "tablecloth second-45f2865c6be39ac8", "butter knife-53bbd99953a05cb6", "tong-e7370639c35b1b04", "plate from fridge-ecd0a2153426fac1", "yellow lid-7ac45c583477930d", "plastic box-70391c347c91a747", "white lid-e431f95e1b126d73", "cup-4afd3f8b8ee42a28", "box of juice-5c2bef829fcbd77a", "cleaning sponge-be589be085c055bd", "cleaning up liquid-5df5aed82d89db57", "the other rarely used sponge-b110567c14bcfec3", "some spoons-83d2c3b240890f42", "bottle-b407866279dbd617", "problematic track both sponges moved-3366bfb5ca0837c6", "lid of bottle-81b76427030c72d4", "second bottle-5a7bc705331948a8", "second bottle lid-86444162e8387f03", "green glass bottle-898faacaa1799e3d", "red steel bottle-ed8e06acdd9af392", "second green glass bottle-7f114b79c49c2087", "tablecloth third-7a144af344191a77", "frying pan-9e23ac56c59b36cf", "transparent bottle-e88dc4f9936f5268", "transparent bottle cap-5c39da8bad5aa3b2", "container with yellow lid-64cab09e0480f540", "weighing balance-8fd437b5090abd8c", "phone-96d82abe02abc89f", "bottle green-11b949afb6b981e7" ], "reasoning": "1. I observe the user take the vegetable oil from the cabinet at 02:25.\n2. At 02:30 I observe the user placing the cap (cap of vegetable oil bottle-c78dd56eeeb4b271) of the vegetable oil (bottle of vegetable oil-e4628dd7f2603a09) down on the counter.\n3. At 02:30, I observe the user touch the handle to open the drawer with their right hand.\n4. At 02:31, I observe the user take a spoon (spoon first-eca5e5111ccd402f) from the drawer with their right hand.\n5. I determine that the second object the user touches with their right hand is a spoon.", "referenced_object_masks": [ "spoon first-eca5e5111ccd402f", "bottle of vegetable oil-e4628dd7f2603a09", "cap of vegetable oil bottle-c78dd56eeeb4b271" ] }, { "key": "P06-20240510-140459:65876862e7a1d139c7394825209cf9fad743486f", "video_id": "P06-20240510-140459", "question": "For what reason do I empty the second bowl for the second time?", "answer_choice_0": "To wash the second bowl", "answer_choice_1": "To weigh the filling", "answer_choice_2": "To put the filling on the rolled-out dough", "answer_choice_3": "To put it back into the first bowl", "answer_choice_4": "To weigh the second bowl", "answer_id": 2, "answer": "To put the filling on the rolled-out dough", "question_type": [ "Goal Reasoning", "Action Recognition", "Cause and Effect", "Counting", "Object Recognition" ], "objects_of_interest": [ "outer plastic packaging of packet-655ef824f77ae4d6", "saucepan-8eebf5d83b84ef0b", "glass-e08f5e0176246774", "second glass-1cee33a9db671045", "pen-3ba6055a1b650d1c", "bottle of vegetable oil-e4628dd7f2603a09", "cap of vegetable oil bottle-c78dd56eeeb4b271", "spoon first-eca5e5111ccd402f", "chopping board-e341e1f5cd647520", "tablecloth-8bba3349418a6941", "notebook-7c11ad8d39252294", "translucent plate-3d101df93945a08c", "orange packet-85a33c3522b0b3dc", "saucepan with curry-2efc9fd3bf9cf271", "pressure cooker-8f57c2e0c741b6a7", "rolling pin-c83c2c5391e13ec0", "second translucent plate-c8be4725fbbf6120", "yellow packet-ce733daf63efa7bc", "pack of butter-8eb8574ad3113fe1", "blue box-83cad2b4827645a7", "bowl of mashed potatoes-002d269d89eedde4", "spoon with black plastic-e6e8ceecf358d42f", "bowl white first-c2e0407519ee7422", "tablecloth second-45f2865c6be39ac8", "butter knife-53bbd99953a05cb6", "tong-e7370639c35b1b04", "plate from fridge-ecd0a2153426fac1", "yellow lid-7ac45c583477930d", "plastic box-70391c347c91a747", "white lid-e431f95e1b126d73", "cup-4afd3f8b8ee42a28", "box of juice-5c2bef829fcbd77a", "cleaning sponge-be589be085c055bd", "cleaning up liquid-5df5aed82d89db57", "the other rarely used sponge-b110567c14bcfec3", "some spoons-83d2c3b240890f42", "bottle-b407866279dbd617", "problematic track both sponges moved-3366bfb5ca0837c6", "lid of bottle-81b76427030c72d4", "second bottle-5a7bc705331948a8", "second bottle lid-86444162e8387f03", "green glass bottle-898faacaa1799e3d", "red steel bottle-ed8e06acdd9af392", "second green glass bottle-7f114b79c49c2087", "tablecloth third-7a144af344191a77", "frying pan-9e23ac56c59b36cf", "transparent bottle-e88dc4f9936f5268", "transparent bottle cap-5c39da8bad5aa3b2", "container with yellow lid-64cab09e0480f540", "weighing balance-8fd437b5090abd8c", "phone-96d82abe02abc89f", "bottle green-11b949afb6b981e7" ], "reasoning": "1. I watch for the user to find the bowl (bowl of mashed potatoes-002d269d89eedde4) with filling in it, which is first seen at 10:40, when the user takes it out of the refrigerator.\n2. From 11:17-11:30, the user arranges the filling on the rolled-out dough.\n3. At 11:53, the user puts the filling into a second bowl (bowl white first-c2e0407519ee7422).\n4. I continued watching until the user took the filling out of the second bowl (bowl white first-c2e0407519ee7422) which occurs for the first time at 12:05. The user then puts the empty second bowl (bowl white first-c2e0407519ee7422) on the scale at 12:10 to weigh it, being this the first instance the user empties the second bowl (bowl white first-c2e0407519ee7422). \n5. The user removes the second bowl (bowl white first-c2e0407519ee7422) from the scale and puts some filling in it again at 12:22. \n6. At 12:27, the user places the now filled second bowl (bowl white first-c2e0407519ee7422) on the scale again. \n7. I continued watching to identify the second instance where the user empties the second bowl (bowl white first-c2e0407519ee7422), which happens at 12:55. In this instance, the user emptied the bowl (bowl white first-c2e0407519ee7422) to put the filling on the rolled-out dough. \n8. With this information, I conclude that the user takes the filling out of the second bowl (bowl white first-c2e0407519ee7422) for the second time so they can put the filling on the rolled-out dough. Therefore, the correct answer is (C) To put the filling on the rolled-out dough.", "referenced_object_masks": [ "bowl white first-c2e0407519ee7422", "bowl of mashed potatoes-002d269d89eedde4" ] }, { "key": "P06-20240510-140459:8132228c430a01433525a48806e8898de0540551", "video_id": "P06-20240510-140459", "question": "In which direction do I consistently glance while using the glass as a scoop?", "answer_choice_0": "Down into the bag", "answer_choice_1": "Left towards the scale", "answer_choice_2": "Ahead at the cabinets", "answer_choice_3": "Down at the floor", "answer_choice_4": "Right to the stove", "answer_id": 1, "answer": "Left towards the scale", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "outer plastic packaging of packet-655ef824f77ae4d6", "saucepan-8eebf5d83b84ef0b", "glass-e08f5e0176246774", "second glass-1cee33a9db671045", "pen-3ba6055a1b650d1c", "bottle of vegetable oil-e4628dd7f2603a09", "cap of vegetable oil bottle-c78dd56eeeb4b271", "spoon first-eca5e5111ccd402f", "chopping board-e341e1f5cd647520", "tablecloth-8bba3349418a6941", "notebook-7c11ad8d39252294", "translucent plate-3d101df93945a08c", "orange packet-85a33c3522b0b3dc", "saucepan with curry-2efc9fd3bf9cf271", "pressure cooker-8f57c2e0c741b6a7", "rolling pin-c83c2c5391e13ec0", "second translucent plate-c8be4725fbbf6120", "yellow packet-ce733daf63efa7bc", "pack of butter-8eb8574ad3113fe1", "blue box-83cad2b4827645a7", "bowl of mashed potatoes-002d269d89eedde4", "spoon with black plastic-e6e8ceecf358d42f", "bowl white first-c2e0407519ee7422", "tablecloth second-45f2865c6be39ac8", "butter knife-53bbd99953a05cb6", "tong-e7370639c35b1b04", "plate from fridge-ecd0a2153426fac1", "yellow lid-7ac45c583477930d", "plastic box-70391c347c91a747", "white lid-e431f95e1b126d73", "cup-4afd3f8b8ee42a28", "box of juice-5c2bef829fcbd77a", "cleaning sponge-be589be085c055bd", "cleaning up liquid-5df5aed82d89db57", "the other rarely used sponge-b110567c14bcfec3", "some spoons-83d2c3b240890f42", "bottle-b407866279dbd617", "problematic track both sponges moved-3366bfb5ca0837c6", "lid of bottle-81b76427030c72d4", "second bottle-5a7bc705331948a8", "second bottle lid-86444162e8387f03", "green glass bottle-898faacaa1799e3d", "red steel bottle-ed8e06acdd9af392", "second green glass bottle-7f114b79c49c2087", "tablecloth third-7a144af344191a77", "frying pan-9e23ac56c59b36cf", "transparent bottle-e88dc4f9936f5268", "transparent bottle cap-5c39da8bad5aa3b2", "container with yellow lid-64cab09e0480f540", "weighing balance-8fd437b5090abd8c", "phone-96d82abe02abc89f", "bottle green-11b949afb6b981e7" ], "reasoning": "1. I watch for the user to utilize the glass (glass-e08f5e0176246774) as a scoop, but they do not do this with the first glass (glass-e08f5e0176246774) that they pick up at 00:11.\n2. The user picks up a second glass (second glass-1cee33a9db671045) at 00:13, and this glass (second glass-1cee33a9db671045) is utilized to scoop grain from a plastic bag from 00:33 to 00:43, and again from 00:53 to 01:05.\n3. I looked to see where the user glances during these two time spans, and they glance left at the scale twice from 00:33-00:43, and twice from 00:53-01:05, only looking down at the plastic bag the rest of the time.\n4. The answer is left towards the scale .", "referenced_object_masks": [ "glass-e08f5e0176246774", "second glass-1cee33a9db671045" ] }, { "key": "P06-20240510-140459:84916c4f9c685e95eed98225bda17178509fb394", "video_id": "P06-20240510-140459", "question": "After putting the saucepan in the sink, how many objects (not counting water) do I put into it until I return to the chopping board to knead the dough?", "answer_choice_0": "4", "answer_choice_1": "2", "answer_choice_2": "5", "answer_choice_3": "3", "answer_choice_4": "1", "answer_id": 3, "answer": "3", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counting" ], "objects_of_interest": [ "outer plastic packaging of packet-655ef824f77ae4d6", "saucepan-8eebf5d83b84ef0b", "glass-e08f5e0176246774", "second glass-1cee33a9db671045", "pen-3ba6055a1b650d1c", "bottle of vegetable oil-e4628dd7f2603a09", "cap of vegetable oil bottle-c78dd56eeeb4b271", "spoon first-eca5e5111ccd402f", "chopping board-e341e1f5cd647520", "tablecloth-8bba3349418a6941", "notebook-7c11ad8d39252294", "translucent plate-3d101df93945a08c", "orange packet-85a33c3522b0b3dc", "saucepan with curry-2efc9fd3bf9cf271", "pressure cooker-8f57c2e0c741b6a7", "rolling pin-c83c2c5391e13ec0", "second translucent plate-c8be4725fbbf6120", "yellow packet-ce733daf63efa7bc", "pack of butter-8eb8574ad3113fe1", "blue box-83cad2b4827645a7", "bowl of mashed potatoes-002d269d89eedde4", "spoon with black plastic-e6e8ceecf358d42f", "bowl white first-c2e0407519ee7422", "tablecloth second-45f2865c6be39ac8", "butter knife-53bbd99953a05cb6", "tong-e7370639c35b1b04", "plate from fridge-ecd0a2153426fac1", "yellow lid-7ac45c583477930d", "plastic box-70391c347c91a747", "white lid-e431f95e1b126d73", "cup-4afd3f8b8ee42a28", "box of juice-5c2bef829fcbd77a", "cleaning sponge-be589be085c055bd", "cleaning up liquid-5df5aed82d89db57", "the other rarely used sponge-b110567c14bcfec3", "some spoons-83d2c3b240890f42", "bottle-b407866279dbd617", "problematic track both sponges moved-3366bfb5ca0837c6", "lid of bottle-81b76427030c72d4", "second bottle-5a7bc705331948a8", "second bottle lid-86444162e8387f03", "green glass bottle-898faacaa1799e3d", "red steel bottle-ed8e06acdd9af392", "second green glass bottle-7f114b79c49c2087", "tablecloth third-7a144af344191a77", "frying pan-9e23ac56c59b36cf", "transparent bottle-e88dc4f9936f5268", "transparent bottle cap-5c39da8bad5aa3b2", "container with yellow lid-64cab09e0480f540", "weighing balance-8fd437b5090abd8c", "phone-96d82abe02abc89f", "bottle green-11b949afb6b981e7" ], "reasoning": "1. At 00:00, I locate the saucepan (saucepan-8eebf5d83b84ef0b) on the counter in front of the user and the chopping board (chopping board-e341e1f5cd647520) on the far end of the counter to the right of the user.\n2. At 04:57, I observe the user place the saucepan (saucepan-8eebf5d83b84ef0b) into the sink directly in front of them.\n3. At 06:00 I observe the user return to the chopping board (chopping board-e341e1f5cd647520) to continue kneading.\n4. From 04:57 to 06:00, I observe the user place two glasses (glass-e08f5e0176246774, second glass-1cee33a9db671045) at 05:35 and a spoon (spoon first-eca5e5111ccd402f) at 05:49 into the saucepan (saucepan-8eebf5d83b84ef0b).\n5. I determine that the user places 3 objects into the saucepan (saucepan-8eebf5d83b84ef0b).", "referenced_object_masks": [ "saucepan-8eebf5d83b84ef0b", "spoon first-eca5e5111ccd402f", "chopping board-e341e1f5cd647520", "glass-e08f5e0176246774", "second glass-1cee33a9db671045" ] }, { "key": "P06-20240510-140459:93244a32378ce0dd8ef37c12e8a63bd93344d41f", "video_id": "P06-20240510-140459", "question": "What is the ratio of clockwise to counterclockwise circles I make with my hand as the mixture changes from ingredients into dough before clenching my hand in a fist?", "answer_choice_0": "4:5", "answer_choice_1": "1:2", "answer_choice_2": "3:1", "answer_choice_3": "5:4", "answer_choice_4": "2:3", "answer_id": 0, "answer": "4:5", "question_type": [ "Action Recognition", "Object Recognition", "Counting", "State Changes" ], "objects_of_interest": [ "outer plastic packaging of packet-655ef824f77ae4d6", "saucepan-8eebf5d83b84ef0b", "glass-e08f5e0176246774", "second glass-1cee33a9db671045", "pen-3ba6055a1b650d1c", "bottle of vegetable oil-e4628dd7f2603a09", "cap of vegetable oil bottle-c78dd56eeeb4b271", "spoon first-eca5e5111ccd402f", "chopping board-e341e1f5cd647520", "tablecloth-8bba3349418a6941", "notebook-7c11ad8d39252294", "translucent plate-3d101df93945a08c", "orange packet-85a33c3522b0b3dc", "saucepan with curry-2efc9fd3bf9cf271", "pressure cooker-8f57c2e0c741b6a7", "rolling pin-c83c2c5391e13ec0", "second translucent plate-c8be4725fbbf6120", "yellow packet-ce733daf63efa7bc", "pack of butter-8eb8574ad3113fe1", "blue box-83cad2b4827645a7", "bowl of mashed potatoes-002d269d89eedde4", "spoon with black plastic-e6e8ceecf358d42f", "bowl white first-c2e0407519ee7422", "tablecloth second-45f2865c6be39ac8", "butter knife-53bbd99953a05cb6", "tong-e7370639c35b1b04", "plate from fridge-ecd0a2153426fac1", "yellow lid-7ac45c583477930d", "plastic box-70391c347c91a747", "white lid-e431f95e1b126d73", "cup-4afd3f8b8ee42a28", "box of juice-5c2bef829fcbd77a", "cleaning sponge-be589be085c055bd", "cleaning up liquid-5df5aed82d89db57", "the other rarely used sponge-b110567c14bcfec3", "some spoons-83d2c3b240890f42", "bottle-b407866279dbd617", "problematic track both sponges moved-3366bfb5ca0837c6", "lid of bottle-81b76427030c72d4", "second bottle-5a7bc705331948a8", "second bottle lid-86444162e8387f03", "green glass bottle-898faacaa1799e3d", "red steel bottle-ed8e06acdd9af392", "second green glass bottle-7f114b79c49c2087", "tablecloth third-7a144af344191a77", "frying pan-9e23ac56c59b36cf", "transparent bottle-e88dc4f9936f5268", "transparent bottle cap-5c39da8bad5aa3b2", "container with yellow lid-64cab09e0480f540", "weighing balance-8fd437b5090abd8c", "phone-96d82abe02abc89f", "bottle green-11b949afb6b981e7" ], "reasoning": "1. I watch for the user to use their hand to stir the mixture in the saucepan (saucepan-8eebf5d83b84ef0b), which they start to do at 02:51.\n2. The user moves their right hand in circles in the following order: counterclockwise x 1 at 02:51, clockwise x 1 at 02:52, counterclockwise x 2 from 02:53 to 02:54, clockwise x 2 from 02:54 to 02:55, counterclockwise x 2 from 02:55 to 02:56, and clockwise x 1 at 02:56.\n3. At 02:57, the user makes a fist in the mixture.\n4. The user moves their hand clockwise 4 times, and counterclockwise 5 times.\n5. The ratio is 4:5.", "referenced_object_masks": [ "saucepan-8eebf5d83b84ef0b" ] }, { "key": "P06-20240510-140459:a2a87bd1af8d930ff439d4f6916185b5ba895abb", "video_id": "P06-20240510-140459", "question": "What is the fourth action that I do with the vegetable oil besides opening, closing, and pouring it?", "answer_choice_0": "Bump it out of the way", "answer_choice_1": "Put it back into the cabinet", "answer_choice_2": "Throw it away", "answer_choice_3": "Move it to the other side of the stove", "answer_choice_4": "Get it out of the cabinet", "answer_id": 0, "answer": "Bump it out of the way", "question_type": [ "Action Recognition", "Object Recognition", "Counting", "Numerical Reasoning" ], "objects_of_interest": [ "outer plastic packaging of packet-655ef824f77ae4d6", "saucepan-8eebf5d83b84ef0b", "glass-e08f5e0176246774", "second glass-1cee33a9db671045", "pen-3ba6055a1b650d1c", "bottle of vegetable oil-e4628dd7f2603a09", "cap of vegetable oil bottle-c78dd56eeeb4b271", "spoon first-eca5e5111ccd402f", "chopping board-e341e1f5cd647520", "tablecloth-8bba3349418a6941", "notebook-7c11ad8d39252294", "translucent plate-3d101df93945a08c", "orange packet-85a33c3522b0b3dc", "saucepan with curry-2efc9fd3bf9cf271", "pressure cooker-8f57c2e0c741b6a7", "rolling pin-c83c2c5391e13ec0", "second translucent plate-c8be4725fbbf6120", "yellow packet-ce733daf63efa7bc", "pack of butter-8eb8574ad3113fe1", "blue box-83cad2b4827645a7", "bowl of mashed potatoes-002d269d89eedde4", "spoon with black plastic-e6e8ceecf358d42f", "bowl white first-c2e0407519ee7422", "tablecloth second-45f2865c6be39ac8", "butter knife-53bbd99953a05cb6", "tong-e7370639c35b1b04", "plate from fridge-ecd0a2153426fac1", "yellow lid-7ac45c583477930d", "plastic box-70391c347c91a747", "white lid-e431f95e1b126d73", "cup-4afd3f8b8ee42a28", "box of juice-5c2bef829fcbd77a", "cleaning sponge-be589be085c055bd", "cleaning up liquid-5df5aed82d89db57", "the other rarely used sponge-b110567c14bcfec3", "some spoons-83d2c3b240890f42", "bottle-b407866279dbd617", "problematic track both sponges moved-3366bfb5ca0837c6", "lid of bottle-81b76427030c72d4", "second bottle-5a7bc705331948a8", "second bottle lid-86444162e8387f03", "green glass bottle-898faacaa1799e3d", "red steel bottle-ed8e06acdd9af392", "second green glass bottle-7f114b79c49c2087", "tablecloth third-7a144af344191a77", "frying pan-9e23ac56c59b36cf", "transparent bottle-e88dc4f9936f5268", "transparent bottle cap-5c39da8bad5aa3b2", "container with yellow lid-64cab09e0480f540", "weighing balance-8fd437b5090abd8c", "phone-96d82abe02abc89f", "bottle green-11b949afb6b981e7" ], "reasoning": "1. I watch for the user to perform tasks with the bottle of vegetable oil (bottle of vegetable oil-e4628dd7f2603a09).\n2. The user retrieves the bottle of vegetable oil (bottle of vegetable oil-e4628dd7f2603a09) from the cabinet at 02:25, which is the first thing they do with it.\n3. At 02:29, the user takes the cap (cap of vegetable oil bottle-c78dd56eeeb4b271) off, which does not count in the list.\n4. From 02:33-02:37, the user pours from the bottle of vegetable oil (bottle of vegetable oil-e4628dd7f2603a09) into a spoon (spoon first-eca5e5111ccd402f) which does not count in the list.\n5. At 02:39, the user places the bottle of vegetable oil (bottle of vegetable oil-e4628dd7f2603a09) on the counter next to the notebook (notebook-7c11ad8d39252294) at 02:39, which counts as the second thing that they do with it.\n6. The user puts the cap (cap of vegetable oil bottle-c78dd56eeeb4b271) back on at 02:42, which does not count in the list.\n7. At 02:49, the user puts the bottle of vegetable oil (bottle of vegetable oil-e4628dd7f2603a09) on the other side of the counter, which counts as the third thing they do with it.\n8. At 04:24, the user bumps the bottle of vegetable oil (bottle of vegetable oil-e4628dd7f2603a09) out of the way with the chooping board (chopping board-e341e1f5cd647520), making it the fourth thing they do with the bottle of vegetable oil (bottle of vegetable oil-e4628dd7f2603a09).", "referenced_object_masks": [ "spoon first-eca5e5111ccd402f", "chopping board-e341e1f5cd647520", "bottle of vegetable oil-e4628dd7f2603a09", "notebook-7c11ad8d39252294", "cap of vegetable oil bottle-c78dd56eeeb4b271" ] }, { "key": "P06-20240510-140459:d12473d1da81e32f11d27e0384d704f6d22a5f7f", "video_id": "P06-20240510-140459", "question": "After placing the plastic bag on top of the fridge, what is the second object I touch with my right hand when I return to the kitchen?", "answer_choice_0": "Clean glass", "answer_choice_1": "Notebook", "answer_choice_2": "Digital scale", "answer_choice_3": "Faucet", "answer_choice_4": "Pencil", "answer_id": 0, "answer": "Clean glass", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counting" ], "objects_of_interest": [ "outer plastic packaging of packet-655ef824f77ae4d6", "saucepan-8eebf5d83b84ef0b", "glass-e08f5e0176246774", "second glass-1cee33a9db671045", "pen-3ba6055a1b650d1c", "bottle of vegetable oil-e4628dd7f2603a09", "cap of vegetable oil bottle-c78dd56eeeb4b271", "spoon first-eca5e5111ccd402f", "chopping board-e341e1f5cd647520", "tablecloth-8bba3349418a6941", "notebook-7c11ad8d39252294", "translucent plate-3d101df93945a08c", "orange packet-85a33c3522b0b3dc", "saucepan with curry-2efc9fd3bf9cf271", "pressure cooker-8f57c2e0c741b6a7", "rolling pin-c83c2c5391e13ec0", "second translucent plate-c8be4725fbbf6120", "yellow packet-ce733daf63efa7bc", "pack of butter-8eb8574ad3113fe1", "blue box-83cad2b4827645a7", "bowl of mashed potatoes-002d269d89eedde4", "spoon with black plastic-e6e8ceecf358d42f", "bowl white first-c2e0407519ee7422", "tablecloth second-45f2865c6be39ac8", "butter knife-53bbd99953a05cb6", "tong-e7370639c35b1b04", "plate from fridge-ecd0a2153426fac1", "yellow lid-7ac45c583477930d", "plastic box-70391c347c91a747", "white lid-e431f95e1b126d73", "cup-4afd3f8b8ee42a28", "box of juice-5c2bef829fcbd77a", "cleaning sponge-be589be085c055bd", "cleaning up liquid-5df5aed82d89db57", "the other rarely used sponge-b110567c14bcfec3", "some spoons-83d2c3b240890f42", "bottle-b407866279dbd617", "problematic track both sponges moved-3366bfb5ca0837c6", "lid of bottle-81b76427030c72d4", "second bottle-5a7bc705331948a8", "second bottle lid-86444162e8387f03", "green glass bottle-898faacaa1799e3d", "red steel bottle-ed8e06acdd9af392", "second green glass bottle-7f114b79c49c2087", "tablecloth third-7a144af344191a77", "frying pan-9e23ac56c59b36cf", "transparent bottle-e88dc4f9936f5268", "transparent bottle cap-5c39da8bad5aa3b2", "container with yellow lid-64cab09e0480f540", "weighing balance-8fd437b5090abd8c", "phone-96d82abe02abc89f", "bottle green-11b949afb6b981e7" ], "reasoning": "1. I locate the plastic bag (outer plastic packaging of packet-655ef824f77ae4d6) at 00:00 in front of the user on the counter.\n2. I observe the user place the plastic bag (outer plastic packaging of packet-655ef824f77ae4d6) on top of the fridge at 01:34.\n3. I observe the user walk back into the kitchen at 01:36.\n4. At 01:40 I observe the user touch the dirty glass (second glass-1cee33a9db671045) at 01:40 and the clean glass (glass-e08f5e0176246774) at 01:43.\n5. I determine the second object the user touches with their right hand is the clean glass (glass-e08f5e0176246774).", "referenced_object_masks": [ "glass-e08f5e0176246774", "outer plastic packaging of packet-655ef824f77ae4d6", "second glass-1cee33a9db671045" ] }, { "key": "P06-20240510-140459:f8a6e104594f34c62195419c19e101fbf3adabb1", "video_id": "P06-20240510-140459", "question": "What number would I get if I added together the largest digit from the first two measurements?", "answer_choice_0": "5", "answer_choice_1": "8", "answer_choice_2": "9", "answer_choice_3": "6", "answer_choice_4": "7", "answer_id": 2, "answer": "9", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning", "Numerical Reasoning", "Counting" ], "objects_of_interest": [ "outer plastic packaging of packet-655ef824f77ae4d6", "saucepan-8eebf5d83b84ef0b", "glass-e08f5e0176246774", "second glass-1cee33a9db671045", "pen-3ba6055a1b650d1c", "bottle of vegetable oil-e4628dd7f2603a09", "cap of vegetable oil bottle-c78dd56eeeb4b271", "spoon first-eca5e5111ccd402f", "chopping board-e341e1f5cd647520", "tablecloth-8bba3349418a6941", "notebook-7c11ad8d39252294", "translucent plate-3d101df93945a08c", "orange packet-85a33c3522b0b3dc", "saucepan with curry-2efc9fd3bf9cf271", "pressure cooker-8f57c2e0c741b6a7", "rolling pin-c83c2c5391e13ec0", "second translucent plate-c8be4725fbbf6120", "yellow packet-ce733daf63efa7bc", "pack of butter-8eb8574ad3113fe1", "blue box-83cad2b4827645a7", "bowl of mashed potatoes-002d269d89eedde4", "spoon with black plastic-e6e8ceecf358d42f", "bowl white first-c2e0407519ee7422", "tablecloth second-45f2865c6be39ac8", "butter knife-53bbd99953a05cb6", "tong-e7370639c35b1b04", "plate from fridge-ecd0a2153426fac1", "yellow lid-7ac45c583477930d", "plastic box-70391c347c91a747", "white lid-e431f95e1b126d73", "cup-4afd3f8b8ee42a28", "box of juice-5c2bef829fcbd77a", "cleaning sponge-be589be085c055bd", "cleaning up liquid-5df5aed82d89db57", "the other rarely used sponge-b110567c14bcfec3", "some spoons-83d2c3b240890f42", "bottle-b407866279dbd617", "problematic track both sponges moved-3366bfb5ca0837c6", "lid of bottle-81b76427030c72d4", "second bottle-5a7bc705331948a8", "second bottle lid-86444162e8387f03", "green glass bottle-898faacaa1799e3d", "red steel bottle-ed8e06acdd9af392", "second green glass bottle-7f114b79c49c2087", "tablecloth third-7a144af344191a77", "frying pan-9e23ac56c59b36cf", "transparent bottle-e88dc4f9936f5268", "transparent bottle cap-5c39da8bad5aa3b2", "container with yellow lid-64cab09e0480f540", "weighing balance-8fd437b5090abd8c", "phone-96d82abe02abc89f", "bottle green-11b949afb6b981e7" ], "reasoning": "1. I locate the digital scale (weighing balance-8fd437b5090abd8c) to the left of the user on the counter at 00:02.\n2. I observe the digital scale (weighing balance-8fd437b5090abd8c) showing 0, 1, 0, and 4 for the glass (second glass-1cee33a9db671045) at 00:29.\n3. I observe the digital scale (weighing balance-8fd437b5090abd8c) showing 0, 5, and 4 for the glass (second glass-1cee33a9db671045) filled with the flour from inside the bag (outer plastic packaging of packet-655ef824f77ae4d6) at 00:45.\n4. I determine the largest digit from the first and second readings are 4 and 5 respectively.\n5. I add 4 and 5 together to get 9.", "referenced_object_masks": [ "weighing balance-8fd437b5090abd8c", "outer plastic packaging of packet-655ef824f77ae4d6", "second glass-1cee33a9db671045" ] }, { "key": "P06-20240511-102207:008c91126b4d899f1a80e169f76e655e3f9a3edc", "video_id": "P06-20240511-102207", "question": "After I turn on three objects, how many cuts do I make the second time I touch the knife?", "answer_choice_0": "72", "answer_choice_1": "30", "answer_choice_2": "40", "answer_choice_3": "17", "answer_choice_4": "51", "answer_id": 1, "answer": "30", "question_type": [ "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counting" ], "objects_of_interest": [ "box-640d272cf8349df6", "box2-a9b0e10c2420cd6e", "box3-ea42b156f2ad559a", "carton of eggs-55be73789bc20b80", "pack of bread-f3a646eccced80d0", "milk bottle-bba3b2999c1daaec", "pack of butter-1c8ad2cb527706c4", "pen-0ec8edf8b9be4bdf", "notebook-aa7029dac4f75bd9", "weighing balance-aa98fc514dd3d318", "bowl-1adf54e7494b619f", "chopping board-971cefd7dca430b4", "onion-da6bc7290991b46c", "frying pan-1d1076c0ee9048c3", "knife-2f7108372eb0e0ed", "bowl2-50fec6214b51c091", "half onion-babd1ead2c43711c", "bowl3-e2e6d93ce91f7526", "egg-d2f8ae1fc6a46525", "egg2-7e12972ee992d944", "egg3-41dffe3fe003b3cb", "egg4-8c48fba2f5f37fcd", "cloth-604cc06f7bb797d1", "spoon-12cdc7b4e0b84262", "bottle of salt-6e1f787f9b19866e", "jar of spices with green lid-51d8167cdc8ff787", "green lid of jar-337755054caa5449", "spoon2-4373d15a83be3fe1", "one bread-43c1df3042dce609", "second bread-0d6ba7767da84772", "butter knife-e78e149815883f02", "measuring spoon-d13cc25ae512badf", "piece of butter-5c047e91c35c4fb8", "spatula-317d6d35610bf64e", "plate-fdf4ad1d588f85fc", "lid of box-6bf78bf8202064c8", "piece of pancake-ae26365748386fc6", "piece of onion-5fdecd10aadac956", "paper towel roll-eb876c03d8e18f69", "tablecloth-b9632cfe889b0e73", "mug-e2b33a685ee3a4dd", "mug2-4246bb59f3f9ca2e", "mug3-287f7c88aa72378b", "mug4-4575f761a634e5f7", "cover of milk bottle-51b0cd515664ab5c", "lid of milk carton-d862c5091443439f", "bread omelette-3c3617096e753b83", "plate2-80ec713704c97f03", "spoon3-dd29784d38c2efaf", "unused jar-7812e55e59c91b48", "bottle of ketchup-ba77c3b51f0094af", "coffee jar-0f9a1a19e2b8d431", "bottle of oil-a6e4315c20f9fb2f", "jar of brown sugar-ddde96782a0abd30", "lid of sugar jar-371d975182a92a55", "lid of coffee jar-e5341ea5e919e1b0", "mug5-e68fd8aa2f632ba6", "bottle of dishwashing liquid-2152a55c290ad1ee", "electric kettle-4a1c2430fde81b5d", "milk frother-396a13b0bb9c8864", "stand of milk frother-d8640ec0fc0ff6a0", "plastic box-b3f2b2d40130eeb1", "another plastic box-924886b597cc2ac6", "another plastic box2-1b2f7c36e3313af3", "chopping board2-199392c7b2d821c7", "glass-8accf78b890dc922", "bowl4-295536e18f4dee4d", "spatula2-f6b7653f877a2f05", "butter knife2-cd28a7ebb40a3971", "spoon4-a69c5d162276f081", "butter knife3-023310228a51a42a", "tablecloth2-2b4da187831358f3", "sponge-2e4bbff4fb224ada", "sponge2-0545c267e65bd6d5", "spoon5-07fdb7a150641b40", "lid of ketchup bottle-9321836ac6ae2868", "lid of pressure cooker-dfa8a08d2c3832df", "wooden spoon-e99cb6c127e2b9bb", "pressure cooker-c0ae81c09633f6ba", "spoon6-a7ee870dd05ec00c", "spoon7-7bb703ff8228d628", "spatula3-fff665c3b13115c6" ], "reasoning": "1. I observe the user press the light switch at 00:01 in front of them to the right.\n2. I observe the user press the stove light at 00:04 in front of them.\n3. I observe the user turn on the digital scale (weighing balance-aa98fc514dd3d318) at 02:59 on the counter in front of them.\n4. Now that I found the 3 things the user turned on, I look for the second time they touch a knife (knife-2f7108372eb0e0ed).\n4. I observe the user move a knife (knife-2f7108372eb0e0ed) on the cutting board (chopping board-971cefd7dca430b4) at 03:00. I count this as one time.\n5. I observe the user pick up the knife (knife-2f7108372eb0e0ed) again at 03:09. This is my second time.\n6. Between 03:09 and 04:21, I observe the user make 30 cuts to the left onion half with the knife (knife-2f7108372eb0e0ed).\n7. I observe the user drop the knife (knife-2f7108372eb0e0ed) at 04:21, so I stop counting.\n8. Therefore, my answer is 30 cuts.", "referenced_object_masks": [ "chopping board-971cefd7dca430b4", "knife-2f7108372eb0e0ed", "weighing balance-aa98fc514dd3d318" ] }, { "key": "P06-20240511-102207:0649a22b433bb99ae3804a9dd862187071260e6a", "video_id": "P06-20240511-102207", "question": "What direction would I have turned to look at the clock if it were in the opposite direction while I wrote in the notebook?", "answer_choice_0": "90 degrees to the left", "answer_choice_1": "90 degrees to the right", "answer_choice_2": "90 degrees upward", "answer_choice_3": "180 degrees around", "answer_choice_4": "90 degrees downward", "answer_id": 0, "answer": "90 degrees to the left", "question_type": [ "Object Recognition", "Action Recognition", "Cause and Effect", "Situational Awareness", "Egocentric & Spatial Perception", "Temporal Reasoning", "Counterfactual" ], "objects_of_interest": [ "box-640d272cf8349df6", "box2-a9b0e10c2420cd6e", "box3-ea42b156f2ad559a", "carton of eggs-55be73789bc20b80", "pack of bread-f3a646eccced80d0", "milk bottle-bba3b2999c1daaec", "pack of butter-1c8ad2cb527706c4", "pen-0ec8edf8b9be4bdf", "notebook-aa7029dac4f75bd9", "weighing balance-aa98fc514dd3d318", "bowl-1adf54e7494b619f", "chopping board-971cefd7dca430b4", "onion-da6bc7290991b46c", "frying pan-1d1076c0ee9048c3", "knife-2f7108372eb0e0ed", "bowl2-50fec6214b51c091", "half onion-babd1ead2c43711c", "bowl3-e2e6d93ce91f7526", "egg-d2f8ae1fc6a46525", "egg2-7e12972ee992d944", "egg3-41dffe3fe003b3cb", "egg4-8c48fba2f5f37fcd", "cloth-604cc06f7bb797d1", "spoon-12cdc7b4e0b84262", "bottle of salt-6e1f787f9b19866e", "jar of spices with green lid-51d8167cdc8ff787", "green lid of jar-337755054caa5449", "spoon2-4373d15a83be3fe1", "one bread-43c1df3042dce609", "second bread-0d6ba7767da84772", "butter knife-e78e149815883f02", "measuring spoon-d13cc25ae512badf", "piece of butter-5c047e91c35c4fb8", "spatula-317d6d35610bf64e", "plate-fdf4ad1d588f85fc", "lid of box-6bf78bf8202064c8", "piece of pancake-ae26365748386fc6", "piece of onion-5fdecd10aadac956", "paper towel roll-eb876c03d8e18f69", "tablecloth-b9632cfe889b0e73", "mug-e2b33a685ee3a4dd", "mug2-4246bb59f3f9ca2e", "mug3-287f7c88aa72378b", "mug4-4575f761a634e5f7", "cover of milk bottle-51b0cd515664ab5c", "lid of milk carton-d862c5091443439f", "bread omelette-3c3617096e753b83", "plate2-80ec713704c97f03", "spoon3-dd29784d38c2efaf", "unused jar-7812e55e59c91b48", "bottle of ketchup-ba77c3b51f0094af", "coffee jar-0f9a1a19e2b8d431", "bottle of oil-a6e4315c20f9fb2f", "jar of brown sugar-ddde96782a0abd30", "lid of sugar jar-371d975182a92a55", "lid of coffee jar-e5341ea5e919e1b0", "mug5-e68fd8aa2f632ba6", "bottle of dishwashing liquid-2152a55c290ad1ee", "electric kettle-4a1c2430fde81b5d", "milk frother-396a13b0bb9c8864", "stand of milk frother-d8640ec0fc0ff6a0", "plastic box-b3f2b2d40130eeb1", "another plastic box-924886b597cc2ac6", "another plastic box2-1b2f7c36e3313af3", "chopping board2-199392c7b2d821c7", "glass-8accf78b890dc922", "bowl4-295536e18f4dee4d", "spatula2-f6b7653f877a2f05", "butter knife2-cd28a7ebb40a3971", "spoon4-a69c5d162276f081", "butter knife3-023310228a51a42a", "tablecloth2-2b4da187831358f3", "sponge-2e4bbff4fb224ada", "sponge2-0545c267e65bd6d5", "spoon5-07fdb7a150641b40", "lid of ketchup bottle-9321836ac6ae2868", "lid of pressure cooker-dfa8a08d2c3832df", "wooden spoon-e99cb6c127e2b9bb", "pressure cooker-c0ae81c09633f6ba", "spoon6-a7ee870dd05ec00c", "spoon7-7bb703ff8228d628", "spatula3-fff665c3b13115c6" ], "reasoning": "1. I watched the video and found the notebook (notebook-aa7029dac4f75bd9) at 00:02.\n2. I saw the user write in the notebook (notebook-aa7029dac4f75bd9) with a pen (pen-0ec8edf8b9be4bdf) at 01:12-01:31.\n3. I found the clock at 01:31 when the user turned to the right to look at it.\n4. Since the user turned 90 degrees to the right to look at the clock at 01:31, the clock was consequently to the right of the user.\n5. The opposite of \"to the right\" is \"to the left\".\n6. Therefore, the user would have turned 90 degrees to the left to look at the clock if it were in the opposite direction while they wrote in the notebook (notebook-aa7029dac4f75bd9).", "referenced_object_masks": [ "pen-0ec8edf8b9be4bdf", "notebook-aa7029dac4f75bd9" ] }, { "key": "P06-20240511-102207:135a5d49402b46aab14b732429bf20d9d4aaa16c", "video_id": "P06-20240511-102207", "question": "What was the second item I put back in the refrigerator after taking it out?", "answer_choice_0": "A bright blue tupperware", "answer_choice_1": "A gray egg carton", "answer_choice_2": "A white milk jug", "answer_choice_3": "A white tupperware", "answer_choice_4": "A green tupperware", "answer_id": 4, "answer": "A green tupperware", "question_type": [ "Situational Awareness", "Counting", "Object Recognition", "Egocentric & Spatial Perception", "Action Recognition", "State Changes" ], "objects_of_interest": [ "box-640d272cf8349df6", "box2-a9b0e10c2420cd6e", "box3-ea42b156f2ad559a", "carton of eggs-55be73789bc20b80", "pack of bread-f3a646eccced80d0", "milk bottle-bba3b2999c1daaec", "pack of butter-1c8ad2cb527706c4", "pen-0ec8edf8b9be4bdf", "notebook-aa7029dac4f75bd9", "weighing balance-aa98fc514dd3d318", "bowl-1adf54e7494b619f", "chopping board-971cefd7dca430b4", "onion-da6bc7290991b46c", "frying pan-1d1076c0ee9048c3", "knife-2f7108372eb0e0ed", "bowl2-50fec6214b51c091", "half onion-babd1ead2c43711c", "bowl3-e2e6d93ce91f7526", "egg-d2f8ae1fc6a46525", "egg2-7e12972ee992d944", "egg3-41dffe3fe003b3cb", "egg4-8c48fba2f5f37fcd", "cloth-604cc06f7bb797d1", "spoon-12cdc7b4e0b84262", "bottle of salt-6e1f787f9b19866e", "jar of spices with green lid-51d8167cdc8ff787", "green lid of jar-337755054caa5449", "spoon2-4373d15a83be3fe1", "one bread-43c1df3042dce609", "second bread-0d6ba7767da84772", "butter knife-e78e149815883f02", "measuring spoon-d13cc25ae512badf", "piece of butter-5c047e91c35c4fb8", "spatula-317d6d35610bf64e", "plate-fdf4ad1d588f85fc", "lid of box-6bf78bf8202064c8", "piece of pancake-ae26365748386fc6", "piece of onion-5fdecd10aadac956", "paper towel roll-eb876c03d8e18f69", "tablecloth-b9632cfe889b0e73", "mug-e2b33a685ee3a4dd", "mug2-4246bb59f3f9ca2e", "mug3-287f7c88aa72378b", "mug4-4575f761a634e5f7", "cover of milk bottle-51b0cd515664ab5c", "lid of milk carton-d862c5091443439f", "bread omelette-3c3617096e753b83", "plate2-80ec713704c97f03", "spoon3-dd29784d38c2efaf", "unused jar-7812e55e59c91b48", "bottle of ketchup-ba77c3b51f0094af", "coffee jar-0f9a1a19e2b8d431", "bottle of oil-a6e4315c20f9fb2f", "jar of brown sugar-ddde96782a0abd30", "lid of sugar jar-371d975182a92a55", "lid of coffee jar-e5341ea5e919e1b0", "mug5-e68fd8aa2f632ba6", "bottle of dishwashing liquid-2152a55c290ad1ee", "electric kettle-4a1c2430fde81b5d", "milk frother-396a13b0bb9c8864", "stand of milk frother-d8640ec0fc0ff6a0", "plastic box-b3f2b2d40130eeb1", "another plastic box-924886b597cc2ac6", "another plastic box2-1b2f7c36e3313af3", "chopping board2-199392c7b2d821c7", "glass-8accf78b890dc922", "bowl4-295536e18f4dee4d", "spatula2-f6b7653f877a2f05", "butter knife2-cd28a7ebb40a3971", "spoon4-a69c5d162276f081", "butter knife3-023310228a51a42a", "tablecloth2-2b4da187831358f3", "sponge-2e4bbff4fb224ada", "sponge2-0545c267e65bd6d5", "spoon5-07fdb7a150641b40", "lid of ketchup bottle-9321836ac6ae2868", "lid of pressure cooker-dfa8a08d2c3832df", "wooden spoon-e99cb6c127e2b9bb", "pressure cooker-c0ae81c09633f6ba", "spoon6-a7ee870dd05ec00c", "spoon7-7bb703ff8228d628", "spatula3-fff665c3b13115c6" ], "reasoning": "1. I watched the video and found the refrigerator at 00:15.\n2. I counted each item the user put back in the refrigerator after taking it out.\n3. At 00:17, I saw the user take out a green tupperware (box-640d272cf8349df6) from the refrigerator.\n4. At 00:23, I saw the user take out a white tupperware (box3-ea42b156f2ad559a) from the refrigerator.\n5. At 00:25, I saw the user put the white tupperware (box3-ea42b156f2ad559a) back in the refrigerator, which was the first item to be put back after being taken out.\n6. At 00:39, I saw the user put the green tupperware (box-640d272cf8349df6) back in the refrigerator, which was the second item to be put back after being taken out.\n7. Therefore, the second item the user put back in the refrigerator after taking it out was a green tupperware (box-640d272cf8349df6).", "referenced_object_masks": [ "box3-ea42b156f2ad559a", "box-640d272cf8349df6" ] }, { "key": "P06-20240511-102207:17ca1e6c5bd2e5b5d1ec3e0b8da41483a9482b7e", "video_id": "P06-20240511-102207", "question": "After I slide the first white bowl twice, where are the eggs located relative to the first place I moved them to?", "answer_choice_0": "Above and to the right", "answer_choice_1": "Behind and to the right", "answer_choice_2": "Directly ahead", "answer_choice_3": "Directly below", "answer_choice_4": "To the left", "answer_id": 4, "answer": "To the left", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counting" ], "objects_of_interest": [ "box-640d272cf8349df6", "box2-a9b0e10c2420cd6e", "box3-ea42b156f2ad559a", "carton of eggs-55be73789bc20b80", "pack of bread-f3a646eccced80d0", "milk bottle-bba3b2999c1daaec", "pack of butter-1c8ad2cb527706c4", "pen-0ec8edf8b9be4bdf", "notebook-aa7029dac4f75bd9", "weighing balance-aa98fc514dd3d318", "bowl-1adf54e7494b619f", "chopping board-971cefd7dca430b4", "onion-da6bc7290991b46c", "frying pan-1d1076c0ee9048c3", "knife-2f7108372eb0e0ed", "bowl2-50fec6214b51c091", "half onion-babd1ead2c43711c", "bowl3-e2e6d93ce91f7526", "egg-d2f8ae1fc6a46525", "egg2-7e12972ee992d944", "egg3-41dffe3fe003b3cb", "egg4-8c48fba2f5f37fcd", "cloth-604cc06f7bb797d1", "spoon-12cdc7b4e0b84262", "bottle of salt-6e1f787f9b19866e", "jar of spices with green lid-51d8167cdc8ff787", "green lid of jar-337755054caa5449", "spoon2-4373d15a83be3fe1", "one bread-43c1df3042dce609", "second bread-0d6ba7767da84772", "butter knife-e78e149815883f02", "measuring spoon-d13cc25ae512badf", "piece of butter-5c047e91c35c4fb8", "spatula-317d6d35610bf64e", "plate-fdf4ad1d588f85fc", "lid of box-6bf78bf8202064c8", "piece of pancake-ae26365748386fc6", "piece of onion-5fdecd10aadac956", "paper towel roll-eb876c03d8e18f69", "tablecloth-b9632cfe889b0e73", "mug-e2b33a685ee3a4dd", "mug2-4246bb59f3f9ca2e", "mug3-287f7c88aa72378b", "mug4-4575f761a634e5f7", "cover of milk bottle-51b0cd515664ab5c", "lid of milk carton-d862c5091443439f", "bread omelette-3c3617096e753b83", "plate2-80ec713704c97f03", "spoon3-dd29784d38c2efaf", "unused jar-7812e55e59c91b48", "bottle of ketchup-ba77c3b51f0094af", "coffee jar-0f9a1a19e2b8d431", "bottle of oil-a6e4315c20f9fb2f", "jar of brown sugar-ddde96782a0abd30", "lid of sugar jar-371d975182a92a55", "lid of coffee jar-e5341ea5e919e1b0", "mug5-e68fd8aa2f632ba6", "bottle of dishwashing liquid-2152a55c290ad1ee", "electric kettle-4a1c2430fde81b5d", "milk frother-396a13b0bb9c8864", "stand of milk frother-d8640ec0fc0ff6a0", "plastic box-b3f2b2d40130eeb1", "another plastic box-924886b597cc2ac6", "another plastic box2-1b2f7c36e3313af3", "chopping board2-199392c7b2d821c7", "glass-8accf78b890dc922", "bowl4-295536e18f4dee4d", "spatula2-f6b7653f877a2f05", "butter knife2-cd28a7ebb40a3971", "spoon4-a69c5d162276f081", "butter knife3-023310228a51a42a", "tablecloth2-2b4da187831358f3", "sponge-2e4bbff4fb224ada", "sponge2-0545c267e65bd6d5", "spoon5-07fdb7a150641b40", "lid of ketchup bottle-9321836ac6ae2868", "lid of pressure cooker-dfa8a08d2c3832df", "wooden spoon-e99cb6c127e2b9bb", "pressure cooker-c0ae81c09633f6ba", "spoon6-a7ee870dd05ec00c", "spoon7-7bb703ff8228d628", "spatula3-fff665c3b13115c6" ], "reasoning": "1. I observe the user place the eggs (carton of eggs-55be73789bc20b80) down on the counter in front and to the left of the microwave at 00:32.\n2. I observe the user grab the first white bowl (bowl-1adf54e7494b619f) at 01:40.\n3. I observe the user slide the first white bowl (bowl-1adf54e7494b619f) across the counter forwards at 01:49.\n4. I observe the user slide the first white bowl (bowl-1adf54e7494b619f) across the counter to the left at 01:56.\n5. I identify the eggs (carton of eggs-55be73789bc20b80) to the right of the first white bowl (bowl-1adf54e7494b619f) at 01:56.\n6. I determine the eggs (carton of eggs-55be73789bc20b80) are to the left of the location where they were first placed.", "referenced_object_masks": [ "bowl-1adf54e7494b619f", "carton of eggs-55be73789bc20b80" ] }, { "key": "P06-20240511-102207:1f9ddf440c73d74fa76662379ed7ecb991149128", "video_id": "P06-20240511-102207", "question": "What is the final place I move the first white bowl?", "answer_choice_0": "A drying rack", "answer_choice_1": "A refrigerator", "answer_choice_2": "A dishwasher", "answer_choice_3": "A table", "answer_choice_4": "A sink", "answer_id": 0, "answer": "A drying rack", "question_type": [ "Object Recognition", "Action Recognition", "State Changes", "Situational Awareness" ], "objects_of_interest": [ "box-640d272cf8349df6", "box2-a9b0e10c2420cd6e", "box3-ea42b156f2ad559a", "carton of eggs-55be73789bc20b80", "pack of bread-f3a646eccced80d0", "milk bottle-bba3b2999c1daaec", "pack of butter-1c8ad2cb527706c4", "pen-0ec8edf8b9be4bdf", "notebook-aa7029dac4f75bd9", "weighing balance-aa98fc514dd3d318", "bowl-1adf54e7494b619f", "chopping board-971cefd7dca430b4", "onion-da6bc7290991b46c", "frying pan-1d1076c0ee9048c3", "knife-2f7108372eb0e0ed", "bowl2-50fec6214b51c091", "half onion-babd1ead2c43711c", "bowl3-e2e6d93ce91f7526", "egg-d2f8ae1fc6a46525", "egg2-7e12972ee992d944", "egg3-41dffe3fe003b3cb", "egg4-8c48fba2f5f37fcd", "cloth-604cc06f7bb797d1", "spoon-12cdc7b4e0b84262", "bottle of salt-6e1f787f9b19866e", "jar of spices with green lid-51d8167cdc8ff787", "green lid of jar-337755054caa5449", "spoon2-4373d15a83be3fe1", "one bread-43c1df3042dce609", "second bread-0d6ba7767da84772", "butter knife-e78e149815883f02", "measuring spoon-d13cc25ae512badf", "piece of butter-5c047e91c35c4fb8", "spatula-317d6d35610bf64e", "plate-fdf4ad1d588f85fc", "lid of box-6bf78bf8202064c8", "piece of pancake-ae26365748386fc6", "piece of onion-5fdecd10aadac956", "paper towel roll-eb876c03d8e18f69", "tablecloth-b9632cfe889b0e73", "mug-e2b33a685ee3a4dd", "mug2-4246bb59f3f9ca2e", "mug3-287f7c88aa72378b", "mug4-4575f761a634e5f7", "cover of milk bottle-51b0cd515664ab5c", "lid of milk carton-d862c5091443439f", "bread omelette-3c3617096e753b83", "plate2-80ec713704c97f03", "spoon3-dd29784d38c2efaf", "unused jar-7812e55e59c91b48", "bottle of ketchup-ba77c3b51f0094af", "coffee jar-0f9a1a19e2b8d431", "bottle of oil-a6e4315c20f9fb2f", "jar of brown sugar-ddde96782a0abd30", "lid of sugar jar-371d975182a92a55", "lid of coffee jar-e5341ea5e919e1b0", "mug5-e68fd8aa2f632ba6", "bottle of dishwashing liquid-2152a55c290ad1ee", "electric kettle-4a1c2430fde81b5d", "milk frother-396a13b0bb9c8864", "stand of milk frother-d8640ec0fc0ff6a0", "plastic box-b3f2b2d40130eeb1", "another plastic box-924886b597cc2ac6", "another plastic box2-1b2f7c36e3313af3", "chopping board2-199392c7b2d821c7", "glass-8accf78b890dc922", "bowl4-295536e18f4dee4d", "spatula2-f6b7653f877a2f05", "butter knife2-cd28a7ebb40a3971", "spoon4-a69c5d162276f081", "butter knife3-023310228a51a42a", "tablecloth2-2b4da187831358f3", "sponge-2e4bbff4fb224ada", "sponge2-0545c267e65bd6d5", "spoon5-07fdb7a150641b40", "lid of ketchup bottle-9321836ac6ae2868", "lid of pressure cooker-dfa8a08d2c3832df", "wooden spoon-e99cb6c127e2b9bb", "pressure cooker-c0ae81c09633f6ba", "spoon6-a7ee870dd05ec00c", "spoon7-7bb703ff8228d628", "spatula3-fff665c3b13115c6" ], "reasoning": "1. I watch the video and find the first white bowl (bowl-1adf54e7494b619f) at 01:40.\n2. At 01:42, I see the user put the bowl (bowl-1adf54e7494b619f) on a kitchen countertop.\n3. From 12:55-12:58, I notice the user put the same bowl (bowl-1adf54e7494b619f) into the sink.\n4. From 34:18-34:35, the user washes the bowl (bowl-1adf54e7494b619f) and puts it onto a drying rack.\n5. After this, the bowl (bowl-1adf54e7494b619f) isn't moved.\n6. Therefore, the final position of the first white bowl (bowl-1adf54e7494b619f) is a drying rack.", "referenced_object_masks": [ "bowl-1adf54e7494b619f" ] }, { "key": "P06-20240511-102207:9c85a9b63a24fec30b7519b313835c6b66272add", "video_id": "P06-20240511-102207", "question": "When I grab the white bowl for the first time, where is the third object I took from the fridge and put on the counter in relation to me?", "answer_choice_0": "Behind and to the right", "answer_choice_1": "Directly behind", "answer_choice_2": "Behind and to the left", "answer_choice_3": "Directly ahead", "answer_choice_4": "Directly above", "answer_id": 0, "answer": "Behind and to the right", "question_type": [ "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counting", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "box-640d272cf8349df6", "box2-a9b0e10c2420cd6e", "box3-ea42b156f2ad559a", "carton of eggs-55be73789bc20b80", "pack of bread-f3a646eccced80d0", "milk bottle-bba3b2999c1daaec", "pack of butter-1c8ad2cb527706c4", "pen-0ec8edf8b9be4bdf", "notebook-aa7029dac4f75bd9", "weighing balance-aa98fc514dd3d318", "bowl-1adf54e7494b619f", "chopping board-971cefd7dca430b4", "onion-da6bc7290991b46c", "frying pan-1d1076c0ee9048c3", "knife-2f7108372eb0e0ed", "bowl2-50fec6214b51c091", "half onion-babd1ead2c43711c", "bowl3-e2e6d93ce91f7526", "egg-d2f8ae1fc6a46525", "egg2-7e12972ee992d944", "egg3-41dffe3fe003b3cb", "egg4-8c48fba2f5f37fcd", "cloth-604cc06f7bb797d1", "spoon-12cdc7b4e0b84262", "bottle of salt-6e1f787f9b19866e", "jar of spices with green lid-51d8167cdc8ff787", "green lid of jar-337755054caa5449", "spoon2-4373d15a83be3fe1", "one bread-43c1df3042dce609", "second bread-0d6ba7767da84772", "butter knife-e78e149815883f02", "measuring spoon-d13cc25ae512badf", "piece of butter-5c047e91c35c4fb8", "spatula-317d6d35610bf64e", "plate-fdf4ad1d588f85fc", "lid of box-6bf78bf8202064c8", "piece of pancake-ae26365748386fc6", "piece of onion-5fdecd10aadac956", "paper towel roll-eb876c03d8e18f69", "tablecloth-b9632cfe889b0e73", "mug-e2b33a685ee3a4dd", "mug2-4246bb59f3f9ca2e", "mug3-287f7c88aa72378b", "mug4-4575f761a634e5f7", "cover of milk bottle-51b0cd515664ab5c", "lid of milk carton-d862c5091443439f", "bread omelette-3c3617096e753b83", "plate2-80ec713704c97f03", "spoon3-dd29784d38c2efaf", "unused jar-7812e55e59c91b48", "bottle of ketchup-ba77c3b51f0094af", "coffee jar-0f9a1a19e2b8d431", "bottle of oil-a6e4315c20f9fb2f", "jar of brown sugar-ddde96782a0abd30", "lid of sugar jar-371d975182a92a55", "lid of coffee jar-e5341ea5e919e1b0", "mug5-e68fd8aa2f632ba6", "bottle of dishwashing liquid-2152a55c290ad1ee", "electric kettle-4a1c2430fde81b5d", "milk frother-396a13b0bb9c8864", "stand of milk frother-d8640ec0fc0ff6a0", "plastic box-b3f2b2d40130eeb1", "another plastic box-924886b597cc2ac6", "another plastic box2-1b2f7c36e3313af3", "chopping board2-199392c7b2d821c7", "glass-8accf78b890dc922", "bowl4-295536e18f4dee4d", "spatula2-f6b7653f877a2f05", "butter knife2-cd28a7ebb40a3971", "spoon4-a69c5d162276f081", "butter knife3-023310228a51a42a", "tablecloth2-2b4da187831358f3", "sponge-2e4bbff4fb224ada", "sponge2-0545c267e65bd6d5", "spoon5-07fdb7a150641b40", "lid of ketchup bottle-9321836ac6ae2868", "lid of pressure cooker-dfa8a08d2c3832df", "wooden spoon-e99cb6c127e2b9bb", "pressure cooker-c0ae81c09633f6ba", "spoon6-a7ee870dd05ec00c", "spoon7-7bb703ff8228d628", "spatula3-fff665c3b13115c6" ], "reasoning": "1. I observe the user open the fridge at 00:15 and close it at 00:57.\n2. By 01:00, I observe the user place onto the counter the following items they removed from the fridge : eggs (carton of eggs-55be73789bc20b80), box with blue lid (box2-a9b0e10c2420cd6e), and butter (pack of butter-1c8ad2cb527706c4).\n3. I determine the third object removed from the fridge is the butter (pack of butter-1c8ad2cb527706c4) placed to the right of the stove .\n4. I observe the user grab the white bowl (bowl-1adf54e7494b619f) at 01:40.\n5. I determine that the butter (pack of butter-1c8ad2cb527706c4) is behind the user on the counter to the right.", "referenced_object_masks": [ "carton of eggs-55be73789bc20b80", "bowl-1adf54e7494b619f", "pack of butter-1c8ad2cb527706c4", "box2-a9b0e10c2420cd6e" ] }, { "key": "P06-20240511-102207:b8cb2721ca271e55f5b661711a39b6d87e69ae1e", "video_id": "P06-20240511-102207", "question": "What is the mathematical difference between the number of items I brought to the kitchen counter after both my first and second visit to the refrigerator?", "answer_choice_0": "1", "answer_choice_1": "4", "answer_choice_2": "0", "answer_choice_3": "3", "answer_choice_4": "2", "answer_id": 0, "answer": "1", "question_type": [ "Situational Awareness", "Action Recognition", "Counting", "Temporal Reasoning", "State Changes", "Numerical Reasoning" ], "objects_of_interest": [ "box-640d272cf8349df6", "box2-a9b0e10c2420cd6e", "box3-ea42b156f2ad559a", "carton of eggs-55be73789bc20b80", "pack of bread-f3a646eccced80d0", "milk bottle-bba3b2999c1daaec", "pack of butter-1c8ad2cb527706c4", "pen-0ec8edf8b9be4bdf", "notebook-aa7029dac4f75bd9", "weighing balance-aa98fc514dd3d318", "bowl-1adf54e7494b619f", "chopping board-971cefd7dca430b4", "onion-da6bc7290991b46c", "frying pan-1d1076c0ee9048c3", "knife-2f7108372eb0e0ed", "bowl2-50fec6214b51c091", "half onion-babd1ead2c43711c", "bowl3-e2e6d93ce91f7526", "egg-d2f8ae1fc6a46525", "egg2-7e12972ee992d944", "egg3-41dffe3fe003b3cb", "egg4-8c48fba2f5f37fcd", "cloth-604cc06f7bb797d1", "spoon-12cdc7b4e0b84262", "bottle of salt-6e1f787f9b19866e", "jar of spices with green lid-51d8167cdc8ff787", "green lid of jar-337755054caa5449", "spoon2-4373d15a83be3fe1", "one bread-43c1df3042dce609", "second bread-0d6ba7767da84772", "butter knife-e78e149815883f02", "measuring spoon-d13cc25ae512badf", "piece of butter-5c047e91c35c4fb8", "spatula-317d6d35610bf64e", "plate-fdf4ad1d588f85fc", "lid of box-6bf78bf8202064c8", "piece of pancake-ae26365748386fc6", "piece of onion-5fdecd10aadac956", "paper towel roll-eb876c03d8e18f69", "tablecloth-b9632cfe889b0e73", "mug-e2b33a685ee3a4dd", "mug2-4246bb59f3f9ca2e", "mug3-287f7c88aa72378b", "mug4-4575f761a634e5f7", "cover of milk bottle-51b0cd515664ab5c", "lid of milk carton-d862c5091443439f", "bread omelette-3c3617096e753b83", "plate2-80ec713704c97f03", "spoon3-dd29784d38c2efaf", "unused jar-7812e55e59c91b48", "bottle of ketchup-ba77c3b51f0094af", "coffee jar-0f9a1a19e2b8d431", "bottle of oil-a6e4315c20f9fb2f", "jar of brown sugar-ddde96782a0abd30", "lid of sugar jar-371d975182a92a55", "lid of coffee jar-e5341ea5e919e1b0", "mug5-e68fd8aa2f632ba6", "bottle of dishwashing liquid-2152a55c290ad1ee", "electric kettle-4a1c2430fde81b5d", "milk frother-396a13b0bb9c8864", "stand of milk frother-d8640ec0fc0ff6a0", "plastic box-b3f2b2d40130eeb1", "another plastic box-924886b597cc2ac6", "another plastic box2-1b2f7c36e3313af3", "chopping board2-199392c7b2d821c7", "glass-8accf78b890dc922", "bowl4-295536e18f4dee4d", "spatula2-f6b7653f877a2f05", "butter knife2-cd28a7ebb40a3971", "spoon4-a69c5d162276f081", "butter knife3-023310228a51a42a", "tablecloth2-2b4da187831358f3", "sponge-2e4bbff4fb224ada", "sponge2-0545c267e65bd6d5", "spoon5-07fdb7a150641b40", "lid of ketchup bottle-9321836ac6ae2868", "lid of pressure cooker-dfa8a08d2c3832df", "wooden spoon-e99cb6c127e2b9bb", "pressure cooker-c0ae81c09633f6ba", "spoon6-a7ee870dd05ec00c", "spoon7-7bb703ff8228d628", "spatula3-fff665c3b13115c6" ], "reasoning": "1. I watched the video and found the kitchen counter at 00:01.\n2. I found the refrigerator at 00:15.\n3. I saw the user visit the refrigerator for the first time at 00:15-00:28.\n4. After the user visited the refrigerator for the first time at 00:15-00:28, I saw the user bring a number of items to the kitchen counter at 00:32-00:34.\n5. I counted the number of items the user brought to the kitchen counter at 00:32-00:34, calculating a total of 2, which were an egg carton (carton of eggs-55be73789bc20b80) and a blue tupperware (box2-a9b0e10c2420cd6e).\n6. I saw the user visit the refrigerator for the second time at 00:37-00:57.\n7. After the user visited the refrigerator for the second time at 00:37-00:57, I saw the user bring a number of items to the kitchen counter at 00:59-01:04.\n8. I counted the number of items the user brought to the kitchen counter at 00:59-01:04, calculating a total of 3, which were a pack of butter (pack of butter-1c8ad2cb527706c4), a packaged loaf of bread (pack of bread-f3a646eccced80d0), and a jug of milk (milk bottle-bba3b2999c1daaec).\n9. I calculated 3-2=1.\n10. Therefore, the mathematical difference between the number of items the user brought to the kitchen counter after both their first and second visit to the refrigerator is 1.", "referenced_object_masks": [ "carton of eggs-55be73789bc20b80", "pack of butter-1c8ad2cb527706c4", "pack of bread-f3a646eccced80d0", "milk bottle-bba3b2999c1daaec", "box2-a9b0e10c2420cd6e" ] }, { "key": "P06-20240511-102207:c752e70c67df8bd6126220d5239a91ae9ba24e8f", "video_id": "P06-20240511-102207", "question": "If I doubled the number of items I placed on the black kitchen countertop before grabbing the notebook, how many items would I have placed on the counter?", "answer_choice_0": "4", "answer_choice_1": "6", "answer_choice_2": "2", "answer_choice_3": "10", "answer_choice_4": "8", "answer_id": 3, "answer": "10", "question_type": [ "Situational Awareness", "Object Recognition", "Action Recognition", "Counting", "Temporal Reasoning", "Numerical Reasoning", "Counterfactual", "State Changes", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "box-640d272cf8349df6", "box2-a9b0e10c2420cd6e", "box3-ea42b156f2ad559a", "carton of eggs-55be73789bc20b80", "pack of bread-f3a646eccced80d0", "milk bottle-bba3b2999c1daaec", "pack of butter-1c8ad2cb527706c4", "pen-0ec8edf8b9be4bdf", "notebook-aa7029dac4f75bd9", "weighing balance-aa98fc514dd3d318", "bowl-1adf54e7494b619f", "chopping board-971cefd7dca430b4", "onion-da6bc7290991b46c", "frying pan-1d1076c0ee9048c3", "knife-2f7108372eb0e0ed", "bowl2-50fec6214b51c091", "half onion-babd1ead2c43711c", "bowl3-e2e6d93ce91f7526", "egg-d2f8ae1fc6a46525", "egg2-7e12972ee992d944", "egg3-41dffe3fe003b3cb", "egg4-8c48fba2f5f37fcd", "cloth-604cc06f7bb797d1", "spoon-12cdc7b4e0b84262", "bottle of salt-6e1f787f9b19866e", "jar of spices with green lid-51d8167cdc8ff787", "green lid of jar-337755054caa5449", "spoon2-4373d15a83be3fe1", "one bread-43c1df3042dce609", "second bread-0d6ba7767da84772", "butter knife-e78e149815883f02", "measuring spoon-d13cc25ae512badf", "piece of butter-5c047e91c35c4fb8", "spatula-317d6d35610bf64e", "plate-fdf4ad1d588f85fc", "lid of box-6bf78bf8202064c8", "piece of pancake-ae26365748386fc6", "piece of onion-5fdecd10aadac956", "paper towel roll-eb876c03d8e18f69", "tablecloth-b9632cfe889b0e73", "mug-e2b33a685ee3a4dd", "mug2-4246bb59f3f9ca2e", "mug3-287f7c88aa72378b", "mug4-4575f761a634e5f7", "cover of milk bottle-51b0cd515664ab5c", "lid of milk carton-d862c5091443439f", "bread omelette-3c3617096e753b83", "plate2-80ec713704c97f03", "spoon3-dd29784d38c2efaf", "unused jar-7812e55e59c91b48", "bottle of ketchup-ba77c3b51f0094af", "coffee jar-0f9a1a19e2b8d431", "bottle of oil-a6e4315c20f9fb2f", "jar of brown sugar-ddde96782a0abd30", "lid of sugar jar-371d975182a92a55", "lid of coffee jar-e5341ea5e919e1b0", "mug5-e68fd8aa2f632ba6", "bottle of dishwashing liquid-2152a55c290ad1ee", "electric kettle-4a1c2430fde81b5d", "milk frother-396a13b0bb9c8864", "stand of milk frother-d8640ec0fc0ff6a0", "plastic box-b3f2b2d40130eeb1", "another plastic box-924886b597cc2ac6", "another plastic box2-1b2f7c36e3313af3", "chopping board2-199392c7b2d821c7", "glass-8accf78b890dc922", "bowl4-295536e18f4dee4d", "spatula2-f6b7653f877a2f05", "butter knife2-cd28a7ebb40a3971", "spoon4-a69c5d162276f081", "butter knife3-023310228a51a42a", "tablecloth2-2b4da187831358f3", "sponge-2e4bbff4fb224ada", "sponge2-0545c267e65bd6d5", "spoon5-07fdb7a150641b40", "lid of ketchup bottle-9321836ac6ae2868", "lid of pressure cooker-dfa8a08d2c3832df", "wooden spoon-e99cb6c127e2b9bb", "pressure cooker-c0ae81c09633f6ba", "spoon6-a7ee870dd05ec00c", "spoon7-7bb703ff8228d628", "spatula3-fff665c3b13115c6" ], "reasoning": "1. I watched the video and found the black kitchen countertop at 00:01.\n2. I found the notebook (notebook-aa7029dac4f75bd9) on the black kitchen countertop at 00:02.\n3. I saw the user grab the notebook (notebook-aa7029dac4f75bd9) at 01:06.\n4. I counted each item the user placed on the black kitchen countertop before grabbing the notebook (notebook-aa7029dac4f75bd9) at 01:06.\n5. I saw the user place 2 items on the black kitchen countertop at 00:32-00:34, which were an egg carton (carton of eggs-55be73789bc20b80) and a blue tupperware (box2-a9b0e10c2420cd6e).\n6. I saw the user place 3 more items on the black kitchen countertop at 00:59-01:04, which were a pack of butter (pack of butter-1c8ad2cb527706c4), a packaged loaf of bread (pack of bread-f3a646eccced80d0), and a jug of milk (milk bottle-bba3b2999c1daaec).\n7. Since the user did not place anymore items on the black kitchen countertop after 01:04, I counted a total of 5 items the user placed on the black kitchen countertop before grabbing the notebook (notebook-aa7029dac4f75bd9) at 01:06.\n8. I calculated 5*2=10.\n9. Therefore, if the user doubled the number of items they placed on the black kitchen countertop before grabbing the notebook (notebook-aa7029dac4f75bd9), they would have placed 10 items on the counter.", "referenced_object_masks": [ "carton of eggs-55be73789bc20b80", "notebook-aa7029dac4f75bd9", "pack of butter-1c8ad2cb527706c4", "pack of bread-f3a646eccced80d0", "milk bottle-bba3b2999c1daaec", "box2-a9b0e10c2420cd6e" ] }, { "key": "P06-20240511-102207:d068c87a65bc68c995d9c481a33544dbd3ef2786", "video_id": "P06-20240511-102207", "question": "What would I find in the container directly behind the rightmost white bowl when I take a piece of bread from the orange bag?", "answer_choice_0": "Egg shells", "answer_choice_1": "Onion skins", "answer_choice_2": "An onion", "answer_choice_3": "Salt", "answer_choice_4": "An egg", "answer_id": 4, "answer": "An egg", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counterfactual" ], "objects_of_interest": [ "box-640d272cf8349df6", "box2-a9b0e10c2420cd6e", "box3-ea42b156f2ad559a", "carton of eggs-55be73789bc20b80", "pack of bread-f3a646eccced80d0", "milk bottle-bba3b2999c1daaec", "pack of butter-1c8ad2cb527706c4", "pen-0ec8edf8b9be4bdf", "notebook-aa7029dac4f75bd9", "weighing balance-aa98fc514dd3d318", "bowl-1adf54e7494b619f", "chopping board-971cefd7dca430b4", "onion-da6bc7290991b46c", "frying pan-1d1076c0ee9048c3", "knife-2f7108372eb0e0ed", "bowl2-50fec6214b51c091", "half onion-babd1ead2c43711c", "bowl3-e2e6d93ce91f7526", "egg-d2f8ae1fc6a46525", "egg2-7e12972ee992d944", "egg3-41dffe3fe003b3cb", "egg4-8c48fba2f5f37fcd", "cloth-604cc06f7bb797d1", "spoon-12cdc7b4e0b84262", "bottle of salt-6e1f787f9b19866e", "jar of spices with green lid-51d8167cdc8ff787", "green lid of jar-337755054caa5449", "spoon2-4373d15a83be3fe1", "one bread-43c1df3042dce609", "second bread-0d6ba7767da84772", "butter knife-e78e149815883f02", "measuring spoon-d13cc25ae512badf", "piece of butter-5c047e91c35c4fb8", "spatula-317d6d35610bf64e", "plate-fdf4ad1d588f85fc", "lid of box-6bf78bf8202064c8", "piece of pancake-ae26365748386fc6", "piece of onion-5fdecd10aadac956", "paper towel roll-eb876c03d8e18f69", "tablecloth-b9632cfe889b0e73", "mug-e2b33a685ee3a4dd", "mug2-4246bb59f3f9ca2e", "mug3-287f7c88aa72378b", "mug4-4575f761a634e5f7", "cover of milk bottle-51b0cd515664ab5c", "lid of milk carton-d862c5091443439f", "bread omelette-3c3617096e753b83", "plate2-80ec713704c97f03", "spoon3-dd29784d38c2efaf", "unused jar-7812e55e59c91b48", "bottle of ketchup-ba77c3b51f0094af", "coffee jar-0f9a1a19e2b8d431", "bottle of oil-a6e4315c20f9fb2f", "jar of brown sugar-ddde96782a0abd30", "lid of sugar jar-371d975182a92a55", "lid of coffee jar-e5341ea5e919e1b0", "mug5-e68fd8aa2f632ba6", "bottle of dishwashing liquid-2152a55c290ad1ee", "electric kettle-4a1c2430fde81b5d", "milk frother-396a13b0bb9c8864", "stand of milk frother-d8640ec0fc0ff6a0", "plastic box-b3f2b2d40130eeb1", "another plastic box-924886b597cc2ac6", "another plastic box2-1b2f7c36e3313af3", "chopping board2-199392c7b2d821c7", "glass-8accf78b890dc922", "bowl4-295536e18f4dee4d", "spatula2-f6b7653f877a2f05", "butter knife2-cd28a7ebb40a3971", "spoon4-a69c5d162276f081", "butter knife3-023310228a51a42a", "tablecloth2-2b4da187831358f3", "sponge-2e4bbff4fb224ada", "sponge2-0545c267e65bd6d5", "spoon5-07fdb7a150641b40", "lid of ketchup bottle-9321836ac6ae2868", "lid of pressure cooker-dfa8a08d2c3832df", "wooden spoon-e99cb6c127e2b9bb", "pressure cooker-c0ae81c09633f6ba", "spoon6-a7ee870dd05ec00c", "spoon7-7bb703ff8228d628", "spatula3-fff665c3b13115c6" ], "reasoning": "1. I observe the user take a slice of bread (one bread-43c1df3042dce609) from the bag of bread (pack of bread-f3a646eccced80d0) at 08:23.\n2. I locate the white bowl (bowl3-e2e6d93ce91f7526) behind the bag of bread (pack of bread-f3a646eccced80d0) in the user's hands.\n3. I identify the object behind the white bowl (bowl3-e2e6d93ce91f7526) as the carton of eggs (carton of eggs-55be73789bc20b80).\n4. I rewind to when the egg carton (carton of eggs-55be73789bc20b80) was last open at 08:08.\n5. I count 1 egg inside the egg carton (carton of eggs-55be73789bc20b80).\n6. I determine that the user would see an egg if they opened the carton of eggs (carton of eggs-55be73789bc20b80) at 08:23.", "referenced_object_masks": [ "carton of eggs-55be73789bc20b80", "one bread-43c1df3042dce609", "bowl3-e2e6d93ce91f7526", "pack of bread-f3a646eccced80d0" ] }, { "key": "P06-20240511-102207:fa2db261c71219e52f3089ad85f84e222eef9411", "video_id": "P06-20240511-102207", "question": "If I add the first ingredient I add to the egg yolks to the container I first placed onto the counter from the fridge, what would be in the container when I first place it?", "answer_choice_0": "Red Spice and 5 eggs", "answer_choice_1": "Bread and red spice", "answer_choice_2": "Salt and 6 eggs", "answer_choice_3": "Onions and salt", "answer_choice_4": "Salt and 5 eggs", "answer_id": 4, "answer": "Salt and 5 eggs", "question_type": [ "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counting", "Counterfactual" ], "objects_of_interest": [ "box-640d272cf8349df6", "box2-a9b0e10c2420cd6e", "box3-ea42b156f2ad559a", "carton of eggs-55be73789bc20b80", "pack of bread-f3a646eccced80d0", "milk bottle-bba3b2999c1daaec", "pack of butter-1c8ad2cb527706c4", "pen-0ec8edf8b9be4bdf", "notebook-aa7029dac4f75bd9", "weighing balance-aa98fc514dd3d318", "bowl-1adf54e7494b619f", "chopping board-971cefd7dca430b4", "onion-da6bc7290991b46c", "frying pan-1d1076c0ee9048c3", "knife-2f7108372eb0e0ed", "bowl2-50fec6214b51c091", "half onion-babd1ead2c43711c", "bowl3-e2e6d93ce91f7526", "egg-d2f8ae1fc6a46525", "egg2-7e12972ee992d944", "egg3-41dffe3fe003b3cb", "egg4-8c48fba2f5f37fcd", "cloth-604cc06f7bb797d1", "spoon-12cdc7b4e0b84262", "bottle of salt-6e1f787f9b19866e", "jar of spices with green lid-51d8167cdc8ff787", "green lid of jar-337755054caa5449", "spoon2-4373d15a83be3fe1", "one bread-43c1df3042dce609", "second bread-0d6ba7767da84772", "butter knife-e78e149815883f02", "measuring spoon-d13cc25ae512badf", "piece of butter-5c047e91c35c4fb8", "spatula-317d6d35610bf64e", "plate-fdf4ad1d588f85fc", "lid of box-6bf78bf8202064c8", "piece of pancake-ae26365748386fc6", "piece of onion-5fdecd10aadac956", "paper towel roll-eb876c03d8e18f69", "tablecloth-b9632cfe889b0e73", "mug-e2b33a685ee3a4dd", "mug2-4246bb59f3f9ca2e", "mug3-287f7c88aa72378b", "mug4-4575f761a634e5f7", "cover of milk bottle-51b0cd515664ab5c", "lid of milk carton-d862c5091443439f", "bread omelette-3c3617096e753b83", "plate2-80ec713704c97f03", "spoon3-dd29784d38c2efaf", "unused jar-7812e55e59c91b48", "bottle of ketchup-ba77c3b51f0094af", "coffee jar-0f9a1a19e2b8d431", "bottle of oil-a6e4315c20f9fb2f", "jar of brown sugar-ddde96782a0abd30", "lid of sugar jar-371d975182a92a55", "lid of coffee jar-e5341ea5e919e1b0", "mug5-e68fd8aa2f632ba6", "bottle of dishwashing liquid-2152a55c290ad1ee", "electric kettle-4a1c2430fde81b5d", "milk frother-396a13b0bb9c8864", "stand of milk frother-d8640ec0fc0ff6a0", "plastic box-b3f2b2d40130eeb1", "another plastic box-924886b597cc2ac6", "another plastic box2-1b2f7c36e3313af3", "chopping board2-199392c7b2d821c7", "glass-8accf78b890dc922", "bowl4-295536e18f4dee4d", "spatula2-f6b7653f877a2f05", "butter knife2-cd28a7ebb40a3971", "spoon4-a69c5d162276f081", "butter knife3-023310228a51a42a", "tablecloth2-2b4da187831358f3", "sponge-2e4bbff4fb224ada", "sponge2-0545c267e65bd6d5", "spoon5-07fdb7a150641b40", "lid of ketchup bottle-9321836ac6ae2868", "lid of pressure cooker-dfa8a08d2c3832df", "wooden spoon-e99cb6c127e2b9bb", "pressure cooker-c0ae81c09633f6ba", "spoon6-a7ee870dd05ec00c", "spoon7-7bb703ff8228d628", "spatula3-fff665c3b13115c6" ], "reasoning": "1. I observe the user add salt from a bottle (bottle of salt-6e1f787f9b19866e) to the bowl of egg yolks at 06:42.\n2. I rewind to observe the user open the fridge at 00:15 and close it at 00:57.\n3. At 00:33, I observe the user place the eggs (carton of eggs-55be73789bc20b80) onto the counter.\n4. At 01:57 I observe the user open the egg carton (carton of eggs-55be73789bc20b80) to reveal 5 eggs .\n5. I determine that the user would get salt and 5 eggs .", "referenced_object_masks": [ "carton of eggs-55be73789bc20b80", "bottle of salt-6e1f787f9b19866e" ] }, { "key": "P06-20240511-121048:332353c54199b880782170e9339dd948b64e9a63", "video_id": "P06-20240511-121048", "question": "If I multiplied the total number of dishes and cutlery I washed by 5, what number would I get?", "answer_choice_0": "35", "answer_choice_1": "20", "answer_choice_2": "30", "answer_choice_3": "15", "answer_choice_4": "25", "answer_id": 0, "answer": "35", "question_type": [ "Counterfactual", "Object Recognition", "Numerical Reasoning", "Counting" ], "objects_of_interest": [ "spatula-dd4af5995a5399d5", "knife-9a2f0efcadb0ba85", "wooden spoon-86772c312f3478b1", "spoon1-047c03e8645cf634", "spoon2-f2a83f7cb2384c8e", "spoon3-5c6ae592d2ac4470", "spoon4-e6808f2668bfbaa2", "butter knife-f34962fd9e1974b0", "lid of pressure cooker-064b3219870eaa50", "pressure cooker-a63621f91d1ac404", "glass-a2ab768dfe80a946", "bowl1-695de76cef1532bc", "mug1-99633d26fcb096da", "mug2-9783b4462e4c764a", "bag-dd91649013b9d8f2", "frying pan-4f814548b46c7748", "flask-ba0d73e55bd04fd2", "sponge1-328ca3b9ed1ccc0f", "sponge2-e4d2a2e53f4081fc", "cup-50e5ba79c0bc5034", "bottle of washing up liquid-006b3e4ecf111dfa", "bits of food-c351dd88b8716e71", "mug3-84c182270cf7b669", "mug4-71339f97f80cbab3", "plate1-b297c2be283eac7d", "plate2-a7314af035ebdf43", "plate3-d3be86233615c3cc", "plate4-ca9511e989abc44f", "spoon-f18400e30d4a5263", "tea towel-c592a30828532a3c", "spray bottle-bd912f54c7b79877", "tea towel2-56dc361ef0a024ed", "tea towel3-067a5504c02abb57", "frying pan2-47a12fab98b96e6a", "plug of sandwich maker-e7b3f9a5cdf5f068", "sandwich maker-314fca63c24f5356", "tea towel4-b5caeb4a77106a8a" ], "reasoning": "1. I watched the video and waited for the user to start washing the dishes and saw it happen for the first time at 02:03 when the user started washing a mug (mug3-84c182270cf7b669).\n2. I then observed the user washing another mug (mug4-71339f97f80cbab3) afterwards at 02:15.\n3. I then observed the user washing a plate (plate1-b297c2be283eac7d) at 02:28.\n4. I then observed the user washing a second plate (plate2-a7314af035ebdf43) at 02:38.\n5. I then observed the user washing a third plate (plate3-d3be86233615c3cc) at 02:53.\n6. I then observed the user washing a fourth plate (plate4-ca9511e989abc44f) at 03:09.\n7. I then observed the user washing a spoon (spoon-f18400e30d4a5263) at 03:20.\n8. Therefore, adding up the number of dishes washed between 02:03-03:20 comes to 7.\n9. 7x5=35", "referenced_object_masks": [ "mug4-71339f97f80cbab3", "plate4-ca9511e989abc44f", "spoon-f18400e30d4a5263", "plate2-a7314af035ebdf43", "plate1-b297c2be283eac7d", "mug3-84c182270cf7b669", "plate3-d3be86233615c3cc" ] }, { "key": "P06-20240511-121048:44b3bbe248c63679c15dc1529c5aa481d6702330", "video_id": "P06-20240511-121048", "question": "When I'm washing the dishes, what would be the relative cardinal direction of the artwork featuring limes in water?", "answer_choice_0": "West", "answer_choice_1": "Northeast", "answer_choice_2": "Southwest", "answer_choice_3": "South", "answer_choice_4": "East", "answer_id": 3, "answer": "South", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "spatula-dd4af5995a5399d5", "knife-9a2f0efcadb0ba85", "wooden spoon-86772c312f3478b1", "spoon1-047c03e8645cf634", "spoon2-f2a83f7cb2384c8e", "spoon3-5c6ae592d2ac4470", "spoon4-e6808f2668bfbaa2", "butter knife-f34962fd9e1974b0", "lid of pressure cooker-064b3219870eaa50", "pressure cooker-a63621f91d1ac404", "glass-a2ab768dfe80a946", "bowl1-695de76cef1532bc", "mug1-99633d26fcb096da", "mug2-9783b4462e4c764a", "bag-dd91649013b9d8f2", "frying pan-4f814548b46c7748", "flask-ba0d73e55bd04fd2", "sponge1-328ca3b9ed1ccc0f", "sponge2-e4d2a2e53f4081fc", "cup-50e5ba79c0bc5034", "bottle of washing up liquid-006b3e4ecf111dfa", "bits of food-c351dd88b8716e71", "mug3-84c182270cf7b669", "mug4-71339f97f80cbab3", "plate1-b297c2be283eac7d", "plate2-a7314af035ebdf43", "plate3-d3be86233615c3cc", "plate4-ca9511e989abc44f", "spoon-f18400e30d4a5263", "tea towel-c592a30828532a3c", "spray bottle-bd912f54c7b79877", "tea towel2-56dc361ef0a024ed", "tea towel3-067a5504c02abb57", "frying pan2-47a12fab98b96e6a", "plug of sandwich maker-e7b3f9a5cdf5f068", "sandwich maker-314fca63c24f5356", "tea towel4-b5caeb4a77106a8a" ], "reasoning": "1. I watched the video to see what position the user is in when they are washing the dishes and saw this position at 02:03.\n2. I then watched the video to find out where the artwork featuring \"limes in water\" is and found it on the wall at 05:34.\n3. The user makes a complete 180 degree turn from the sink when they begin to face the artwork of the limes in water, so the artwork would be to the south of them when they are washing the dishes.", "referenced_object_masks": [] }, { "key": "P06-20240511-121048:994cb3d58d3d245967ac354a03de2b4c4068b3f7", "video_id": "P06-20240511-121048", "question": "How many seconds did I approximately spend using the various towels around my kitchen?", "answer_choice_0": "65 seconds", "answer_choice_1": "33 seconds", "answer_choice_2": "97 seconds", "answer_choice_3": "25 seconds", "answer_choice_4": "121 seconds", "answer_id": 2, "answer": "97 seconds", "question_type": [ "Action Recognition", "Numerical Reasoning", "Temporal Reasoning", "Object Recognition" ], "objects_of_interest": [ "spatula-dd4af5995a5399d5", "knife-9a2f0efcadb0ba85", "wooden spoon-86772c312f3478b1", "spoon1-047c03e8645cf634", "spoon2-f2a83f7cb2384c8e", "spoon3-5c6ae592d2ac4470", "spoon4-e6808f2668bfbaa2", "butter knife-f34962fd9e1974b0", "lid of pressure cooker-064b3219870eaa50", "pressure cooker-a63621f91d1ac404", "glass-a2ab768dfe80a946", "bowl1-695de76cef1532bc", "mug1-99633d26fcb096da", "mug2-9783b4462e4c764a", "bag-dd91649013b9d8f2", "frying pan-4f814548b46c7748", "flask-ba0d73e55bd04fd2", "sponge1-328ca3b9ed1ccc0f", "sponge2-e4d2a2e53f4081fc", "cup-50e5ba79c0bc5034", "bottle of washing up liquid-006b3e4ecf111dfa", "bits of food-c351dd88b8716e71", "mug3-84c182270cf7b669", "mug4-71339f97f80cbab3", "plate1-b297c2be283eac7d", "plate2-a7314af035ebdf43", "plate3-d3be86233615c3cc", "plate4-ca9511e989abc44f", "spoon-f18400e30d4a5263", "tea towel-c592a30828532a3c", "spray bottle-bd912f54c7b79877", "tea towel2-56dc361ef0a024ed", "tea towel3-067a5504c02abb57", "frying pan2-47a12fab98b96e6a", "plug of sandwich maker-e7b3f9a5cdf5f068", "sandwich maker-314fca63c24f5356", "tea towel4-b5caeb4a77106a8a" ], "reasoning": "1. I watched the video and waited for the user to start using a towel and saw this happen for the first time between 05:04-05:26 when the user used a tea towel (tea towel-c592a30828532a3c) to clean up the area around the sink.\n2. I then saw the user use another towel (tea towel3-067a5504c02abb57) to dry their hands between 05:40-05:52.\n3. I then saw the user use the same towel (tea towel3-067a5504c02abb57) to dry their hands again and also dry the sandwich maker (sandwich maker-314fca63c24f5356) between 08:37-09:21\n4. I then saw the user grab the tea towel (tea towel-c592a30828532a3c) and position it so the sandwich maker (sandwich maker-314fca63c24f5356) can dry on top of it. This happened between 09:35-09:46.\n5. I then saw the user fold a third towel (tea towel2-56dc361ef0a024ed) between 10:20-10:28.\n6. Adding all of these seconds up comes to 97 seconds.", "referenced_object_masks": [ "sandwich maker-314fca63c24f5356", "tea towel-c592a30828532a3c", "tea towel2-56dc361ef0a024ed", "tea towel3-067a5504c02abb57" ] }, { "key": "P06-20240511-121048:c66c79f7950e23f98ace0ae118b6a361a73c0e22", "video_id": "P06-20240511-121048", "question": "When I throw out the trash, on how many of my sides are doorways visible?", "answer_choice_0": "2", "answer_choice_1": "4", "answer_choice_2": "5", "answer_choice_3": "1", "answer_choice_4": "3", "answer_id": 3, "answer": "1", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness", "Action Recognition" ], "objects_of_interest": [ "spatula-dd4af5995a5399d5", "knife-9a2f0efcadb0ba85", "wooden spoon-86772c312f3478b1", "spoon1-047c03e8645cf634", "spoon2-f2a83f7cb2384c8e", "spoon3-5c6ae592d2ac4470", "spoon4-e6808f2668bfbaa2", "butter knife-f34962fd9e1974b0", "lid of pressure cooker-064b3219870eaa50", "pressure cooker-a63621f91d1ac404", "glass-a2ab768dfe80a946", "bowl1-695de76cef1532bc", "mug1-99633d26fcb096da", "mug2-9783b4462e4c764a", "bag-dd91649013b9d8f2", "frying pan-4f814548b46c7748", "flask-ba0d73e55bd04fd2", "sponge1-328ca3b9ed1ccc0f", "sponge2-e4d2a2e53f4081fc", "cup-50e5ba79c0bc5034", "bottle of washing up liquid-006b3e4ecf111dfa", "bits of food-c351dd88b8716e71", "mug3-84c182270cf7b669", "mug4-71339f97f80cbab3", "plate1-b297c2be283eac7d", "plate2-a7314af035ebdf43", "plate3-d3be86233615c3cc", "plate4-ca9511e989abc44f", "spoon-f18400e30d4a5263", "tea towel-c592a30828532a3c", "spray bottle-bd912f54c7b79877", "tea towel2-56dc361ef0a024ed", "tea towel3-067a5504c02abb57", "frying pan2-47a12fab98b96e6a", "plug of sandwich maker-e7b3f9a5cdf5f068", "sandwich maker-314fca63c24f5356", "tea towel4-b5caeb4a77106a8a" ], "reasoning": "1. I watched the video to see when the user threw out their trash and found this moment at 01:54.\n2. On the approach to the trash can , at 01:52, the user walks through one doorway and I can see another one directly across from them.\n3. As the user bends down to throw out the trash from 01:52-01:54, I note that the doorways the user passed on the way to the trashcan pass out of view, and are no longer visible in the user's field of view.\n4. I conclude that 1 door is visible in the user's front left field of vision when they throw out the trash .", "referenced_object_masks": [] }, { "key": "P06-20240511-121048:d2080cd1235f968b4acf35b57eb3497aaeb9d25b", "video_id": "P06-20240511-121048", "question": "How many items from the drying rack did I store away?", "answer_choice_0": "17", "answer_choice_1": "11", "answer_choice_2": "13", "answer_choice_3": "15", "answer_choice_4": "20", "answer_id": 3, "answer": "15", "question_type": [ "Action Recognition", "Counting", "Object Recognition" ], "objects_of_interest": [ "spatula-dd4af5995a5399d5", "knife-9a2f0efcadb0ba85", "wooden spoon-86772c312f3478b1", "spoon1-047c03e8645cf634", "spoon2-f2a83f7cb2384c8e", "spoon3-5c6ae592d2ac4470", "spoon4-e6808f2668bfbaa2", "butter knife-f34962fd9e1974b0", "lid of pressure cooker-064b3219870eaa50", "pressure cooker-a63621f91d1ac404", "glass-a2ab768dfe80a946", "bowl1-695de76cef1532bc", "mug1-99633d26fcb096da", "mug2-9783b4462e4c764a", "bag-dd91649013b9d8f2", "frying pan-4f814548b46c7748", "flask-ba0d73e55bd04fd2", "sponge1-328ca3b9ed1ccc0f", "sponge2-e4d2a2e53f4081fc", "cup-50e5ba79c0bc5034", "bottle of washing up liquid-006b3e4ecf111dfa", "bits of food-c351dd88b8716e71", "mug3-84c182270cf7b669", "mug4-71339f97f80cbab3", "plate1-b297c2be283eac7d", "plate2-a7314af035ebdf43", "plate3-d3be86233615c3cc", "plate4-ca9511e989abc44f", "spoon-f18400e30d4a5263", "tea towel-c592a30828532a3c", "spray bottle-bd912f54c7b79877", "tea towel2-56dc361ef0a024ed", "tea towel3-067a5504c02abb57", "frying pan2-47a12fab98b96e6a", "plug of sandwich maker-e7b3f9a5cdf5f068", "sandwich maker-314fca63c24f5356", "tea towel4-b5caeb4a77106a8a" ], "reasoning": "1. I watched the video and identified the drying rack at 00:06. The user starts taking items from the rack to store them away at 00:17, beginning with a wooden spoon (wooden spoon-86772c312f3478b1).\n2. I then observed the user put away a spatula (spatula-dd4af5995a5399d5) at 00:19.\n3. I then observed the user put away a knife (knife-9a2f0efcadb0ba85) at 00:21.\n4. I then observed the user put away a group of silverware in quick succession, which consisted of 4 spoons (spoon1-047c03e8645cf634, spoon2-f2a83f7cb2384c8e, spoon3-5c6ae592d2ac4470, spoon4-e6808f2668bfbaa2) and 1 knife (butter knife-f34962fd9e1974b0) between 00:23-00:32.\n5. I then observed the user putting away two items at once, a pot (pressure cooker-a63621f91d1ac404) and a lid (lid of pressure cooker-064b3219870eaa50), between 00:38-00:53.\n6. I then observed the user putting away a bowl (bowl1-695de76cef1532bc) at 01:04.\n7. I then observed the user put away a group of cups in quick succession, which consisted of 2 mugs (mug1-99633d26fcb096da, mug2-9783b4462e4c764a) and 1 glass (glass-a2ab768dfe80a946) between 01:07-01:16.\n8. I then observed the user put away a pan (frying pan-4f814548b46c7748) at 01:33.\n9. The user does not store any more items from the drying rack after the pan (frying pan-4f814548b46c7748).Therefore, by adding up the number of dishes that were put away by the user between 00:17-01:33, I arrived at 15.", "referenced_object_masks": [ "knife-9a2f0efcadb0ba85", "glass-a2ab768dfe80a946", "lid of pressure cooker-064b3219870eaa50", "wooden spoon-86772c312f3478b1", "frying pan-4f814548b46c7748", "spoon1-047c03e8645cf634", "spoon2-f2a83f7cb2384c8e", "spoon3-5c6ae592d2ac4470", "spoon4-e6808f2668bfbaa2", "butter knife-f34962fd9e1974b0", "spatula-dd4af5995a5399d5", "mug1-99633d26fcb096da", "mug2-9783b4462e4c764a", "bowl1-695de76cef1532bc", "pressure cooker-a63621f91d1ac404" ] }, { "key": "P06-20240511-122221:24448d014d18b4971ce116bdc7053a398f85200d", "video_id": "P06-20240511-122221", "question": "Where is the second place that I put the dishtowel once I am done drying dishes?", "answer_choice_0": "In the drawer", "answer_choice_1": "On the counter", "answer_choice_2": "On the dishrack", "answer_choice_3": "On the drawer pull", "answer_choice_4": "On my shoulder", "answer_id": 3, "answer": "On the drawer pull", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Counting", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "mixer jar-f078c81b3696a4ef", "lid of mixer jar-e706117899cd4a57", "bag-0039e1cb50c1ae5c", "plant pot-6508d949a04b9d40", "plastic box-d52859778ca3916f", "lid of plastic box-6c8c33bcd9bcf56a", "tea towel-560aac366cfde0fb", "plate-ed7e5d43e2940c7a", "plate2-4accaa334655f4f6", "plate3-b2bc75c000f38989", "plate4-ed3b2b2101a0ce9b", "stack of bowls-f157eb7e4ece905a", "stack of plates-c3f70b7662f8b247", "stack of bowls2-8b28a4b76834f9ae", "stack of bowls3-69c28c190fc8a6e0", "mug1-58724dcab82aa4be", "mug2-f36d0ad494655075", "sandwich maker-b706be3cf0ff6055" ], "reasoning": "1. I watched for the user to finish drying dishes, which happens at 01:40.\n2. I looked to see where the user first puts the dishtowel (tea towel-560aac366cfde0fb) next, and they toss it onto the counter at 01:41.\n3. I watched for the user to pick up the dishtowel (tea towel-560aac366cfde0fb) again, which they do at 01:47.\n4. The second place that the user puts the dishtowel (tea towel-560aac366cfde0fb) is on the drawer pull at 01:49.", "referenced_object_masks": [ "tea towel-560aac366cfde0fb" ] }, { "key": "P06-20240511-122221:66b8f209ee35f4fb55f94b308f3dbf6a4eec4170", "video_id": "P06-20240511-122221", "question": "If the plates I dried had been mugs, how many mugs would I have dried in total?", "answer_choice_0": "5", "answer_choice_1": "7", "answer_choice_2": "3", "answer_choice_3": "1", "answer_choice_4": "6", "answer_id": 4, "answer": "6", "question_type": [ "Counting", "Action Recognition", "Object Recognition", "Counterfactual" ], "objects_of_interest": [ "mixer jar-f078c81b3696a4ef", "lid of mixer jar-e706117899cd4a57", "bag-0039e1cb50c1ae5c", "plant pot-6508d949a04b9d40", "plastic box-d52859778ca3916f", "lid of plastic box-6c8c33bcd9bcf56a", "tea towel-560aac366cfde0fb", "plate-ed7e5d43e2940c7a", "plate2-4accaa334655f4f6", "plate3-b2bc75c000f38989", "plate4-ed3b2b2101a0ce9b", "stack of bowls-f157eb7e4ece905a", "stack of plates-c3f70b7662f8b247", "stack of bowls2-8b28a4b76834f9ae", "stack of bowls3-69c28c190fc8a6e0", "mug1-58724dcab82aa4be", "mug2-f36d0ad494655075", "sandwich maker-b706be3cf0ff6055" ], "reasoning": "1. I watched to see how many plates the user dries, and they dry 4 plates [(plate-ed7e5d43e2940c7a), (plate2-4accaa334655f4f6), (plate3-b2bc75c000f38989), and (plate4-ed3b2b2101a0ce9b)] between 00:39 and 01:11.\n2. I then watched to see how many mugs the user dries, and they dry 2 mugs [(mug1-58724dcab82aa4be) and (mug2-f36d0ad494655075)] between 01:27 and 01:40.\n3. If the dried plates [(plate-ed7e5d43e2940c7a), (plate2-4accaa334655f4f6), (plate3-b2bc75c000f38989), and (plate4-ed3b2b2101a0ce9b)] were added to the dried mugs [(mug1-58724dcab82aa4be) and (mug2-f36d0ad494655075)] there would be 6 dried mugs.", "referenced_object_masks": [ "plate3-b2bc75c000f38989", "plate2-4accaa334655f4f6", "plate-ed7e5d43e2940c7a", "mug1-58724dcab82aa4be", "mug2-f36d0ad494655075", "plate4-ed3b2b2101a0ce9b" ] }, { "key": "P06-20240511-122221:af1ba4060941623378a87dc489ffd289b395d723", "video_id": "P06-20240511-122221", "question": "What is the ratio of plants in opaque pots on the windowsill to potted plants on the counter?", "answer_choice_0": "1:2", "answer_choice_1": "1:3", "answer_choice_2": "3:1", "answer_choice_3": "3:4", "answer_choice_4": "1:1", "answer_id": 2, "answer": "3:1", "question_type": [ "Counting", "Object Recognition", "Numerical Reasoning" ], "objects_of_interest": [ "mixer jar-f078c81b3696a4ef", "lid of mixer jar-e706117899cd4a57", "bag-0039e1cb50c1ae5c", "plant pot-6508d949a04b9d40", "plastic box-d52859778ca3916f", "lid of plastic box-6c8c33bcd9bcf56a", "tea towel-560aac366cfde0fb", "plate-ed7e5d43e2940c7a", "plate2-4accaa334655f4f6", "plate3-b2bc75c000f38989", "plate4-ed3b2b2101a0ce9b", "stack of bowls-f157eb7e4ece905a", "stack of plates-c3f70b7662f8b247", "stack of bowls2-8b28a4b76834f9ae", "stack of bowls3-69c28c190fc8a6e0", "mug1-58724dcab82aa4be", "mug2-f36d0ad494655075", "sandwich maker-b706be3cf0ff6055" ], "reasoning": "1. First, I looked to identify the plants on the windowsill. I saw a clear view of them at 00:31, and noticed that there were 3 plants in opaque pots and 1 plant , which I did not count, in a clear glass . \n2. Then, I looked to see how many potted plants are visible on the counter, and saw 1 potted plant (plant pot-6508d949a04b9d40) clearly at 00:12.\n3. Therefore, the ratio is 3:1.", "referenced_object_masks": [ "plant pot-6508d949a04b9d40" ] }, { "key": "P06-20240511-122221:c921456db978e17d1a5929bafccf8035588d4c5a", "video_id": "P06-20240511-122221", "question": "How many more things do I dry than get wet?", "answer_choice_0": "3", "answer_choice_1": "6", "answer_choice_2": "1", "answer_choice_3": "0", "answer_choice_4": "2", "answer_id": 1, "answer": "6", "question_type": [ "Action Recognition", "Object Recognition", "Counting", "Numerical Reasoning" ], "objects_of_interest": [ "mixer jar-f078c81b3696a4ef", "lid of mixer jar-e706117899cd4a57", "bag-0039e1cb50c1ae5c", "plant pot-6508d949a04b9d40", "plastic box-d52859778ca3916f", "lid of plastic box-6c8c33bcd9bcf56a", "tea towel-560aac366cfde0fb", "plate-ed7e5d43e2940c7a", "plate2-4accaa334655f4f6", "plate3-b2bc75c000f38989", "plate4-ed3b2b2101a0ce9b", "stack of bowls-f157eb7e4ece905a", "stack of plates-c3f70b7662f8b247", "stack of bowls2-8b28a4b76834f9ae", "stack of bowls3-69c28c190fc8a6e0", "mug1-58724dcab82aa4be", "mug2-f36d0ad494655075", "sandwich maker-b706be3cf0ff6055" ], "reasoning": "1. I watched to see how many things the user dried.\n2. The user dries plates at 00:38 (plate-ed7e5d43e2940c7a), 00:48 (plate2-4accaa334655f4f6), 00:58 (plate3-b2bc75c000f38989), and 01:07 (plate4-ed3b2b2101a0ce9b).\n3. The user also dries mugs at 01:28 (mug1-58724dcab82aa4be) and 01:35 (mug2-f36d0ad494655075).\n4. The user then shakes the sandwich maker (sandwich maker-b706be3cf0ff6055) dry at 01:53.\n5. Conversely, the user only gets 1 thing wet, which is the potted plant (plant pot-6508d949a04b9d40) that water is added to at 00:18.\n6. The user gets 7 things dry and 1 thing wet, for a difference of 6.", "referenced_object_masks": [ "plate3-b2bc75c000f38989", "plate2-4accaa334655f4f6", "plate-ed7e5d43e2940c7a", "mug1-58724dcab82aa4be", "mug2-f36d0ad494655075", "sandwich maker-b706be3cf0ff6055", "plate4-ed3b2b2101a0ce9b", "plant pot-6508d949a04b9d40" ] }, { "key": "P06-20240511-122221:e97479975ca760c8693a084e9d08a56e843ca275", "video_id": "P06-20240511-122221", "question": "Why don't I put the plastic box and lid on the upper shelf in the cabinet?", "answer_choice_0": "There is only one shelf", "answer_choice_1": "There isn't room", "answer_choice_2": "I'm taking them off of the shelf", "answer_choice_3": "They aren't dry", "answer_choice_4": "It's where the bowls go", "answer_id": 1, "answer": "There isn't room", "question_type": [ "Egocentric & Spatial Perception", "Action Recognition", "Counterfactual", "Cause and Effect", "Object Recognition", "Situational Awareness", "Goal Reasoning" ], "objects_of_interest": [ "mixer jar-f078c81b3696a4ef", "lid of mixer jar-e706117899cd4a57", "bag-0039e1cb50c1ae5c", "plant pot-6508d949a04b9d40", "plastic box-d52859778ca3916f", "lid of plastic box-6c8c33bcd9bcf56a", "tea towel-560aac366cfde0fb", "plate-ed7e5d43e2940c7a", "plate2-4accaa334655f4f6", "plate3-b2bc75c000f38989", "plate4-ed3b2b2101a0ce9b", "stack of bowls-f157eb7e4ece905a", "stack of plates-c3f70b7662f8b247", "stack of bowls2-8b28a4b76834f9ae", "stack of bowls3-69c28c190fc8a6e0", "mug1-58724dcab82aa4be", "mug2-f36d0ad494655075", "sandwich maker-b706be3cf0ff6055" ], "reasoning": "1. I watched to see what the user does with the plastic box (plastic box-d52859778ca3916f) and plastic lid (lid of plastic box-6c8c33bcd9bcf56a).\n2. The user picks up the plastic box (plastic box-d52859778ca3916f) and plastic lid (lid of plastic box-6c8c33bcd9bcf56a) at 00:24 and 00:25 respectively. The user then walks to their left and opens a lower cabinet at 00:27.\n3. I saw the user initially move to put the plastic box (plastic box-d52859778ca3916f) and plastic lid (lid of plastic box-6c8c33bcd9bcf56a) on the upper shelf at 00:28, but there isn't room for it, and they put them on the lower shelf instead.", "referenced_object_masks": [ "lid of plastic box-6c8c33bcd9bcf56a", "plastic box-d52859778ca3916f" ] }, { "key": "P07-20240529-102652:0d2c9e897db10baa2c2fc9a93927a841588bc65f", "video_id": "P07-20240529-102652", "question": "If you multiplied the number of mugs that I removed from the dishwasher the first time I removed them by the number of scoops of coffee that I added to the portafilter, what would the result be?", "answer_choice_0": "15", "answer_choice_1": "10", "answer_choice_2": "6", "answer_choice_3": "12", "answer_choice_4": "18", "answer_id": 4, "answer": "18", "question_type": [ "Action Recognition", "Numerical Reasoning", "Object Recognition", "Counting" ], "objects_of_interest": [ "bowl-7deff761870a158e", "mug-0ecc7dc3447ba548", "mug2-7daa9bd68ab0f9b2", "mug3-5179d5332f77444b", "mug4-accfd108d135ea96", "mug5-3c5a3b0ce0c17e1c", "mug6-4b8f9eabee67e377", "tongs-4a970b6754c38337", "bowl2-d02c1f20415c5e91", "bowl3-e56880be3d689b74", "tupperware container-0c0e53e6ca1f0934", "tupperware container lid-154c841f449ed934", "small grey bowl-abc12621c35b841b", "metal ring-78dde7195603f324", "mug7-aea95e5929f6a9bb", "spatula-f0ec3848fa7269bc", "cup-dd74e19e46620464", "drinking glass-0091421d7c87e0f0", "lids-e15a563f4191aa67", "plate-aa46127bcb6922e7", "three plates-52934356246712f4", "another plate-dbf0b7f816901a91", "plate2-de17a8c463e9b644", "frying pan-f2d392d1af498a43", "saucepan-36d6d99f081e767c", "glass-4e7f72166ebccd90", "glass2-dc2f54bb1aeebd32", "mug8-bc76814539ac11a6", "knife-cc1da5126bdaedb9", "knife2-7d35f8ae9c5629ef", "small wire brush-c1bb1fdc4f4e564f", "bowl4-653728d18f001a05", "bowl5-2764a117e804f1b8", "cutlery-33db068339ba7a75", "cutlery2-065ac73e435b73fd", "spoon-5f8f4c6e8744d0ac", "spoon2-f5fadd1c1d3a55fa", "spoon3-ae1796504e10313d", "spoon4-12ccd3fe415424e1", "more cutlery-d22ed6f87f644be3", "spoon5-c63e015aa7229d37", "bowl6-004f3e5d08b101c7", "spoon6-b38286d84085d6f9", "two mugs-cbfa6799ec5addab", "yellow mug-8dc29969040ce054", "glass3-2ae0e6716c4d471e", "glass4-4b8d06b314384387", "saucepan lid-c30eeca48fdf0086", "fork-9ba3d12b02a92b55", "teaspoon-1b4679763269a37a", "three spoons-86e8c45c7e8dec1e", "tablespoon3-f5306b5b98bbab9e", "second bowl-5b6ca123b8bceb58", "third bowl-669fca83fb96fd68", "cloth-6b1d73fadb02d0ec", "dish brush-ec57da4f863920ba", "drain cover-13a8b3a44509883a", "lid of food waste bin-221e5c9d812df91b", "bottle of dishwashing liquid-282c90601f1c5f42", "drain cover2-f279789c441a5084", "frying pan2-586500a422b272d4", "dirty saucepan-3d78070aad67ba5d", "lid-02d18babe87d7aca", "large white jug of dishwashing liquid-2e54bce4d331713e", "lid2-8f369368ceaade18", "orange sponge-882c1f24c7a2e305", "scourer-aaab02140eb6207d", "tea towel-d18b5c8120e0871b", "chair-b2c89e4ad0cd1a67", "work surface-fa26c9ad53962227", "piece of ceramic-1fdf6c31e5174a26", "mug9-3acac054623baed7", "clock-1dfb73cef062d8e1", "work surface2-9731a8bdcbf4eeed", "box-4e05e69bc82c0bb3", "coffee machine-4a539fa6446aa4b4", "portafilter-be99f9411d680045", "teaspoon3-a463b6f92df06daf", "drawer-026b57c0033788df", "kettle base-815bc99542489ad6", "salt-c521370ec81ae21d", "pepper shakers-3209c23f0ee4e97f" ], "reasoning": "1. I watched the video and looked for when the user first removes mugs from the dishwasher . I found that this occurs at 00:13.\n2. I watched carefully to see how many mugs they placed into their hands. I see them place 4 mugs into their left hand (mug-0ecc7dc3447ba548, mug2-7daa9bd68ab0f9b2, mug3-5179d5332f77444b, mug4-accfd108d135ea96) and 2 into her right hand (mug5-3c5a3b0ce0c17e1c, mug6-4b8f9eabee67e377). At 00:23, they opened a cupboard and put them inside of it, confirming the number of 6.\n3. Then, I looked for when the user scooped coffee into the portafilter (portafilter-be99f9411d680045). I found this at 13:50. I watched carefully to see the user scoop 3 times.\n4. I took 6*3 to get a final total of 18.", "referenced_object_masks": [ "mug-0ecc7dc3447ba548", "mug2-7daa9bd68ab0f9b2", "mug3-5179d5332f77444b", "mug4-accfd108d135ea96", "portafilter-be99f9411d680045", "mug5-3c5a3b0ce0c17e1c", "mug6-4b8f9eabee67e377" ] }, { "key": "P07-20240529-102652:1b27b73135faad2d4fc71ef1a6727b9ba2ac0fc2", "video_id": "P07-20240529-102652", "question": "If you subtracted the amount of time it took me to unload the dishwasher by the amount of time it took me to load the dishwasher, what would the resulting total amount of time be?", "answer_choice_0": "2 minutes and 3 seconds", "answer_choice_1": "2 minutes and 22 seconds", "answer_choice_2": "2 minutes and 45 seconds", "answer_choice_3": "1 minute and 57 seconds", "answer_choice_4": "1 minute and 36 seconds", "answer_id": 3, "answer": "1 minute and 57 seconds", "question_type": [ "Temporal Reasoning", "Action Recognition", "Numerical Reasoning", "Object Recognition" ], "objects_of_interest": [ "bowl-7deff761870a158e", "mug-0ecc7dc3447ba548", "mug2-7daa9bd68ab0f9b2", "mug3-5179d5332f77444b", "mug4-accfd108d135ea96", "mug5-3c5a3b0ce0c17e1c", "mug6-4b8f9eabee67e377", "tongs-4a970b6754c38337", "bowl2-d02c1f20415c5e91", "bowl3-e56880be3d689b74", "tupperware container-0c0e53e6ca1f0934", "tupperware container lid-154c841f449ed934", "small grey bowl-abc12621c35b841b", "metal ring-78dde7195603f324", "mug7-aea95e5929f6a9bb", "spatula-f0ec3848fa7269bc", "cup-dd74e19e46620464", "drinking glass-0091421d7c87e0f0", "lids-e15a563f4191aa67", "plate-aa46127bcb6922e7", "three plates-52934356246712f4", "another plate-dbf0b7f816901a91", "plate2-de17a8c463e9b644", "frying pan-f2d392d1af498a43", "saucepan-36d6d99f081e767c", "glass-4e7f72166ebccd90", "glass2-dc2f54bb1aeebd32", "mug8-bc76814539ac11a6", "knife-cc1da5126bdaedb9", "knife2-7d35f8ae9c5629ef", "small wire brush-c1bb1fdc4f4e564f", "bowl4-653728d18f001a05", "bowl5-2764a117e804f1b8", "cutlery-33db068339ba7a75", "cutlery2-065ac73e435b73fd", "spoon-5f8f4c6e8744d0ac", "spoon2-f5fadd1c1d3a55fa", "spoon3-ae1796504e10313d", "spoon4-12ccd3fe415424e1", "more cutlery-d22ed6f87f644be3", "spoon5-c63e015aa7229d37", "bowl6-004f3e5d08b101c7", "spoon6-b38286d84085d6f9", "two mugs-cbfa6799ec5addab", "yellow mug-8dc29969040ce054", "glass3-2ae0e6716c4d471e", "glass4-4b8d06b314384387", "saucepan lid-c30eeca48fdf0086", "fork-9ba3d12b02a92b55", "teaspoon-1b4679763269a37a", "three spoons-86e8c45c7e8dec1e", "tablespoon3-f5306b5b98bbab9e", "second bowl-5b6ca123b8bceb58", "third bowl-669fca83fb96fd68", "cloth-6b1d73fadb02d0ec", "dish brush-ec57da4f863920ba", "drain cover-13a8b3a44509883a", "lid of food waste bin-221e5c9d812df91b", "bottle of dishwashing liquid-282c90601f1c5f42", "drain cover2-f279789c441a5084", "frying pan2-586500a422b272d4", "dirty saucepan-3d78070aad67ba5d", "lid-02d18babe87d7aca", "large white jug of dishwashing liquid-2e54bce4d331713e", "lid2-8f369368ceaade18", "orange sponge-882c1f24c7a2e305", "scourer-aaab02140eb6207d", "tea towel-d18b5c8120e0871b", "chair-b2c89e4ad0cd1a67", "work surface-fa26c9ad53962227", "piece of ceramic-1fdf6c31e5174a26", "mug9-3acac054623baed7", "clock-1dfb73cef062d8e1", "work surface2-9731a8bdcbf4eeed", "box-4e05e69bc82c0bb3", "coffee machine-4a539fa6446aa4b4", "portafilter-be99f9411d680045", "teaspoon3-a463b6f92df06daf", "drawer-026b57c0033788df", "kettle base-815bc99542489ad6", "salt-c521370ec81ae21d", "pepper shakers-3209c23f0ee4e97f" ], "reasoning": "1. I watched the video to find the amount of time it took the user to unload the dishwasher . First, I found that the user started unloading the dishwasher at 00:13, when they took the first items out of it, starting with a mug (mug-0ecc7dc3447ba548).\n2. I found that the user finished unloading the dishwasher at 03:08, when they pulled out the final piece of silverware (more cutlery-d22ed6f87f644be3).\n3. 03:08-00:13=02:55, or 2 minutes and 55 seconds in duration.\n4. I looked for when the user started to load the dishwasher . I found this started at 03:12, when they placed a bowl (bowl6-004f3e5d08b101c7) into the basin.\n5. At 04:10, the user finished loading the dishwasher as they closed the door.\n6. 04:10-03:12=00:58, or 58 seconds in duration.\n7. To get the final total, I subtracted 58 seconds from 2 minutes and 55 seconds. I got a final total of 1 minute and 57 seconds.", "referenced_object_masks": [ "more cutlery-d22ed6f87f644be3", "mug-0ecc7dc3447ba548", "bowl6-004f3e5d08b101c7" ] }, { "key": "P07-20240529-102652:27028d24141416613719562c533dec5112571129", "video_id": "P07-20240529-102652", "question": "After I empty out the black pot into the food bin, what is the next item I grab with my left hand?", "answer_choice_0": "The tupperware container", "answer_choice_1": "The drain cover", "answer_choice_2": "The dishwashing liquid bottle", "answer_choice_3": "The frying pan", "answer_choice_4": "The scourer", "answer_id": 0, "answer": "The tupperware container", "question_type": [ "Egocentric & Spatial Perception", "Action Recognition", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "bowl-7deff761870a158e", "mug-0ecc7dc3447ba548", "mug2-7daa9bd68ab0f9b2", "mug3-5179d5332f77444b", "mug4-accfd108d135ea96", "mug5-3c5a3b0ce0c17e1c", "mug6-4b8f9eabee67e377", "tongs-4a970b6754c38337", "bowl2-d02c1f20415c5e91", "bowl3-e56880be3d689b74", "tupperware container-0c0e53e6ca1f0934", "tupperware container lid-154c841f449ed934", "small grey bowl-abc12621c35b841b", "metal ring-78dde7195603f324", "mug7-aea95e5929f6a9bb", "spatula-f0ec3848fa7269bc", "cup-dd74e19e46620464", "drinking glass-0091421d7c87e0f0", "lids-e15a563f4191aa67", "plate-aa46127bcb6922e7", "three plates-52934356246712f4", "another plate-dbf0b7f816901a91", "plate2-de17a8c463e9b644", "frying pan-f2d392d1af498a43", "saucepan-36d6d99f081e767c", "glass-4e7f72166ebccd90", "glass2-dc2f54bb1aeebd32", "mug8-bc76814539ac11a6", "knife-cc1da5126bdaedb9", "knife2-7d35f8ae9c5629ef", "small wire brush-c1bb1fdc4f4e564f", "bowl4-653728d18f001a05", "bowl5-2764a117e804f1b8", "cutlery-33db068339ba7a75", "cutlery2-065ac73e435b73fd", "spoon-5f8f4c6e8744d0ac", "spoon2-f5fadd1c1d3a55fa", "spoon3-ae1796504e10313d", "spoon4-12ccd3fe415424e1", "more cutlery-d22ed6f87f644be3", "spoon5-c63e015aa7229d37", "bowl6-004f3e5d08b101c7", "spoon6-b38286d84085d6f9", "two mugs-cbfa6799ec5addab", "yellow mug-8dc29969040ce054", "glass3-2ae0e6716c4d471e", "glass4-4b8d06b314384387", "saucepan lid-c30eeca48fdf0086", "fork-9ba3d12b02a92b55", "teaspoon-1b4679763269a37a", "three spoons-86e8c45c7e8dec1e", "tablespoon3-f5306b5b98bbab9e", "second bowl-5b6ca123b8bceb58", "third bowl-669fca83fb96fd68", "cloth-6b1d73fadb02d0ec", "dish brush-ec57da4f863920ba", "drain cover-13a8b3a44509883a", "lid of food waste bin-221e5c9d812df91b", "bottle of dishwashing liquid-282c90601f1c5f42", "drain cover2-f279789c441a5084", "frying pan2-586500a422b272d4", "dirty saucepan-3d78070aad67ba5d", "lid-02d18babe87d7aca", "large white jug of dishwashing liquid-2e54bce4d331713e", "lid2-8f369368ceaade18", "orange sponge-882c1f24c7a2e305", "scourer-aaab02140eb6207d", "tea towel-d18b5c8120e0871b", "chair-b2c89e4ad0cd1a67", "work surface-fa26c9ad53962227", "piece of ceramic-1fdf6c31e5174a26", "mug9-3acac054623baed7", "clock-1dfb73cef062d8e1", "work surface2-9731a8bdcbf4eeed", "box-4e05e69bc82c0bb3", "coffee machine-4a539fa6446aa4b4", "portafilter-be99f9411d680045", "teaspoon3-a463b6f92df06daf", "drawer-026b57c0033788df", "kettle base-815bc99542489ad6", "salt-c521370ec81ae21d", "pepper shakers-3209c23f0ee4e97f" ], "reasoning": "1. I watched the video to see when the user emptied out the black pot (dirty saucepan-3d78070aad67ba5d) into the food bin (lid of food waste bin-221e5c9d812df91b). I found this moment from 04:49-04:52.\n2. I watched carefully to see what the next object was that the user grabbed with their left hand. First, I saw them place the black pot (dirty saucepan-3d78070aad67ba5d) inside the sink at 04:56 as they turned the water on with their other hand.\n3. At 05:04, I noticed that the user, after holding their left hand aloft for a moment, grabbed a piece of plastic tupperware (tupperware container-0c0e53e6ca1f0934). I marked this as the next item that they grabbed with their left hand.", "referenced_object_masks": [ "tupperware container-0c0e53e6ca1f0934", "dirty saucepan-3d78070aad67ba5d", "lid of food waste bin-221e5c9d812df91b" ] }, { "key": "P07-20240529-102652:845d0f71a583611318f05372062f2e2aba308e22", "video_id": "P07-20240529-102652", "question": "If I'm standing in front of the pots and pan drawer, and the first frying pan I handle is on the stove, in which direction do I have to move if I want to grab the pan and put it in the sink?", "answer_choice_0": "Rotate right 90 degrees, then step back", "answer_choice_1": "Walk right, then turn 90 degrees", "answer_choice_2": "Turn 90 degrees left and walk forward", "answer_choice_3": "Turn 180 degrees, then walk right", "answer_choice_4": "Step back and walk left", "answer_id": 3, "answer": "Turn 180 degrees, then walk right", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counterfactual", "Situational Awareness" ], "objects_of_interest": [ "bowl-7deff761870a158e", "mug-0ecc7dc3447ba548", "mug2-7daa9bd68ab0f9b2", "mug3-5179d5332f77444b", "mug4-accfd108d135ea96", "mug5-3c5a3b0ce0c17e1c", "mug6-4b8f9eabee67e377", "tongs-4a970b6754c38337", "bowl2-d02c1f20415c5e91", "bowl3-e56880be3d689b74", "tupperware container-0c0e53e6ca1f0934", "tupperware container lid-154c841f449ed934", "small grey bowl-abc12621c35b841b", "metal ring-78dde7195603f324", "mug7-aea95e5929f6a9bb", "spatula-f0ec3848fa7269bc", "cup-dd74e19e46620464", "drinking glass-0091421d7c87e0f0", "lids-e15a563f4191aa67", "plate-aa46127bcb6922e7", "three plates-52934356246712f4", "another plate-dbf0b7f816901a91", "plate2-de17a8c463e9b644", "frying pan-f2d392d1af498a43", "saucepan-36d6d99f081e767c", "glass-4e7f72166ebccd90", "glass2-dc2f54bb1aeebd32", "mug8-bc76814539ac11a6", "knife-cc1da5126bdaedb9", "knife2-7d35f8ae9c5629ef", "small wire brush-c1bb1fdc4f4e564f", "bowl4-653728d18f001a05", "bowl5-2764a117e804f1b8", "cutlery-33db068339ba7a75", "cutlery2-065ac73e435b73fd", "spoon-5f8f4c6e8744d0ac", "spoon2-f5fadd1c1d3a55fa", "spoon3-ae1796504e10313d", "spoon4-12ccd3fe415424e1", "more cutlery-d22ed6f87f644be3", "spoon5-c63e015aa7229d37", "bowl6-004f3e5d08b101c7", "spoon6-b38286d84085d6f9", "two mugs-cbfa6799ec5addab", "yellow mug-8dc29969040ce054", "glass3-2ae0e6716c4d471e", "glass4-4b8d06b314384387", "saucepan lid-c30eeca48fdf0086", "fork-9ba3d12b02a92b55", "teaspoon-1b4679763269a37a", "three spoons-86e8c45c7e8dec1e", "tablespoon3-f5306b5b98bbab9e", "second bowl-5b6ca123b8bceb58", "third bowl-669fca83fb96fd68", "cloth-6b1d73fadb02d0ec", "dish brush-ec57da4f863920ba", "drain cover-13a8b3a44509883a", "lid of food waste bin-221e5c9d812df91b", "bottle of dishwashing liquid-282c90601f1c5f42", "drain cover2-f279789c441a5084", "frying pan2-586500a422b272d4", "dirty saucepan-3d78070aad67ba5d", "lid-02d18babe87d7aca", "large white jug of dishwashing liquid-2e54bce4d331713e", "lid2-8f369368ceaade18", "orange sponge-882c1f24c7a2e305", "scourer-aaab02140eb6207d", "tea towel-d18b5c8120e0871b", "chair-b2c89e4ad0cd1a67", "work surface-fa26c9ad53962227", "piece of ceramic-1fdf6c31e5174a26", "mug9-3acac054623baed7", "clock-1dfb73cef062d8e1", "work surface2-9731a8bdcbf4eeed", "box-4e05e69bc82c0bb3", "coffee machine-4a539fa6446aa4b4", "portafilter-be99f9411d680045", "teaspoon3-a463b6f92df06daf", "drawer-026b57c0033788df", "kettle base-815bc99542489ad6", "salt-c521370ec81ae21d", "pepper shakers-3209c23f0ee4e97f" ], "reasoning": "1. I watched the video from the beginning to identify which drawer is the pots and pans drawer. At 02:03, the user grabs their first frying pan (frying pan-f2d392d1af498a43) from the dishwasher, and then walks toward the kitchen island, opening a drawer of pots and pans at 02:08. \n2. I notice at 02:08 what direction the stove and sink are located in relation to the pots and pans drawer. \n3. I noticed that if the user is standing in front of the pots and pans drawer, the stove is directly behind him, and the sink is to its left. Therefore, if the user wants to move from behind the pots and pans drawer to grab the frying pan (frying pan-f2d392d1af498a43) from its hypothetical position on the stove and put it in the sink, the user needs to turn 180 degrees to walk towards the stove, then walk right towards the sink.", "referenced_object_masks": [ "frying pan-f2d392d1af498a43" ] }, { "key": "P07-20240529-102652:86b068b830dca3ee6c7d95bad771b785fd54316f", "video_id": "P07-20240529-102652", "question": "What is the second black item on the kitchen island that I pass on my right when entering the kitchen after grabbing a bowl from the dining table?", "answer_choice_0": "Cell phone", "answer_choice_1": "Scale", "answer_choice_2": "Rag", "answer_choice_3": "Water bottle", "answer_choice_4": "Bowl", "answer_id": 1, "answer": "Scale", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Counting" ], "objects_of_interest": [ "bowl-7deff761870a158e", "mug-0ecc7dc3447ba548", "mug2-7daa9bd68ab0f9b2", "mug3-5179d5332f77444b", "mug4-accfd108d135ea96", "mug5-3c5a3b0ce0c17e1c", "mug6-4b8f9eabee67e377", "tongs-4a970b6754c38337", "bowl2-d02c1f20415c5e91", "bowl3-e56880be3d689b74", "tupperware container-0c0e53e6ca1f0934", "tupperware container lid-154c841f449ed934", "small grey bowl-abc12621c35b841b", "metal ring-78dde7195603f324", "mug7-aea95e5929f6a9bb", "spatula-f0ec3848fa7269bc", "cup-dd74e19e46620464", "drinking glass-0091421d7c87e0f0", "lids-e15a563f4191aa67", "plate-aa46127bcb6922e7", "three plates-52934356246712f4", "another plate-dbf0b7f816901a91", "plate2-de17a8c463e9b644", "frying pan-f2d392d1af498a43", "saucepan-36d6d99f081e767c", "glass-4e7f72166ebccd90", "glass2-dc2f54bb1aeebd32", "mug8-bc76814539ac11a6", "knife-cc1da5126bdaedb9", "knife2-7d35f8ae9c5629ef", "small wire brush-c1bb1fdc4f4e564f", "bowl4-653728d18f001a05", "bowl5-2764a117e804f1b8", "cutlery-33db068339ba7a75", "cutlery2-065ac73e435b73fd", "spoon-5f8f4c6e8744d0ac", "spoon2-f5fadd1c1d3a55fa", "spoon3-ae1796504e10313d", "spoon4-12ccd3fe415424e1", "more cutlery-d22ed6f87f644be3", "spoon5-c63e015aa7229d37", "bowl6-004f3e5d08b101c7", "spoon6-b38286d84085d6f9", "two mugs-cbfa6799ec5addab", "yellow mug-8dc29969040ce054", "glass3-2ae0e6716c4d471e", "glass4-4b8d06b314384387", "saucepan lid-c30eeca48fdf0086", "fork-9ba3d12b02a92b55", "teaspoon-1b4679763269a37a", "three spoons-86e8c45c7e8dec1e", "tablespoon3-f5306b5b98bbab9e", "second bowl-5b6ca123b8bceb58", "third bowl-669fca83fb96fd68", "cloth-6b1d73fadb02d0ec", "dish brush-ec57da4f863920ba", "drain cover-13a8b3a44509883a", "lid of food waste bin-221e5c9d812df91b", "bottle of dishwashing liquid-282c90601f1c5f42", "drain cover2-f279789c441a5084", "frying pan2-586500a422b272d4", "dirty saucepan-3d78070aad67ba5d", "lid-02d18babe87d7aca", "large white jug of dishwashing liquid-2e54bce4d331713e", "lid2-8f369368ceaade18", "orange sponge-882c1f24c7a2e305", "scourer-aaab02140eb6207d", "tea towel-d18b5c8120e0871b", "chair-b2c89e4ad0cd1a67", "work surface-fa26c9ad53962227", "piece of ceramic-1fdf6c31e5174a26", "mug9-3acac054623baed7", "clock-1dfb73cef062d8e1", "work surface2-9731a8bdcbf4eeed", "box-4e05e69bc82c0bb3", "coffee machine-4a539fa6446aa4b4", "portafilter-be99f9411d680045", "teaspoon3-a463b6f92df06daf", "drawer-026b57c0033788df", "kettle base-815bc99542489ad6", "salt-c521370ec81ae21d", "pepper shakers-3209c23f0ee4e97f" ], "reasoning": "1. I located the moment the user enters the kitchen at 00:00. \n2. The user makes a left and grabs a bowl (bowl-7deff761870a158e) from the table at 00:03.\n3. After this event, the user walks toward the sink, passing the kitchen island on the right. There are several items lined up on the island.\n4. The first black thing the user passes by on the right from 00:05 to 00:06 is a water bottle and the second item immediately after the water bottle is a cell phone . Therefore, the correct answer is cell phone.", "referenced_object_masks": [ "bowl-7deff761870a158e" ] }, { "key": "P07-20240529-102652:8cd52f9492bc87d3f456ff7f0c1296e17e5560fc", "video_id": "P07-20240529-102652", "question": "If I'm standing in front of the stove, what clock position is the bowl on the dining room table found near the beginning of the video?", "answer_choice_0": "1 o'clock", "answer_choice_1": "6 o'clock", "answer_choice_2": "12 o'clock", "answer_choice_3": "9 o'clock", "answer_choice_4": "3 o'clock", "answer_id": 3, "answer": "9 o'clock", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Situational Awareness" ], "objects_of_interest": [ "bowl-7deff761870a158e", "mug-0ecc7dc3447ba548", "mug2-7daa9bd68ab0f9b2", "mug3-5179d5332f77444b", "mug4-accfd108d135ea96", "mug5-3c5a3b0ce0c17e1c", "mug6-4b8f9eabee67e377", "tongs-4a970b6754c38337", "bowl2-d02c1f20415c5e91", "bowl3-e56880be3d689b74", "tupperware container-0c0e53e6ca1f0934", "tupperware container lid-154c841f449ed934", "small grey bowl-abc12621c35b841b", "metal ring-78dde7195603f324", "mug7-aea95e5929f6a9bb", "spatula-f0ec3848fa7269bc", "cup-dd74e19e46620464", "drinking glass-0091421d7c87e0f0", "lids-e15a563f4191aa67", "plate-aa46127bcb6922e7", "three plates-52934356246712f4", "another plate-dbf0b7f816901a91", "plate2-de17a8c463e9b644", "frying pan-f2d392d1af498a43", "saucepan-36d6d99f081e767c", "glass-4e7f72166ebccd90", "glass2-dc2f54bb1aeebd32", "mug8-bc76814539ac11a6", "knife-cc1da5126bdaedb9", "knife2-7d35f8ae9c5629ef", "small wire brush-c1bb1fdc4f4e564f", "bowl4-653728d18f001a05", "bowl5-2764a117e804f1b8", "cutlery-33db068339ba7a75", "cutlery2-065ac73e435b73fd", "spoon-5f8f4c6e8744d0ac", "spoon2-f5fadd1c1d3a55fa", "spoon3-ae1796504e10313d", "spoon4-12ccd3fe415424e1", "more cutlery-d22ed6f87f644be3", "spoon5-c63e015aa7229d37", "bowl6-004f3e5d08b101c7", "spoon6-b38286d84085d6f9", "two mugs-cbfa6799ec5addab", "yellow mug-8dc29969040ce054", "glass3-2ae0e6716c4d471e", "glass4-4b8d06b314384387", "saucepan lid-c30eeca48fdf0086", "fork-9ba3d12b02a92b55", "teaspoon-1b4679763269a37a", "three spoons-86e8c45c7e8dec1e", "tablespoon3-f5306b5b98bbab9e", "second bowl-5b6ca123b8bceb58", "third bowl-669fca83fb96fd68", "cloth-6b1d73fadb02d0ec", "dish brush-ec57da4f863920ba", "drain cover-13a8b3a44509883a", "lid of food waste bin-221e5c9d812df91b", "bottle of dishwashing liquid-282c90601f1c5f42", "drain cover2-f279789c441a5084", "frying pan2-586500a422b272d4", "dirty saucepan-3d78070aad67ba5d", "lid-02d18babe87d7aca", "large white jug of dishwashing liquid-2e54bce4d331713e", "lid2-8f369368ceaade18", "orange sponge-882c1f24c7a2e305", "scourer-aaab02140eb6207d", "tea towel-d18b5c8120e0871b", "chair-b2c89e4ad0cd1a67", "work surface-fa26c9ad53962227", "piece of ceramic-1fdf6c31e5174a26", "mug9-3acac054623baed7", "clock-1dfb73cef062d8e1", "work surface2-9731a8bdcbf4eeed", "box-4e05e69bc82c0bb3", "coffee machine-4a539fa6446aa4b4", "portafilter-be99f9411d680045", "teaspoon3-a463b6f92df06daf", "drawer-026b57c0033788df", "kettle base-815bc99542489ad6", "salt-c521370ec81ae21d", "pepper shakers-3209c23f0ee4e97f" ], "reasoning": "1. I watched the video from 00:00 to 00:05 to locate the dining room and the stove. I noticed the kitchen is in an L-shaped form, with the dining room in the front as the user enters the kitchen at 00:00. \n2. The user then makes a left turn at 0:03, picks up the bowl (bowl-7deff761870a158e) on the dining room table , and walks toward the sink, passing the stove to the left at 00:06.\n3. I realized that the bowl (bowl-7deff761870a158e) is on the edge of the dining room table , on the front left corner if I were facing it head-on from the table's narrower side.\n4. At 00:39, the user stands in front of the stove to place some tongs (tongs-4a970b6754c38337) in the drawer under the stove. \n5. From this position, I realized that if the user is standing in front of the stove, the bowl (bowl-7deff761870a158e) on the dining room table would be at approximately 9 o'clock.", "referenced_object_masks": [ "tongs-4a970b6754c38337", "bowl-7deff761870a158e" ] }, { "key": "P07-20240529-102652:9a4d4e577cfeb63fd24bbabc960584d1e25adc4d", "video_id": "P07-20240529-102652", "question": "After I open a cupboard or drawer for the 3rd time, what actions would I need to take to be facing the 1st cupboard or drawer I opened?", "answer_choice_0": "Turn right, walk forward, turn right, walk forward, turn left", "answer_choice_1": "Turn left, walk forward, turn right, walk forward, turn right", "answer_choice_2": "Turn right, walk forward, turn left", "answer_choice_3": "Turn left, walk forward, turn left", "answer_choice_4": "Turn left, walk forward, turn left, walk forward, turn right", "answer_id": 0, "answer": "Turn right, walk forward, turn right, walk forward, turn left", "question_type": [ "Egocentric & Spatial Perception", "Counting", "Action Recognition", "Counterfactual", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "bowl-7deff761870a158e", "mug-0ecc7dc3447ba548", "mug2-7daa9bd68ab0f9b2", "mug3-5179d5332f77444b", "mug4-accfd108d135ea96", "mug5-3c5a3b0ce0c17e1c", "mug6-4b8f9eabee67e377", "tongs-4a970b6754c38337", "bowl2-d02c1f20415c5e91", "bowl3-e56880be3d689b74", "tupperware container-0c0e53e6ca1f0934", "tupperware container lid-154c841f449ed934", "small grey bowl-abc12621c35b841b", "metal ring-78dde7195603f324", "mug7-aea95e5929f6a9bb", "spatula-f0ec3848fa7269bc", "cup-dd74e19e46620464", "drinking glass-0091421d7c87e0f0", "lids-e15a563f4191aa67", "plate-aa46127bcb6922e7", "three plates-52934356246712f4", "another plate-dbf0b7f816901a91", "plate2-de17a8c463e9b644", "frying pan-f2d392d1af498a43", "saucepan-36d6d99f081e767c", "glass-4e7f72166ebccd90", "glass2-dc2f54bb1aeebd32", "mug8-bc76814539ac11a6", "knife-cc1da5126bdaedb9", "knife2-7d35f8ae9c5629ef", "small wire brush-c1bb1fdc4f4e564f", "bowl4-653728d18f001a05", "bowl5-2764a117e804f1b8", "cutlery-33db068339ba7a75", "cutlery2-065ac73e435b73fd", "spoon-5f8f4c6e8744d0ac", "spoon2-f5fadd1c1d3a55fa", "spoon3-ae1796504e10313d", "spoon4-12ccd3fe415424e1", "more cutlery-d22ed6f87f644be3", "spoon5-c63e015aa7229d37", "bowl6-004f3e5d08b101c7", "spoon6-b38286d84085d6f9", "two mugs-cbfa6799ec5addab", "yellow mug-8dc29969040ce054", "glass3-2ae0e6716c4d471e", "glass4-4b8d06b314384387", "saucepan lid-c30eeca48fdf0086", "fork-9ba3d12b02a92b55", "teaspoon-1b4679763269a37a", "three spoons-86e8c45c7e8dec1e", "tablespoon3-f5306b5b98bbab9e", "second bowl-5b6ca123b8bceb58", "third bowl-669fca83fb96fd68", "cloth-6b1d73fadb02d0ec", "dish brush-ec57da4f863920ba", "drain cover-13a8b3a44509883a", "lid of food waste bin-221e5c9d812df91b", "bottle of dishwashing liquid-282c90601f1c5f42", "drain cover2-f279789c441a5084", "frying pan2-586500a422b272d4", "dirty saucepan-3d78070aad67ba5d", "lid-02d18babe87d7aca", "large white jug of dishwashing liquid-2e54bce4d331713e", "lid2-8f369368ceaade18", "orange sponge-882c1f24c7a2e305", "scourer-aaab02140eb6207d", "tea towel-d18b5c8120e0871b", "chair-b2c89e4ad0cd1a67", "work surface-fa26c9ad53962227", "piece of ceramic-1fdf6c31e5174a26", "mug9-3acac054623baed7", "clock-1dfb73cef062d8e1", "work surface2-9731a8bdcbf4eeed", "box-4e05e69bc82c0bb3", "coffee machine-4a539fa6446aa4b4", "portafilter-be99f9411d680045", "teaspoon3-a463b6f92df06daf", "drawer-026b57c0033788df", "kettle base-815bc99542489ad6", "salt-c521370ec81ae21d", "pepper shakers-3209c23f0ee4e97f" ], "reasoning": "1. I watched the video in order to count the times when the user opened a cupboard or drawer. I found the first such occurrence of this at 00:23, when they opened a cupboard on the right side of the kitchen.\n2. The second occurrence happened at 00:37, when they opened up a drawer in order to put away a pair of tongs (tongs-4a970b6754c38337).\n3. The third occurrence happened at 01:07, when the user moved to the left side of their kitchen near the fridge to open up a cupboard of plates.\n4. Then, I tried to determine what moves the user would need to make by looking back at their previous movements. I knew the third occurrence took place to the left of the sink. More specifically, at 01:06, the user turned to the left sharply from looking at the dishwasher and took at least one step in order to arrive at this cupboard.\n5. At 00:21, the user moved to the right from the dishwasher and took at least one step in order to arrive at the 1st cupboard.\n6. So, from this knowledge, I can say that the user would need to turn right, walk forward, turn right again, walk forward again, and then turn left to find and face the 1st cupboard.", "referenced_object_masks": [ "tongs-4a970b6754c38337" ] }, { "key": "P07-20240529-102652:a07a8f4e5e23c3481c8070daa4d03c31c87cebf1", "video_id": "P07-20240529-102652", "question": "What is the ratio of cabinet doors I close to cabinet doors I open from 04:20 to 06:20?", "answer_choice_0": "1:5", "answer_choice_1": "2:3", "answer_choice_2": "2:1", "answer_choice_3": "3:2", "answer_choice_4": "1:2", "answer_id": 2, "answer": "2:1", "question_type": [ "Object Recognition", "Action Recognition", "Counting", "Numerical Reasoning", "Temporal Reasoning" ], "objects_of_interest": [ "bowl-7deff761870a158e", "mug-0ecc7dc3447ba548", "mug2-7daa9bd68ab0f9b2", "mug3-5179d5332f77444b", "mug4-accfd108d135ea96", "mug5-3c5a3b0ce0c17e1c", "mug6-4b8f9eabee67e377", "tongs-4a970b6754c38337", "bowl2-d02c1f20415c5e91", "bowl3-e56880be3d689b74", "tupperware container-0c0e53e6ca1f0934", "tupperware container lid-154c841f449ed934", "small grey bowl-abc12621c35b841b", "metal ring-78dde7195603f324", "mug7-aea95e5929f6a9bb", "spatula-f0ec3848fa7269bc", "cup-dd74e19e46620464", "drinking glass-0091421d7c87e0f0", "lids-e15a563f4191aa67", "plate-aa46127bcb6922e7", "three plates-52934356246712f4", "another plate-dbf0b7f816901a91", "plate2-de17a8c463e9b644", "frying pan-f2d392d1af498a43", "saucepan-36d6d99f081e767c", "glass-4e7f72166ebccd90", "glass2-dc2f54bb1aeebd32", "mug8-bc76814539ac11a6", "knife-cc1da5126bdaedb9", "knife2-7d35f8ae9c5629ef", "small wire brush-c1bb1fdc4f4e564f", "bowl4-653728d18f001a05", "bowl5-2764a117e804f1b8", "cutlery-33db068339ba7a75", "cutlery2-065ac73e435b73fd", "spoon-5f8f4c6e8744d0ac", "spoon2-f5fadd1c1d3a55fa", "spoon3-ae1796504e10313d", "spoon4-12ccd3fe415424e1", "more cutlery-d22ed6f87f644be3", "spoon5-c63e015aa7229d37", "bowl6-004f3e5d08b101c7", "spoon6-b38286d84085d6f9", "two mugs-cbfa6799ec5addab", "yellow mug-8dc29969040ce054", "glass3-2ae0e6716c4d471e", "glass4-4b8d06b314384387", "saucepan lid-c30eeca48fdf0086", "fork-9ba3d12b02a92b55", "teaspoon-1b4679763269a37a", "three spoons-86e8c45c7e8dec1e", "tablespoon3-f5306b5b98bbab9e", "second bowl-5b6ca123b8bceb58", "third bowl-669fca83fb96fd68", "cloth-6b1d73fadb02d0ec", "dish brush-ec57da4f863920ba", "drain cover-13a8b3a44509883a", "lid of food waste bin-221e5c9d812df91b", "bottle of dishwashing liquid-282c90601f1c5f42", "drain cover2-f279789c441a5084", "frying pan2-586500a422b272d4", "dirty saucepan-3d78070aad67ba5d", "lid-02d18babe87d7aca", "large white jug of dishwashing liquid-2e54bce4d331713e", "lid2-8f369368ceaade18", "orange sponge-882c1f24c7a2e305", "scourer-aaab02140eb6207d", "tea towel-d18b5c8120e0871b", "chair-b2c89e4ad0cd1a67", "work surface-fa26c9ad53962227", "piece of ceramic-1fdf6c31e5174a26", "mug9-3acac054623baed7", "clock-1dfb73cef062d8e1", "work surface2-9731a8bdcbf4eeed", "box-4e05e69bc82c0bb3", "coffee machine-4a539fa6446aa4b4", "portafilter-be99f9411d680045", "teaspoon3-a463b6f92df06daf", "drawer-026b57c0033788df", "kettle base-815bc99542489ad6", "salt-c521370ec81ae21d", "pepper shakers-3209c23f0ee4e97f" ], "reasoning": "1. I watched the entire segment, from 04:20 to 06:20 to count the number of times the user closes and opens the cabinet doors. \n2. At 04:25, the user closes the first kitchen cabinet door . \n3. I continued watching and noticed the user closing the second cabinet door at 04:31. \n4. At 05:22, the user opens the first cabinet door under the sink and closes it at 05:28. \n5. At 6:11, the user opens the cabinet door under the sink again, and closes it at 06:18. \n6. Based on these observations, the user opens the cabinet doors 4 times, and closes cabinet doors twice. Therefore, the ratio is 4:2, which simplifies to 2:1.", "referenced_object_masks": [] }, { "key": "P07-20240529-102652:eef93b6620de5697afdc96246e20f7e6351e4a61", "video_id": "P07-20240529-102652", "question": "What is the second item I remove from the dishwasher that I put back in the sink?", "answer_choice_0": "The knife", "answer_choice_1": "The bowls", "answer_choice_2": "The mugs", "answer_choice_3": "The tongs", "answer_choice_4": "The tupperware with its lid", "answer_id": 4, "answer": "The tupperware with its lid", "question_type": [ "Object Recognition", "Action Recognition", "Event Occurence", "Counting", "Situational Awareness" ], "objects_of_interest": [ "bowl-7deff761870a158e", "mug-0ecc7dc3447ba548", "mug2-7daa9bd68ab0f9b2", "mug3-5179d5332f77444b", "mug4-accfd108d135ea96", "mug5-3c5a3b0ce0c17e1c", "mug6-4b8f9eabee67e377", "tongs-4a970b6754c38337", "bowl2-d02c1f20415c5e91", "bowl3-e56880be3d689b74", "tupperware container-0c0e53e6ca1f0934", "tupperware container lid-154c841f449ed934", "small grey bowl-abc12621c35b841b", "metal ring-78dde7195603f324", "mug7-aea95e5929f6a9bb", "spatula-f0ec3848fa7269bc", "cup-dd74e19e46620464", "drinking glass-0091421d7c87e0f0", "lids-e15a563f4191aa67", "plate-aa46127bcb6922e7", "three plates-52934356246712f4", "another plate-dbf0b7f816901a91", "plate2-de17a8c463e9b644", "frying pan-f2d392d1af498a43", "saucepan-36d6d99f081e767c", "glass-4e7f72166ebccd90", "glass2-dc2f54bb1aeebd32", "mug8-bc76814539ac11a6", "knife-cc1da5126bdaedb9", "knife2-7d35f8ae9c5629ef", "small wire brush-c1bb1fdc4f4e564f", "bowl4-653728d18f001a05", "bowl5-2764a117e804f1b8", "cutlery-33db068339ba7a75", "cutlery2-065ac73e435b73fd", "spoon-5f8f4c6e8744d0ac", "spoon2-f5fadd1c1d3a55fa", "spoon3-ae1796504e10313d", "spoon4-12ccd3fe415424e1", "more cutlery-d22ed6f87f644be3", "spoon5-c63e015aa7229d37", "bowl6-004f3e5d08b101c7", "spoon6-b38286d84085d6f9", "two mugs-cbfa6799ec5addab", "yellow mug-8dc29969040ce054", "glass3-2ae0e6716c4d471e", "glass4-4b8d06b314384387", "saucepan lid-c30eeca48fdf0086", "fork-9ba3d12b02a92b55", "teaspoon-1b4679763269a37a", "three spoons-86e8c45c7e8dec1e", "tablespoon3-f5306b5b98bbab9e", "second bowl-5b6ca123b8bceb58", "third bowl-669fca83fb96fd68", "cloth-6b1d73fadb02d0ec", "dish brush-ec57da4f863920ba", "drain cover-13a8b3a44509883a", "lid of food waste bin-221e5c9d812df91b", "bottle of dishwashing liquid-282c90601f1c5f42", "drain cover2-f279789c441a5084", "frying pan2-586500a422b272d4", "dirty saucepan-3d78070aad67ba5d", "lid-02d18babe87d7aca", "large white jug of dishwashing liquid-2e54bce4d331713e", "lid2-8f369368ceaade18", "orange sponge-882c1f24c7a2e305", "scourer-aaab02140eb6207d", "tea towel-d18b5c8120e0871b", "chair-b2c89e4ad0cd1a67", "work surface-fa26c9ad53962227", "piece of ceramic-1fdf6c31e5174a26", "mug9-3acac054623baed7", "clock-1dfb73cef062d8e1", "work surface2-9731a8bdcbf4eeed", "box-4e05e69bc82c0bb3", "coffee machine-4a539fa6446aa4b4", "portafilter-be99f9411d680045", "teaspoon3-a463b6f92df06daf", "drawer-026b57c0033788df", "kettle base-815bc99542489ad6", "salt-c521370ec81ae21d", "pepper shakers-3209c23f0ee4e97f" ], "reasoning": "1. I located the moment the user starts to empty the dishwasher at 00:12. \n2. First, the user removes 6 mugs (mug-0ecc7dc3447ba548, mug2-7daa9bd68ab0f9b2, mug3-5179d5332f77444b, mug4-accfd108d135ea96, mug5-3c5a3b0ce0c17e1c, mug6-4b8f9eabee67e377) from 00:12 to 00:20 and places them inside the cabinet at 00:24.\n3. The user walks back to the dishwasher, grabs a set of tongs (tongs-4a970b6754c38337) at 00:36 and places it inside a kitchen drawer at 00:38. The user grabs the tongs (tongs-4a970b6754c38337) again after placing them in the drawer and puts them in the sink at 00:41. This is the first item the user removes from the dishwasher and places it in the sink. \n4. From 00:44 to 00:47, the user removes two bowls (bowl2-d02c1f20415c5e91, bowl3-e56880be3d689b74) from the dishwasher and places them on the countertop. \n5. At 00:57, the user grabs a tupperware container with its lid tupperware container-0c0e53e6ca1f0934, tupperware container lid-154c841f449ed934) and inspects it. The user then places the tupperware and its lid tupperware container-0c0e53e6ca1f0934, tupperware container lid-154c841f449ed934) in the sink at 01:04. Therefore, the second item the user places in the sink after removing it from the dishwasher is the tupperware container with its lid (tupperware container-0c0e53e6ca1f0934, tupperware container lid-154c841f449ed934).", "referenced_object_masks": [ "bowl2-d02c1f20415c5e91", "bowl3-e56880be3d689b74", "mug-0ecc7dc3447ba548", "mug2-7daa9bd68ab0f9b2", "mug3-5179d5332f77444b", "mug4-accfd108d135ea96", "mug5-3c5a3b0ce0c17e1c", "mug6-4b8f9eabee67e377", "tupperware container-0c0e53e6ca1f0934", "tupperware container lid-154c841f449ed934", "tongs-4a970b6754c38337" ] }, { "key": "P07-20240529-102652:f7c56e85024c45e2ff405b7c7ea6960be518fecd", "video_id": "P07-20240529-102652", "question": "The fourth time I pick up a cloth to wipe off surfaces, which surface do I wipe off first?", "answer_choice_0": "The counter", "answer_choice_1": "The stovetop", "answer_choice_2": "The backsplash", "answer_choice_3": "The kitchen island", "answer_choice_4": "The cutting board", "answer_id": 0, "answer": "The counter", "question_type": [ "Action Recognition", "Counting", "Object Recognition", "Situational Awareness", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "bowl-7deff761870a158e", "mug-0ecc7dc3447ba548", "mug2-7daa9bd68ab0f9b2", "mug3-5179d5332f77444b", "mug4-accfd108d135ea96", "mug5-3c5a3b0ce0c17e1c", "mug6-4b8f9eabee67e377", "tongs-4a970b6754c38337", "bowl2-d02c1f20415c5e91", "bowl3-e56880be3d689b74", "tupperware container-0c0e53e6ca1f0934", "tupperware container lid-154c841f449ed934", "small grey bowl-abc12621c35b841b", "metal ring-78dde7195603f324", "mug7-aea95e5929f6a9bb", "spatula-f0ec3848fa7269bc", "cup-dd74e19e46620464", "drinking glass-0091421d7c87e0f0", "lids-e15a563f4191aa67", "plate-aa46127bcb6922e7", "three plates-52934356246712f4", "another plate-dbf0b7f816901a91", "plate2-de17a8c463e9b644", "frying pan-f2d392d1af498a43", "saucepan-36d6d99f081e767c", "glass-4e7f72166ebccd90", "glass2-dc2f54bb1aeebd32", "mug8-bc76814539ac11a6", "knife-cc1da5126bdaedb9", "knife2-7d35f8ae9c5629ef", "small wire brush-c1bb1fdc4f4e564f", "bowl4-653728d18f001a05", "bowl5-2764a117e804f1b8", "cutlery-33db068339ba7a75", "cutlery2-065ac73e435b73fd", "spoon-5f8f4c6e8744d0ac", "spoon2-f5fadd1c1d3a55fa", "spoon3-ae1796504e10313d", "spoon4-12ccd3fe415424e1", "more cutlery-d22ed6f87f644be3", "spoon5-c63e015aa7229d37", "bowl6-004f3e5d08b101c7", "spoon6-b38286d84085d6f9", "two mugs-cbfa6799ec5addab", "yellow mug-8dc29969040ce054", "glass3-2ae0e6716c4d471e", "glass4-4b8d06b314384387", "saucepan lid-c30eeca48fdf0086", "fork-9ba3d12b02a92b55", "teaspoon-1b4679763269a37a", "three spoons-86e8c45c7e8dec1e", "tablespoon3-f5306b5b98bbab9e", "second bowl-5b6ca123b8bceb58", "third bowl-669fca83fb96fd68", "cloth-6b1d73fadb02d0ec", "dish brush-ec57da4f863920ba", "drain cover-13a8b3a44509883a", "lid of food waste bin-221e5c9d812df91b", "bottle of dishwashing liquid-282c90601f1c5f42", "drain cover2-f279789c441a5084", "frying pan2-586500a422b272d4", "dirty saucepan-3d78070aad67ba5d", "lid-02d18babe87d7aca", "large white jug of dishwashing liquid-2e54bce4d331713e", "lid2-8f369368ceaade18", "orange sponge-882c1f24c7a2e305", "scourer-aaab02140eb6207d", "tea towel-d18b5c8120e0871b", "chair-b2c89e4ad0cd1a67", "work surface-fa26c9ad53962227", "piece of ceramic-1fdf6c31e5174a26", "mug9-3acac054623baed7", "clock-1dfb73cef062d8e1", "work surface2-9731a8bdcbf4eeed", "box-4e05e69bc82c0bb3", "coffee machine-4a539fa6446aa4b4", "portafilter-be99f9411d680045", "teaspoon3-a463b6f92df06daf", "drawer-026b57c0033788df", "kettle base-815bc99542489ad6", "salt-c521370ec81ae21d", "pepper shakers-3209c23f0ee4e97f" ], "reasoning": "1. I watched the video to look for the times when the user picked up a cloth to wipe off surfaces. This first happened at 06:05 when the user picked up the cloth by the sink (cloth-6b1d73fadb02d0ec) to wipe off the excess soap from the jug of dishwashing liquid (large white jug of dishwashing liquid-2e54bce4d331713e) before putting it back down by the sink at 06:07.\n2. At 09:23, the user picks up the same cloth (cloth-6b1d73fadb02d0ec) at the sink and wipes down several other surfaces from 09:27-10:33.\n3. Continuing to watch, I found that the user picks up a different cloth at 10:42, and then uses it at 10:48 to briefly wipe down the kitchen island.\n4. At 11:15, the user picks up the first cloth (cloth-6b1d73fadb02d0ec), which makes it the fourth time that a cloth is picked up and used to wipe down a surface. At 11:16, they moved to the area of the counter to the left of the sink to wipe it down. That makes this the 1st surface.", "referenced_object_masks": [ "cloth-6b1d73fadb02d0ec", "large white jug of dishwashing liquid-2e54bce4d331713e" ] }, { "key": "P07-20240529-131737:3739e8d778cb19b697e7bcb1a88d5732db53c2e0", "video_id": "P07-20240529-131737", "question": "When I retrieve the second object on which I put the peeled lettuce, where is the sink in relation to me?", "answer_choice_0": "10 o'clock", "answer_choice_1": "8 o'clock", "answer_choice_2": "6 o'clock", "answer_choice_3": "2 o'clock", "answer_choice_4": "4 o'clock", "answer_id": 0, "answer": "10 o'clock", "question_type": [ "Counting", "Action Recognition", "Situational Awareness", "Egocentric & Spatial Perception", "Object Recognition" ], "objects_of_interest": [ "tote bag-fa982882fc4307c7", "can of chickpeas-413edaa109e1a2c9", "glass jar of olives-dd61751558a127a9", "plastic package of lettuce-9ce9319882d0ddf6", "honey-74b7a8f2b292a46e", "cherry tomatoes-e0f3aa72d48600c7", "cheese-6bb4c0a6c85b0ee3", "phone-94b50e93067a4a9a", "can opener-326d12545db496a3", "chopstick-6903fceba9e5e70b", "saucepan lid-3815017d744ff035", "colander-81e7468655846c5a", "circular tray-b75e367bca6e57ed", "rectangular tray-6664223d438ec5d9", "kitchen roll-c88b776f0c45ef52", "piece of kitchen roll-b56c4123852a90da", "piece of paper-9f8350237f711414", "measuring spoons-0bfbfdd7a3d698e8", "bottle of olive oil-78ab86ba31406370", "olive oil cap-06ee3358734e875c", "spice jar1-ff24ad9dc255bc15", "spice jar2-734abd52344f5e32", "spice jar3-4b393e8cbc6fb931", "spice jar4-b323d01c7c225b0b", "spice jar5-c8c83eaef679923e", "lid of spice jar1-710654246fc13f3c", "lid of spice jar5-1e448e536c49220f", "lid of spice jar3-ce95d0d3a6c84023", "lid of spice jar4-c63f970292dc3364", "wooden spatula-dd331eae91903d24", "wooden spoon-3ebc75de953485b0", "lettuce1-0cbf06e0d1af1c96", "lettuce2-d8d0b15dd93e7893", "chopping board-6e787472f43b434d", "tea towel-60ab534851b08e25" ], "reasoning": "1. I located the first instance of the user putting peeled lettuce on an object and found that they placed the lettuce on a wooden cutting board (chopping board-6e787472f43b434d) beginning at 08:12. \n2. I looked for the user to place peeled lettuce on a second object and found that they put it in a colander (colander-81e7468655846c5a) at 09:07. \n3. I then went back and found the moment where the user retrieved the colander (colander-81e7468655846c5a), and noticed that they grabbed it (colander-81e7468655846c5a) from a drawer beneath the island at 01:09. \n4. I noted the location of the sink when it first became visible at 00:02. \n5. Reasoning about the space, I realized that when the user is standing in front of the island retrieving the colander (colander-81e7468655846c5a), the sink is in front of them on the left, at the relative 10 o'clock position.", "referenced_object_masks": [ "chopping board-6e787472f43b434d", "colander-81e7468655846c5a" ] }, { "key": "P07-20240529-131737:3bf71158c4b440a6ab561dfc54f2a68aaf0cfe4c", "video_id": "P07-20240529-131737", "question": "When I rinse the chickpeas, where is the drawer with pans in relation to me?", "answer_choice_0": "4 o'clock", "answer_choice_1": "10 o'clock", "answer_choice_2": "9 o'clock", "answer_choice_3": "2 o'clock", "answer_choice_4": "7 o'clock", "answer_id": 4, "answer": "7 o'clock", "question_type": [ "Action Recognition", "Object Recognition", "Situational Awareness", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "tote bag-fa982882fc4307c7", "can of chickpeas-413edaa109e1a2c9", "glass jar of olives-dd61751558a127a9", "plastic package of lettuce-9ce9319882d0ddf6", "honey-74b7a8f2b292a46e", "cherry tomatoes-e0f3aa72d48600c7", "cheese-6bb4c0a6c85b0ee3", "phone-94b50e93067a4a9a", "can opener-326d12545db496a3", "chopstick-6903fceba9e5e70b", "saucepan lid-3815017d744ff035", "colander-81e7468655846c5a", "circular tray-b75e367bca6e57ed", "rectangular tray-6664223d438ec5d9", "kitchen roll-c88b776f0c45ef52", "piece of kitchen roll-b56c4123852a90da", "piece of paper-9f8350237f711414", "measuring spoons-0bfbfdd7a3d698e8", "bottle of olive oil-78ab86ba31406370", "olive oil cap-06ee3358734e875c", "spice jar1-ff24ad9dc255bc15", "spice jar2-734abd52344f5e32", "spice jar3-4b393e8cbc6fb931", "spice jar4-b323d01c7c225b0b", "spice jar5-c8c83eaef679923e", "lid of spice jar1-710654246fc13f3c", "lid of spice jar5-1e448e536c49220f", "lid of spice jar3-ce95d0d3a6c84023", "lid of spice jar4-c63f970292dc3364", "wooden spatula-dd331eae91903d24", "wooden spoon-3ebc75de953485b0", "lettuce1-0cbf06e0d1af1c96", "lettuce2-d8d0b15dd93e7893", "chopping board-6e787472f43b434d", "tea towel-60ab534851b08e25" ], "reasoning": "1. First, I had to locate the drawer with pans . I saw the user retrieve a pan (rectangular tray-6664223d438ec5d9) from a drawer beneath the oven at 02:02 and took note of the drawer's position in the space. \n2. Next, I located the moment where the user rinses the chickpeas and found they do this at the sink at 01:17-01:46. \n3. Reasoning about the scene, I realized that while the user is at the sink, the drawer with the pans is behind them at the 7 o'clock position.", "referenced_object_masks": [ "rectangular tray-6664223d438ec5d9" ] }, { "key": "P07-20240529-131737:6f1dc25098f4be3600d9c2b8267545516d05ccad", "video_id": "P07-20240529-131737", "question": "If I divide the number of times I removed chickpea skins and placed them on the counter by the number of spices I seasoned the chickpeas with, what would the total be?", "answer_choice_0": "3.5", "answer_choice_1": "5.5", "answer_choice_2": "1.5", "answer_choice_3": "2.5", "answer_choice_4": "4.5", "answer_id": 1, "answer": "5.5", "question_type": [ "Numerical Reasoning", "Action Recognition", "Object Recognition", "Counting" ], "objects_of_interest": [ "tote bag-fa982882fc4307c7", "can of chickpeas-413edaa109e1a2c9", "glass jar of olives-dd61751558a127a9", "plastic package of lettuce-9ce9319882d0ddf6", "honey-74b7a8f2b292a46e", "cherry tomatoes-e0f3aa72d48600c7", "cheese-6bb4c0a6c85b0ee3", "phone-94b50e93067a4a9a", "can opener-326d12545db496a3", "chopstick-6903fceba9e5e70b", "saucepan lid-3815017d744ff035", "colander-81e7468655846c5a", "circular tray-b75e367bca6e57ed", "rectangular tray-6664223d438ec5d9", "kitchen roll-c88b776f0c45ef52", "piece of kitchen roll-b56c4123852a90da", "piece of paper-9f8350237f711414", "measuring spoons-0bfbfdd7a3d698e8", "bottle of olive oil-78ab86ba31406370", "olive oil cap-06ee3358734e875c", "spice jar1-ff24ad9dc255bc15", "spice jar2-734abd52344f5e32", "spice jar3-4b393e8cbc6fb931", "spice jar4-b323d01c7c225b0b", "spice jar5-c8c83eaef679923e", "lid of spice jar1-710654246fc13f3c", "lid of spice jar5-1e448e536c49220f", "lid of spice jar3-ce95d0d3a6c84023", "lid of spice jar4-c63f970292dc3364", "wooden spatula-dd331eae91903d24", "wooden spoon-3ebc75de953485b0", "lettuce1-0cbf06e0d1af1c96", "lettuce2-d8d0b15dd93e7893", "chopping board-6e787472f43b434d", "tea towel-60ab534851b08e25" ], "reasoning": "1. First I counted the number of times the user removed chickpea skins and placed them on the counter. I saw them do this at 02:36, 02:41, 02:43, 02:45, 02:47, 02:50, 02:53, 02:54, 02:56, 02:57, 03:00, 03:02, 03:20, 03:26, 03:27, 03:28, 03:31, 03:33, 03:36, 03:38, 03:41, and 03:46 for a total of 22 times. \n2. Next, I looked for the user to season the chickpeas with spices and saw them do this with individual spices at 05:44, 05:59, 06:13, and 06:34 for a total of 4 spices . \n3. Finally, I divided 22 by 4 and got 5.5.", "referenced_object_masks": [] }, { "key": "P07-20240529-131737:c1b39d891d0729c2110b4ddd965903f0093d5d53", "video_id": "P07-20240529-131737", "question": "If I am facing the sink and wanted to face the place where I retrieved the can of chickpeas, approximately how many degrees would I turn in a clockwise direction?", "answer_choice_0": "180 degrees", "answer_choice_1": "225 degrees", "answer_choice_2": "270 degrees", "answer_choice_3": "45 degrees", "answer_choice_4": "90 degrees", "answer_id": 0, "answer": "180 degrees", "question_type": [ "Counterfactual", "Object Recognition", "Situational Awareness", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "tote bag-fa982882fc4307c7", "can of chickpeas-413edaa109e1a2c9", "glass jar of olives-dd61751558a127a9", "plastic package of lettuce-9ce9319882d0ddf6", "honey-74b7a8f2b292a46e", "cherry tomatoes-e0f3aa72d48600c7", "cheese-6bb4c0a6c85b0ee3", "phone-94b50e93067a4a9a", "can opener-326d12545db496a3", "chopstick-6903fceba9e5e70b", "saucepan lid-3815017d744ff035", "colander-81e7468655846c5a", "circular tray-b75e367bca6e57ed", "rectangular tray-6664223d438ec5d9", "kitchen roll-c88b776f0c45ef52", "piece of kitchen roll-b56c4123852a90da", "piece of paper-9f8350237f711414", "measuring spoons-0bfbfdd7a3d698e8", "bottle of olive oil-78ab86ba31406370", "olive oil cap-06ee3358734e875c", "spice jar1-ff24ad9dc255bc15", "spice jar2-734abd52344f5e32", "spice jar3-4b393e8cbc6fb931", "spice jar4-b323d01c7c225b0b", "spice jar5-c8c83eaef679923e", "lid of spice jar1-710654246fc13f3c", "lid of spice jar5-1e448e536c49220f", "lid of spice jar3-ce95d0d3a6c84023", "lid of spice jar4-c63f970292dc3364", "wooden spatula-dd331eae91903d24", "wooden spoon-3ebc75de953485b0", "lettuce1-0cbf06e0d1af1c96", "lettuce2-d8d0b15dd93e7893", "chopping board-6e787472f43b434d", "tea towel-60ab534851b08e25" ], "reasoning": "1. First, I took note of the location of the sink when it first became clearly visible at 00:02. \n2. Next, I looked for the user to retrieve a can of chickpeas (can of chickpeas-413edaa109e1a2c9) and saw them taking grocery items (glass jar of olives-dd61751558a127a9, plastic package of lettuce-9ce9319882d0ddf6, honey-74b7a8f2b292a46e, cherry tomatoes-e0f3aa72d48600c7) - including the can of chickpeas (can of chickpeas-413edaa109e1a2c9) - out of a bag (tote bag-fa982882fc4307c7) on the kitchen island beginning at 00:12. \n3. I took note of the position of the bag (tote bag-fa982882fc4307c7) on the island which I knew was directly across from the sink. I then imagined the user standing at the sink and realized they would need to turn approximately 180 degrees in a clockwise direction if they wanted to face the bag (tote bag-fa982882fc4307c7) on the island from which they retrieved the chickpeas (can of chickpeas-413edaa109e1a2c9).", "referenced_object_masks": [ "can of chickpeas-413edaa109e1a2c9", "glass jar of olives-dd61751558a127a9", "plastic package of lettuce-9ce9319882d0ddf6", "honey-74b7a8f2b292a46e", "cherry tomatoes-e0f3aa72d48600c7", "tote bag-fa982882fc4307c7" ] }, { "key": "P07-20240529-131737:f8c819c5038b95f8ee83af56d2a55488fc9df15e", "video_id": "P07-20240529-131737", "question": "What do the first and second items I wash in the colander have in common?", "answer_choice_0": "They are both seeds", "answer_choice_1": "They are both vegetables", "answer_choice_2": "They are both green", "answer_choice_3": "They are both fruits", "answer_choice_4": "They are both yellow", "answer_id": 1, "answer": "They are both vegetables", "question_type": [ "Action Recognition", "Counting", "Object Recognition" ], "objects_of_interest": [ "tote bag-fa982882fc4307c7", "can of chickpeas-413edaa109e1a2c9", "glass jar of olives-dd61751558a127a9", "plastic package of lettuce-9ce9319882d0ddf6", "honey-74b7a8f2b292a46e", "cherry tomatoes-e0f3aa72d48600c7", "cheese-6bb4c0a6c85b0ee3", "phone-94b50e93067a4a9a", "can opener-326d12545db496a3", "chopstick-6903fceba9e5e70b", "saucepan lid-3815017d744ff035", "colander-81e7468655846c5a", "circular tray-b75e367bca6e57ed", "rectangular tray-6664223d438ec5d9", "kitchen roll-c88b776f0c45ef52", "piece of kitchen roll-b56c4123852a90da", "piece of paper-9f8350237f711414", "measuring spoons-0bfbfdd7a3d698e8", "bottle of olive oil-78ab86ba31406370", "olive oil cap-06ee3358734e875c", "spice jar1-ff24ad9dc255bc15", "spice jar2-734abd52344f5e32", "spice jar3-4b393e8cbc6fb931", "spice jar4-b323d01c7c225b0b", "spice jar5-c8c83eaef679923e", "lid of spice jar1-710654246fc13f3c", "lid of spice jar5-1e448e536c49220f", "lid of spice jar3-ce95d0d3a6c84023", "lid of spice jar4-c63f970292dc3364", "wooden spatula-dd331eae91903d24", "wooden spoon-3ebc75de953485b0", "lettuce1-0cbf06e0d1af1c96", "lettuce2-d8d0b15dd93e7893", "chopping board-6e787472f43b434d", "tea towel-60ab534851b08e25" ], "reasoning": "1. I looked for the user to wash an item in the colander (colander-81e7468655846c5a) for the first time and found that they washed chickpeas from 01:17-01:46. \n2. Next, I looked for the user to wash a second item in the colander (colander-81e7468655846c5a) and saw that they washed lettuce at 09:12-09:29.\n3. I realized that both chickpeas and lettuce are types of vegetables.", "referenced_object_masks": [ "colander-81e7468655846c5a" ] }, { "key": "P07-20240529-134410:032555e6d7902180f9b4370ee3c0a865a31d7267", "video_id": "P07-20240529-134410", "question": "Which direction from me is the folding stool located in when I write on the piece of paper for the first time?", "answer_choice_0": "Directly behind", "answer_choice_1": "To the left", "answer_choice_2": "Behind, on the left", "answer_choice_3": "To the right", "answer_choice_4": "Behind, on the right", "answer_id": 3, "answer": "To the right", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "wooden spoon-2f1b296660565813", "tea towel-aba94dc4f6a74a86", "baking tray-0a237fea209b80b2", "colander-bca49544f701ab84", "digital weighing scale-041ac302390f98cf", "plate-f6c785389a7b2ddb", "phone-061d78a0d1bc1fcb", "lettuce leaves-edd0f621ad553f57", "empty package-14d980c979057ad4", "paper-385147aa8c9db54b", "stool-d1e0dcd0601715c3", "metal silver bowl-3d4556c60f579605", "packets of cherry tomatoes-41d4a645fa1f884d", "another packet of cherry tomatoes-7fdbd02bd61f499c", "tomato stalk-389e0d0710463969", "tomato-1118da051d6acc63", "tomato stalk2-3b38299be691b941", "tomato stalk3-48708cc5285a9d5f", "cherry tomatoes-45f530cf485621cb", "last cherry tomatoes-4d64ac2bfe38d031", "cardboard packaging-a18ecb608f065ee6", "another empty packaging-4b4de8a8611447f0", "knife-e9d2e59a10859778", "tomatoes-a283436ea0c58184", "pen-8abf31798a32af49", "cucumber-0c0fbaefb4e37652", "glass jar of olives-ca0869d724ee5a33", "lid of jar-1b29b26fb0c1625e", "olive-0207602c2c6fa27d", "olive2-eb8b75a09b99e28d", "olive3-8b3295c8ffc0b806", "olive4-7bf835dec393b0ec", "olive5-ca0593719a95ed74", "olives6-6e36cbf45e7914e1", "olive7-563395735fc526e9", "tea towel2-2fcae4d765e94eab", "blue cheese-798e70e40937b9a3", "plastic packaging-f6c75c32b3ea2c62", "folded piece of kitchen paper-958380181aa86054", "blue cheese piece-048edde01a57b885", "scrubbing brush-873ad82184af0088", "top of olive oil bottle-117e73c40161d5a2", "olive oil bottle-ebeb40f648c0f4bf", "end of cucumber-44d649a6fb05ca3d", "trash-d83f2b8394db0e7e", "spoon-faa01a4f26bd293f", "bottle of dishwashing liquid-2b1dfbe5ce4e5025" ], "reasoning": "1. I identified the stool (stool-d1e0dcd0601715c3) from 02:40 to 02:45, when the user retrieves it and brings it to the kitchen counter next to the oven so that they can step on it to reach the top cabinet.\n2. I note the location of the stool (stool-d1e0dcd0601715c3). It is located on the wall opposite directly opposite from the stove. This is clear from 02:35 to 02:40, when the user walks toward the stool (stool-d1e0dcd0601715c3) to retrieve it.\n3. I observed that the user wrote on the piece of paper (paper-385147aa8c9db54b) for the first time at 01:07. They are writing on the counter to the left of the sink. If the user turned right and walked past the sink to the wall, they would reach the stool (stool-d1e0dcd0601715c3).", "referenced_object_masks": [ "stool-d1e0dcd0601715c3", "paper-385147aa8c9db54b" ] }, { "key": "P07-20240529-134410:1891d3af3e60313924de9c69cfe1ce61acf08db7", "video_id": "P07-20240529-134410", "question": "What is the fifth item that I hold using my right hand?", "answer_choice_0": "Scale", "answer_choice_1": "Wooden spoon", "answer_choice_2": "Pen", "answer_choice_3": "Plate", "answer_choice_4": "Colander", "answer_id": 2, "answer": "Pen", "question_type": [ "Object Recognition", "Counting", "Action Recognition" ], "objects_of_interest": [ "wooden spoon-2f1b296660565813", "tea towel-aba94dc4f6a74a86", "baking tray-0a237fea209b80b2", "colander-bca49544f701ab84", "digital weighing scale-041ac302390f98cf", "plate-f6c785389a7b2ddb", "phone-061d78a0d1bc1fcb", "lettuce leaves-edd0f621ad553f57", "empty package-14d980c979057ad4", "paper-385147aa8c9db54b", "stool-d1e0dcd0601715c3", "metal silver bowl-3d4556c60f579605", "packets of cherry tomatoes-41d4a645fa1f884d", "another packet of cherry tomatoes-7fdbd02bd61f499c", "tomato stalk-389e0d0710463969", "tomato-1118da051d6acc63", "tomato stalk2-3b38299be691b941", "tomato stalk3-48708cc5285a9d5f", "cherry tomatoes-45f530cf485621cb", "last cherry tomatoes-4d64ac2bfe38d031", "cardboard packaging-a18ecb608f065ee6", "another empty packaging-4b4de8a8611447f0", "knife-e9d2e59a10859778", "tomatoes-a283436ea0c58184", "pen-8abf31798a32af49", "cucumber-0c0fbaefb4e37652", "glass jar of olives-ca0869d724ee5a33", "lid of jar-1b29b26fb0c1625e", "olive-0207602c2c6fa27d", "olive2-eb8b75a09b99e28d", "olive3-8b3295c8ffc0b806", "olive4-7bf835dec393b0ec", "olive5-ca0593719a95ed74", "olives6-6e36cbf45e7914e1", "olive7-563395735fc526e9", "tea towel2-2fcae4d765e94eab", "blue cheese-798e70e40937b9a3", "plastic packaging-f6c75c32b3ea2c62", "folded piece of kitchen paper-958380181aa86054", "blue cheese piece-048edde01a57b885", "scrubbing brush-873ad82184af0088", "top of olive oil bottle-117e73c40161d5a2", "olive oil bottle-ebeb40f648c0f4bf", "end of cucumber-44d649a6fb05ca3d", "trash-d83f2b8394db0e7e", "spoon-faa01a4f26bd293f", "bottle of dishwashing liquid-2b1dfbe5ce4e5025" ], "reasoning": "1. From the beginning of the video, I counted each item the user touched with their right hand. The first item is a wooden spoon (wooden spoon-2f1b296660565813) at 00:05.\n2. The second item is a colander (colander-bca49544f701ab84) at 00:42.\n3. The third item, the scale (digital weighing scale-041ac302390f98cf) is at 00:46 and the fourth item, a plate (plate-f6c785389a7b2ddb) is at 00:55.\n4. The fifth item is a pen (pen-8abf31798a32af49) at 01:05.", "referenced_object_masks": [ "colander-bca49544f701ab84", "plate-f6c785389a7b2ddb", "pen-8abf31798a32af49", "wooden spoon-2f1b296660565813", "digital weighing scale-041ac302390f98cf" ] }, { "key": "P07-20240529-134410:3c84d6c3eaf999aa86e4492d769c32359ac7c520", "video_id": "P07-20240529-134410", "question": "Which set of directions would lead me from where I cut the cucumber to the woodframed orange image on a wall?", "answer_choice_0": "Walk left, then turn left at the oven", "answer_choice_1": "Walk left, then turn right at the dining table", "answer_choice_2": "Turn 180 degrees and walk straight", "answer_choice_3": "Walk left, then turn left at the dining table", "answer_choice_4": "Turn 90 degrees to the right and walk straight", "answer_id": 3, "answer": "Walk left, then turn left at the dining table", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness", "Action Recognition", "Counterfactual" ], "objects_of_interest": [ "wooden spoon-2f1b296660565813", "tea towel-aba94dc4f6a74a86", "baking tray-0a237fea209b80b2", "colander-bca49544f701ab84", "digital weighing scale-041ac302390f98cf", "plate-f6c785389a7b2ddb", "phone-061d78a0d1bc1fcb", "lettuce leaves-edd0f621ad553f57", "empty package-14d980c979057ad4", "paper-385147aa8c9db54b", "stool-d1e0dcd0601715c3", "metal silver bowl-3d4556c60f579605", "packets of cherry tomatoes-41d4a645fa1f884d", "another packet of cherry tomatoes-7fdbd02bd61f499c", "tomato stalk-389e0d0710463969", "tomato-1118da051d6acc63", "tomato stalk2-3b38299be691b941", "tomato stalk3-48708cc5285a9d5f", "cherry tomatoes-45f530cf485621cb", "last cherry tomatoes-4d64ac2bfe38d031", "cardboard packaging-a18ecb608f065ee6", "another empty packaging-4b4de8a8611447f0", "knife-e9d2e59a10859778", "tomatoes-a283436ea0c58184", "pen-8abf31798a32af49", "cucumber-0c0fbaefb4e37652", "glass jar of olives-ca0869d724ee5a33", "lid of jar-1b29b26fb0c1625e", "olive-0207602c2c6fa27d", "olive2-eb8b75a09b99e28d", "olive3-8b3295c8ffc0b806", "olive4-7bf835dec393b0ec", "olive5-ca0593719a95ed74", "olives6-6e36cbf45e7914e1", "olive7-563395735fc526e9", "tea towel2-2fcae4d765e94eab", "blue cheese-798e70e40937b9a3", "plastic packaging-f6c75c32b3ea2c62", "folded piece of kitchen paper-958380181aa86054", "blue cheese piece-048edde01a57b885", "scrubbing brush-873ad82184af0088", "top of olive oil bottle-117e73c40161d5a2", "olive oil bottle-ebeb40f648c0f4bf", "end of cucumber-44d649a6fb05ca3d", "trash-d83f2b8394db0e7e", "spoon-faa01a4f26bd293f", "bottle of dishwashing liquid-2b1dfbe5ce4e5025" ], "reasoning": "1. The user began cutting the cucumber (cucumber-0c0fbaefb4e37652) at 05:20. During this time, the user is facing the kitchen counter and stands to the right of the stove.\n2. I identified the woodframed orange image hanging from a wall at 01:04 near a dining table.\n3. I searched for a clear view of this side of the kitchen and found a moment at 00:44, when the user has already turned left from the counter where they cut the cucumber (cucumber-0c0fbaefb4e37652).\n4. If the user continues to walk straight and then turns left upon reaching the dining table, the user would arrive at the woodframed orange image.", "referenced_object_masks": [ "cucumber-0c0fbaefb4e37652" ] }, { "key": "P07-20240529-134410:4611fa8b7e1d9b0a3ec0b10c339643a122c22ad2", "video_id": "P07-20240529-134410", "question": "What is the sum of the red number on the oven when I first opened the oven door and the number displayed on the weighing scale after I filled up the bowl on the scale with lettuce?", "answer_choice_0": "1093", "answer_choice_1": "1026", "answer_choice_2": "1061", "answer_choice_3": "1018", "answer_choice_4": "1042", "answer_id": 4, "answer": "1042", "question_type": [ "Action Recognition", "Numerical Reasoning", "Object Recognition", "Situational Awareness", "Reading" ], "objects_of_interest": [ "wooden spoon-2f1b296660565813", "tea towel-aba94dc4f6a74a86", "baking tray-0a237fea209b80b2", "colander-bca49544f701ab84", "digital weighing scale-041ac302390f98cf", "plate-f6c785389a7b2ddb", "phone-061d78a0d1bc1fcb", "lettuce leaves-edd0f621ad553f57", "empty package-14d980c979057ad4", "paper-385147aa8c9db54b", "stool-d1e0dcd0601715c3", "metal silver bowl-3d4556c60f579605", "packets of cherry tomatoes-41d4a645fa1f884d", "another packet of cherry tomatoes-7fdbd02bd61f499c", "tomato stalk-389e0d0710463969", "tomato-1118da051d6acc63", "tomato stalk2-3b38299be691b941", "tomato stalk3-48708cc5285a9d5f", "cherry tomatoes-45f530cf485621cb", "last cherry tomatoes-4d64ac2bfe38d031", "cardboard packaging-a18ecb608f065ee6", "another empty packaging-4b4de8a8611447f0", "knife-e9d2e59a10859778", "tomatoes-a283436ea0c58184", "pen-8abf31798a32af49", "cucumber-0c0fbaefb4e37652", "glass jar of olives-ca0869d724ee5a33", "lid of jar-1b29b26fb0c1625e", "olive-0207602c2c6fa27d", "olive2-eb8b75a09b99e28d", "olive3-8b3295c8ffc0b806", "olive4-7bf835dec393b0ec", "olive5-ca0593719a95ed74", "olives6-6e36cbf45e7914e1", "olive7-563395735fc526e9", "tea towel2-2fcae4d765e94eab", "blue cheese-798e70e40937b9a3", "plastic packaging-f6c75c32b3ea2c62", "folded piece of kitchen paper-958380181aa86054", "blue cheese piece-048edde01a57b885", "scrubbing brush-873ad82184af0088", "top of olive oil bottle-117e73c40161d5a2", "olive oil bottle-ebeb40f648c0f4bf", "end of cucumber-44d649a6fb05ca3d", "trash-d83f2b8394db0e7e", "spoon-faa01a4f26bd293f", "bottle of dishwashing liquid-2b1dfbe5ce4e5025" ], "reasoning": "1. I observed the user opening the oven door for the first time at 00:13. The red number displayed on the oven reads \u201c746.\u201d\n2. The user began filling the metal bowl (metal silver bowl-3d4556c60f579605) with lettuce (lettuce leaves-edd0f621ad553f57) at 01:25 and finished filling up the metal bowl (metal silver bowl-3d4556c60f579605) at 02:21.\n3. At 02:21, the number on the scale (digital weighing scale-041ac302390f98cf) reads \u201c296.\u201d\n4. The sum of 746 and 296 is 1,042.", "referenced_object_masks": [ "lettuce leaves-edd0f621ad553f57", "metal silver bowl-3d4556c60f579605", "digital weighing scale-041ac302390f98cf" ] }, { "key": "P07-20240529-134410:4bff1d60fde2b13129b711337a74069c623b1a15", "video_id": "P07-20240529-134410", "question": "What is the sum of the numerical weight values of the first three salad ingredients I add to the bowl?", "answer_choice_0": "654", "answer_choice_1": "615", "answer_choice_2": "675", "answer_choice_3": "701", "answer_choice_4": "692", "answer_id": 0, "answer": "654", "question_type": [ "Action Recognition", "Reading", "Numerical Reasoning", "Temporal Reasoning" ], "objects_of_interest": [ "wooden spoon-2f1b296660565813", "tea towel-aba94dc4f6a74a86", "baking tray-0a237fea209b80b2", "colander-bca49544f701ab84", "digital weighing scale-041ac302390f98cf", "plate-f6c785389a7b2ddb", "phone-061d78a0d1bc1fcb", "lettuce leaves-edd0f621ad553f57", "empty package-14d980c979057ad4", "paper-385147aa8c9db54b", "stool-d1e0dcd0601715c3", "metal silver bowl-3d4556c60f579605", "packets of cherry tomatoes-41d4a645fa1f884d", "another packet of cherry tomatoes-7fdbd02bd61f499c", "tomato stalk-389e0d0710463969", "tomato-1118da051d6acc63", "tomato stalk2-3b38299be691b941", "tomato stalk3-48708cc5285a9d5f", "cherry tomatoes-45f530cf485621cb", "last cherry tomatoes-4d64ac2bfe38d031", "cardboard packaging-a18ecb608f065ee6", "another empty packaging-4b4de8a8611447f0", "knife-e9d2e59a10859778", "tomatoes-a283436ea0c58184", "pen-8abf31798a32af49", "cucumber-0c0fbaefb4e37652", "glass jar of olives-ca0869d724ee5a33", "lid of jar-1b29b26fb0c1625e", "olive-0207602c2c6fa27d", "olive2-eb8b75a09b99e28d", "olive3-8b3295c8ffc0b806", "olive4-7bf835dec393b0ec", "olive5-ca0593719a95ed74", "olives6-6e36cbf45e7914e1", "olive7-563395735fc526e9", "tea towel2-2fcae4d765e94eab", "blue cheese-798e70e40937b9a3", "plastic packaging-f6c75c32b3ea2c62", "folded piece of kitchen paper-958380181aa86054", "blue cheese piece-048edde01a57b885", "scrubbing brush-873ad82184af0088", "top of olive oil bottle-117e73c40161d5a2", "olive oil bottle-ebeb40f648c0f4bf", "end of cucumber-44d649a6fb05ca3d", "trash-d83f2b8394db0e7e", "spoon-faa01a4f26bd293f", "bottle of dishwashing liquid-2b1dfbe5ce4e5025" ], "reasoning": "1. I watched the video to see the user begin to hand shred the lettuce (lettuce leaves-edd0f621ad553f57) from 01:19-01:22.\n2. Then, the user shreds and places the lettuce (lettuce leaves-edd0f621ad553f57) in the large bowl (metal silver bowl-3d4556c60f579605) on the scale (digital weighing scale-041ac302390f98cf) from 01:24-02:21.\n3. At 02:21, I saw that the scale read \"296\".\n4. Then, I observed the user move all of the rest of the sliced tomatoes (cherry tomatoes-45f530cf485621cb) onto the scale (digital weighing scale-041ac302390f98cf) from 04:43-04:47.\n5. When the user finishes this, the scale (digital weighing scale-041ac302390f98cf) reads 146.\n6. Next, from 05:28-05:53, the user begins slicing and dicing a cucumber (cucumber-0c0fbaefb4e37652).\n7. After placing some chopped cucumbers (cucumber-0c0fbaefb4e37652), they chop more cucumbers (cucumber-0c0fbaefb4e37652) placing rhem on the scale (digital weighing scale-041ac302390f98cf) from 06:18-06:35.\n8. At 06:35, the scale reads \"212\".\n9. Final math is 296 + 146 + 212 = 654.", "referenced_object_masks": [ "cucumber-0c0fbaefb4e37652", "metal silver bowl-3d4556c60f579605", "lettuce leaves-edd0f621ad553f57", "digital weighing scale-041ac302390f98cf", "cherry tomatoes-45f530cf485621cb" ] }, { "key": "P07-20240529-134410:5dd8667dcd845ef609964755ad82149286053401", "video_id": "P07-20240529-134410", "question": "How many items do I touch with my left hand in the 30 seconds after I first set down the step stool?", "answer_choice_0": "5", "answer_choice_1": "7", "answer_choice_2": "4", "answer_choice_3": "8", "answer_choice_4": "6", "answer_id": 4, "answer": "6", "question_type": [ "Counting", "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "wooden spoon-2f1b296660565813", "tea towel-aba94dc4f6a74a86", "baking tray-0a237fea209b80b2", "colander-bca49544f701ab84", "digital weighing scale-041ac302390f98cf", "plate-f6c785389a7b2ddb", "phone-061d78a0d1bc1fcb", "lettuce leaves-edd0f621ad553f57", "empty package-14d980c979057ad4", "paper-385147aa8c9db54b", "stool-d1e0dcd0601715c3", "metal silver bowl-3d4556c60f579605", "packets of cherry tomatoes-41d4a645fa1f884d", "another packet of cherry tomatoes-7fdbd02bd61f499c", "tomato stalk-389e0d0710463969", "tomato-1118da051d6acc63", "tomato stalk2-3b38299be691b941", "tomato stalk3-48708cc5285a9d5f", "cherry tomatoes-45f530cf485621cb", "last cherry tomatoes-4d64ac2bfe38d031", "cardboard packaging-a18ecb608f065ee6", "another empty packaging-4b4de8a8611447f0", "knife-e9d2e59a10859778", "tomatoes-a283436ea0c58184", "pen-8abf31798a32af49", "cucumber-0c0fbaefb4e37652", "glass jar of olives-ca0869d724ee5a33", "lid of jar-1b29b26fb0c1625e", "olive-0207602c2c6fa27d", "olive2-eb8b75a09b99e28d", "olive3-8b3295c8ffc0b806", "olive4-7bf835dec393b0ec", "olive5-ca0593719a95ed74", "olives6-6e36cbf45e7914e1", "olive7-563395735fc526e9", "tea towel2-2fcae4d765e94eab", "blue cheese-798e70e40937b9a3", "plastic packaging-f6c75c32b3ea2c62", "folded piece of kitchen paper-958380181aa86054", "blue cheese piece-048edde01a57b885", "scrubbing brush-873ad82184af0088", "top of olive oil bottle-117e73c40161d5a2", "olive oil bottle-ebeb40f648c0f4bf", "end of cucumber-44d649a6fb05ca3d", "trash-d83f2b8394db0e7e", "spoon-faa01a4f26bd293f", "bottle of dishwashing liquid-2b1dfbe5ce4e5025" ], "reasoning": "1. I watched the video to observe the user first touch the step stool (stool-d1e0dcd0601715c3) at 02:40.\n2. The user grabs the stool (stool-d1e0dcd0601715c3) and sets it down at 02:45.\n3. The user proceeds to step up on the stool (stool-d1e0dcd0601715c3) to look at the top shelf of the cabinet, touching the shelf with their left hand, making it the first item touched.\n4. Next, the user grabs the bowl of lettuce (lettuce leaves-edd0f621ad553f57) with both hands at 02:52, marking this as the second item touched with the left hand.\n5. The user performs tasks with their right hand before grabbing tomatoes (packets of cherry tomatoes-41d4a645fa1f884d) from the counter at 02:59 as the third item touched with the left hand.\n6. Next, at 03:04, the user grabs a package of tomatoes (another packet of cherry tomatoes-7fdbd02bd61f499c), which is the fourth item touched with the left hand.\n7. Then, from 03:06 to 03:09, the user pulls tomatoes (tomato-1118da051d6acc63, cherry tomatoes-45f530cf485621cb) at two separate instances, marking these as the fifth and sixth items touched with the left hand.", "referenced_object_masks": [ "packets of cherry tomatoes-41d4a645fa1f884d", "another packet of cherry tomatoes-7fdbd02bd61f499c", "stool-d1e0dcd0601715c3", "lettuce leaves-edd0f621ad553f57", "tomato-1118da051d6acc63", "cherry tomatoes-45f530cf485621cb" ] }, { "key": "P07-20240529-134410:5e94e012e5fd049414d3d625803aba30cc7da62d", "video_id": "P07-20240529-134410", "question": "What is the difference between the weight shown on the scale after I put all of the tomatoes on the plate and the weight of the first cut tomato?", "answer_choice_0": "141", "answer_choice_1": "127", "answer_choice_2": "135", "answer_choice_3": "146", "answer_choice_4": "120", "answer_id": 2, "answer": "135", "question_type": [ "Numerical Reasoning", "Action Recognition", "Situational Awareness", "Temporal Reasoning" ], "objects_of_interest": [ "wooden spoon-2f1b296660565813", "tea towel-aba94dc4f6a74a86", "baking tray-0a237fea209b80b2", "colander-bca49544f701ab84", "digital weighing scale-041ac302390f98cf", "plate-f6c785389a7b2ddb", "phone-061d78a0d1bc1fcb", "lettuce leaves-edd0f621ad553f57", "empty package-14d980c979057ad4", "paper-385147aa8c9db54b", "stool-d1e0dcd0601715c3", "metal silver bowl-3d4556c60f579605", "packets of cherry tomatoes-41d4a645fa1f884d", "another packet of cherry tomatoes-7fdbd02bd61f499c", "tomato stalk-389e0d0710463969", "tomato-1118da051d6acc63", "tomato stalk2-3b38299be691b941", "tomato stalk3-48708cc5285a9d5f", "cherry tomatoes-45f530cf485621cb", "last cherry tomatoes-4d64ac2bfe38d031", "cardboard packaging-a18ecb608f065ee6", "another empty packaging-4b4de8a8611447f0", "knife-e9d2e59a10859778", "tomatoes-a283436ea0c58184", "pen-8abf31798a32af49", "cucumber-0c0fbaefb4e37652", "glass jar of olives-ca0869d724ee5a33", "lid of jar-1b29b26fb0c1625e", "olive-0207602c2c6fa27d", "olive2-eb8b75a09b99e28d", "olive3-8b3295c8ffc0b806", "olive4-7bf835dec393b0ec", "olive5-ca0593719a95ed74", "olives6-6e36cbf45e7914e1", "olive7-563395735fc526e9", "tea towel2-2fcae4d765e94eab", "blue cheese-798e70e40937b9a3", "plastic packaging-f6c75c32b3ea2c62", "folded piece of kitchen paper-958380181aa86054", "blue cheese piece-048edde01a57b885", "scrubbing brush-873ad82184af0088", "top of olive oil bottle-117e73c40161d5a2", "olive oil bottle-ebeb40f648c0f4bf", "end of cucumber-44d649a6fb05ca3d", "trash-d83f2b8394db0e7e", "spoon-faa01a4f26bd293f", "bottle of dishwashing liquid-2b1dfbe5ce4e5025" ], "reasoning": "1. I watched the user and saw that they first began cutting tomatoes at 04:00.\n2. Specifically, from 04:01-04:03, I watched the user cut one tomato (tomato-1118da051d6acc63) and place it on the plate (plate-f6c785389a7b2ddb) sitting on the scale (digital weighing scale-041ac302390f98cf). However, a clear view of the scale's reading is not clear at this point.\n3. I continued to watch the user slice the tomatoes (cherry tomatoes-45f530cf485621cb), from 04:03-04:42.\n4. I then slowed the video to 0.7x speed to get a clear view of the scale (digital weighing scale-041ac302390f98cf) which, at 04:43, reads 11 as the weight of just the first tomato (tomato-1118da051d6acc63).\n5. Then, I observed the user move all of the rest of the sliced tomatoes (cherry tomatoes-45f530cf485621cb) onto the scale (digital weighing scale-041ac302390f98cf) from 04:43-04:47.\n6. When the user finishes this, the scale (digital weighing scale-041ac302390f98cf) reads 146.\n7. The math for the answer is 146 - 11 = 135.", "referenced_object_masks": [ "plate-f6c785389a7b2ddb", "tomato-1118da051d6acc63", "digital weighing scale-041ac302390f98cf", "cherry tomatoes-45f530cf485621cb" ] }, { "key": "P07-20240529-134410:a0bf4e889c1d617e9a1810a2e72d2efb01951734", "video_id": "P07-20240529-134410", "question": "If the amount of time the parked yellow car was visible during the time the user washed the tomatoes is added to the weight of the tomatoes, what would be the new value?", "answer_choice_0": "152", "answer_choice_1": "154", "answer_choice_2": "159", "answer_choice_3": "150", "answer_choice_4": "156", "answer_id": 2, "answer": "159", "question_type": [ "Situational Awareness", "Object Recognition", "Reading", "Temporal Reasoning", "Numerical Reasoning", "Action Recognition" ], "objects_of_interest": [ "wooden spoon-2f1b296660565813", "tea towel-aba94dc4f6a74a86", "baking tray-0a237fea209b80b2", "colander-bca49544f701ab84", "digital weighing scale-041ac302390f98cf", "plate-f6c785389a7b2ddb", "phone-061d78a0d1bc1fcb", "lettuce leaves-edd0f621ad553f57", "empty package-14d980c979057ad4", "paper-385147aa8c9db54b", "stool-d1e0dcd0601715c3", "metal silver bowl-3d4556c60f579605", "packets of cherry tomatoes-41d4a645fa1f884d", "another packet of cherry tomatoes-7fdbd02bd61f499c", "tomato stalk-389e0d0710463969", "tomato-1118da051d6acc63", "tomato stalk2-3b38299be691b941", "tomato stalk3-48708cc5285a9d5f", "cherry tomatoes-45f530cf485621cb", "last cherry tomatoes-4d64ac2bfe38d031", "cardboard packaging-a18ecb608f065ee6", "another empty packaging-4b4de8a8611447f0", "knife-e9d2e59a10859778", "tomatoes-a283436ea0c58184", "pen-8abf31798a32af49", "cucumber-0c0fbaefb4e37652", "glass jar of olives-ca0869d724ee5a33", "lid of jar-1b29b26fb0c1625e", "olive-0207602c2c6fa27d", "olive2-eb8b75a09b99e28d", "olive3-8b3295c8ffc0b806", "olive4-7bf835dec393b0ec", "olive5-ca0593719a95ed74", "olives6-6e36cbf45e7914e1", "olive7-563395735fc526e9", "tea towel2-2fcae4d765e94eab", "blue cheese-798e70e40937b9a3", "plastic packaging-f6c75c32b3ea2c62", "folded piece of kitchen paper-958380181aa86054", "blue cheese piece-048edde01a57b885", "scrubbing brush-873ad82184af0088", "top of olive oil bottle-117e73c40161d5a2", "olive oil bottle-ebeb40f648c0f4bf", "end of cucumber-44d649a6fb05ca3d", "trash-d83f2b8394db0e7e", "spoon-faa01a4f26bd293f", "bottle of dishwashing liquid-2b1dfbe5ce4e5025" ], "reasoning": "1. I identified the parked yellow car in the background at 03:41 while the user washed the tomatoes (cherry tomatoes-45f530cf485621cb) over the sink. The parked yellow car (OBJ was visible until 03:54, when the user turned away from the sink. This is a total of 13 seconds.\n2. The user began adding tomatoes (cherry tomatoes-45f530cf485621cb) on the scale (digital weighing scale-041ac302390f98cf) at 04:02. I continued watching until the user placed all the tomatoes (cherry tomatoes-45f530cf485621cb) on the scale (digital weighing scale-041ac302390f98cf).\n3. The user finished weighing all the tomatoes (cherry tomatoes-45f530cf485621cb) at 04:47. The scale (digital weighing scale-041ac302390f98cf) displays the number \"146.\"\n4. 13 added to 146 totals 159.", "referenced_object_masks": [ "digital weighing scale-041ac302390f98cf", "cherry tomatoes-45f530cf485621cb" ] }, { "key": "P07-20240529-134410:e562bc082027e3fef2cf91ead6e36d8c71005c17", "video_id": "P07-20240529-134410", "question": "If I were to stop and stand at the counter, facing towards my phone on the counter, how many chairs would be behind me?", "answer_choice_0": "0", "answer_choice_1": "2", "answer_choice_2": "3", "answer_choice_3": "5", "answer_choice_4": "1", "answer_id": 1, "answer": "2", "question_type": [ "Object Recognition", "Egocentric & Spatial Perception", "Counting" ], "objects_of_interest": [ "wooden spoon-2f1b296660565813", "tea towel-aba94dc4f6a74a86", "baking tray-0a237fea209b80b2", "colander-bca49544f701ab84", "digital weighing scale-041ac302390f98cf", "plate-f6c785389a7b2ddb", "phone-061d78a0d1bc1fcb", "lettuce leaves-edd0f621ad553f57", "empty package-14d980c979057ad4", "paper-385147aa8c9db54b", "stool-d1e0dcd0601715c3", "metal silver bowl-3d4556c60f579605", "packets of cherry tomatoes-41d4a645fa1f884d", "another packet of cherry tomatoes-7fdbd02bd61f499c", "tomato stalk-389e0d0710463969", "tomato-1118da051d6acc63", "tomato stalk2-3b38299be691b941", "tomato stalk3-48708cc5285a9d5f", "cherry tomatoes-45f530cf485621cb", "last cherry tomatoes-4d64ac2bfe38d031", "cardboard packaging-a18ecb608f065ee6", "another empty packaging-4b4de8a8611447f0", "knife-e9d2e59a10859778", "tomatoes-a283436ea0c58184", "pen-8abf31798a32af49", "cucumber-0c0fbaefb4e37652", "glass jar of olives-ca0869d724ee5a33", "lid of jar-1b29b26fb0c1625e", "olive-0207602c2c6fa27d", "olive2-eb8b75a09b99e28d", "olive3-8b3295c8ffc0b806", "olive4-7bf835dec393b0ec", "olive5-ca0593719a95ed74", "olives6-6e36cbf45e7914e1", "olive7-563395735fc526e9", "tea towel2-2fcae4d765e94eab", "blue cheese-798e70e40937b9a3", "plastic packaging-f6c75c32b3ea2c62", "folded piece of kitchen paper-958380181aa86054", "blue cheese piece-048edde01a57b885", "scrubbing brush-873ad82184af0088", "top of olive oil bottle-117e73c40161d5a2", "olive oil bottle-ebeb40f648c0f4bf", "end of cucumber-44d649a6fb05ca3d", "trash-d83f2b8394db0e7e", "spoon-faa01a4f26bd293f", "bottle of dishwashing liquid-2b1dfbe5ce4e5025" ], "reasoning": "1. I watched the video to see the user behind at the counter, facing towards the phone (phone-061d78a0d1bc1fcb) at 11:35.\n2. At this point, I observed the kitchen, including the sink , stovetop , and other objects ahead in frame.\n3. I continued to watch the user walk around the kitchen until 12:17-12:20, during which I see there are two chairs at the table sitting behind the area at which the user would stand and face the phone (phone-061d78a0d1bc1fcb).", "referenced_object_masks": [ "phone-061d78a0d1bc1fcb" ] }, { "key": "P07-20240529-134410:f33f058af59823cde103d3f37ef0457c0468a1b9", "video_id": "P07-20240529-134410", "question": "How many times do I rip the third handful of lettuce before putting it in the bowl on the scale?", "answer_choice_0": "3", "answer_choice_1": "2", "answer_choice_2": "1", "answer_choice_3": "5", "answer_choice_4": "4", "answer_id": 1, "answer": "2", "question_type": [ "Counting", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "wooden spoon-2f1b296660565813", "tea towel-aba94dc4f6a74a86", "baking tray-0a237fea209b80b2", "colander-bca49544f701ab84", "digital weighing scale-041ac302390f98cf", "plate-f6c785389a7b2ddb", "phone-061d78a0d1bc1fcb", "lettuce leaves-edd0f621ad553f57", "empty package-14d980c979057ad4", "paper-385147aa8c9db54b", "stool-d1e0dcd0601715c3", "metal silver bowl-3d4556c60f579605", "packets of cherry tomatoes-41d4a645fa1f884d", "another packet of cherry tomatoes-7fdbd02bd61f499c", "tomato stalk-389e0d0710463969", "tomato-1118da051d6acc63", "tomato stalk2-3b38299be691b941", "tomato stalk3-48708cc5285a9d5f", "cherry tomatoes-45f530cf485621cb", "last cherry tomatoes-4d64ac2bfe38d031", "cardboard packaging-a18ecb608f065ee6", "another empty packaging-4b4de8a8611447f0", "knife-e9d2e59a10859778", "tomatoes-a283436ea0c58184", "pen-8abf31798a32af49", "cucumber-0c0fbaefb4e37652", "glass jar of olives-ca0869d724ee5a33", "lid of jar-1b29b26fb0c1625e", "olive-0207602c2c6fa27d", "olive2-eb8b75a09b99e28d", "olive3-8b3295c8ffc0b806", "olive4-7bf835dec393b0ec", "olive5-ca0593719a95ed74", "olives6-6e36cbf45e7914e1", "olive7-563395735fc526e9", "tea towel2-2fcae4d765e94eab", "blue cheese-798e70e40937b9a3", "plastic packaging-f6c75c32b3ea2c62", "folded piece of kitchen paper-958380181aa86054", "blue cheese piece-048edde01a57b885", "scrubbing brush-873ad82184af0088", "top of olive oil bottle-117e73c40161d5a2", "olive oil bottle-ebeb40f648c0f4bf", "end of cucumber-44d649a6fb05ca3d", "trash-d83f2b8394db0e7e", "spoon-faa01a4f26bd293f", "bottle of dishwashing liquid-2b1dfbe5ce4e5025" ], "reasoning": "1. I watched the video to place the bowl (plate-f6c785389a7b2ddb) on the scale (digital weighing scale-041ac302390f98cf) from 00:55-00:58.\n2. Then, I watched the user reach for the lettuce (lettuce leaves-edd0f621ad553f57) sitting on the counter in a colander (colander-bca49544f701ab84) from 01:19-01:22, putting some of the lettuce (lettuce leaves-edd0f621ad553f57) on the cutting board .\n3. From 01:23-01:26, the user grabs a handful of lettuce (lettuce leaves-edd0f621ad553f57) and rips them 3 times before putting them in the bowl .\n4. Next, from 01:27-01:32, the user grabs a second clump of lettuce (lettuce leaves-edd0f621ad553f57) and rips the leaves three times before placing them in the bowl \n5. The user then grabs a third clump of lettuce (lettuce leaves-edd0f621ad553f57), and from 01:32-01:37, they rip them twice before placing the leaves into the bowl .", "referenced_object_masks": [ "colander-bca49544f701ab84", "plate-f6c785389a7b2ddb", "digital weighing scale-041ac302390f98cf", "lettuce leaves-edd0f621ad553f57" ] }, { "key": "P07-20240529-191007:02b08678ad70f94d29d75d4cab00e2c6c13e4332", "video_id": "P07-20240529-191007", "question": "What is the ratio of the amount of time it took me in seconds to search for a jar of garlic in the refrigerator from the moment I opened the door to when I found it, and the number of individual items on the bottom two shelves of the refrigerator door?", "answer_choice_0": "27:5", "answer_choice_1": "6:1", "answer_choice_2": "33:5", "answer_choice_3": "7:1", "answer_choice_4": "29:4", "answer_id": 1, "answer": "6:1", "question_type": [ "Action Recognition", "Counting", "Event Occurence", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "piece of paper-bf8f784b82d48324", "pen-371a37932dee14bf", "smaller glass jug-f7d2085883de312c", "glass jug-8c241d892f6411a6", "digital weighing scales-ef7ce660d85df84f", "packet of mushrooms-014b68c23ae5eabd", "bowl-2ab0508662a40f03", "whole brown onion-cfe6a2f23f89b97c", "mushrooms-98766dba5a7ce407", "second pack of mushrooms-23d01cdd7f7657a3", "packet2-78ab138400b040fc", "cling film-21c626694a46e9a8", "knife-6f748ed0203c9641", "knife2-6114591711a1ccb4", "knife3-5101882187cfd3fd", "red onion-e424e8132fcc9536", "one half of brown onion-f703a1656bb28e95", "second half of brown onion-52545e711115ce73", "kettle-74a8b2a3ae189d8c", "fork-ae2d15829662b135", "phone-5dd4814328600726", "plastic packaging-bc1141ba01e9ba1e", "hummus-8da9a2fc6a7016a9", "jar of garlic-8bbd832082eb48c8", "small bowl-501a63422be3aaa4", "spoon-9183de8b5fcb802d", "clining film2-5b717f132de3399a", "lid-6ea5473e550a474c", "mushrooms packet1-00f6b4fca54b3c58", "mushrooms packet2-86d6e68560326de2", "mushrooms packet3-c3a2365d43481e80", "plastic packaging2-9fa24f4f9ce677f4", "plastic packaging3-8100bb72f8405ad5", "mushroom-4d76082e7b9ec768", "colander-8eb4057a30bdcc55", "baking tray-89ef6f170cea594f", "mushroom2-f9dcaf99333e2b39", "mushroom3-b08b1d06bdbbdef2", "mushroom4-400cd6d015491fee", "mushroom5-1aa477bdaee31f06", "mushroom6-d35733740873c64d", "mushroom7-2d6e4e16444edb63", "mushroom8-be20e4ae1a96e8b5", "mushroom9-c414fc887ed49a13", "mushroom10-946ede5d16786e7d", "mushroom11-4b284c2d842d3e50", "mushroom12-d5e621787be67433", "mushroom13-27243fb877c60087", "tea towel-0c0d9b68216d515a", "mushroom14-d364ff7c7b325854", "plastic film2-fad40cfc20c2b737", "plastic film3-7bbcaf5c768a6d84", "lid of garlic jar-0f0f61129763916e", "saucepan-c67b477f737e7c26", "box of stock cubes-4e0c7cc7a0abce93", "stock cube-cd74c78656d35100", "another stock cube-9f0b56ed6905ca18", "circular metal hob extender-eadbd0fe20998fba", "large frying pan-b42ad6eb492130f5", "spoon1-17d37f7b53b21d74", "bottle of oil-e0d7e6dcf818d784", "wooden spatula-6584758d8278cd5f", "honey-af18e9af046893c4", "glass jar of rice-bb367015db827a88", "spoon2-2df81f56fd9fef2c", "rubbish-4aefab195305ce8a" ], "reasoning": "1. I watched the video to see when the user went to the fridge to fetch a jar of garlic (jar of garlic-8bbd832082eb48c8). This happened at around 06:40.\n2. I then took note of when the user opened the fridge door and when the jar of garlic (jar of garlic-8bbd832082eb48c8) was actually found. The user opened the fridge door at 06:42 and found the jar of garlic (jar of garlic-8bbd832082eb48c8) at 07:12.\n3. I then calculated the amount of time between both events by subtracting the start time from the end time. The amount of time it took the user to find the jar of garlic (jar of garlic-8bbd832082eb48c8) is 07:12 - 06:42 = 00:30, or 30 seconds.\n4. Next, I looked for a clear view of all the contents on the bottom two shelves of the fridge door. At 06:57, we see that there are 5 unique items. \n5. Finally, I calculated the ratio of 30 seconds to the 5 items on the bottom two shelves on the door: 6:1.", "referenced_object_masks": [ "jar of garlic-8bbd832082eb48c8" ] }, { "key": "P07-20240529-191007:1f64d3552807acceaddd41a833e3ed421cb966a9", "video_id": "P07-20240529-191007", "question": "What fraction of the kettle station's circular buttons light up and glow blue as I get it ready?", "answer_choice_0": "1/8", "answer_choice_1": "1/6", "answer_choice_2": "1/3", "answer_choice_3": "1/2", "answer_choice_4": "1/4", "answer_id": 3, "answer": "1/2", "question_type": [ "Numerical Reasoning", "Action Recognition", "Object Recognition", "Counting", "Situational Awareness" ], "objects_of_interest": [ "piece of paper-bf8f784b82d48324", "pen-371a37932dee14bf", "smaller glass jug-f7d2085883de312c", "glass jug-8c241d892f6411a6", "digital weighing scales-ef7ce660d85df84f", "packet of mushrooms-014b68c23ae5eabd", "bowl-2ab0508662a40f03", "whole brown onion-cfe6a2f23f89b97c", "mushrooms-98766dba5a7ce407", "second pack of mushrooms-23d01cdd7f7657a3", "packet2-78ab138400b040fc", "cling film-21c626694a46e9a8", "knife-6f748ed0203c9641", "knife2-6114591711a1ccb4", "knife3-5101882187cfd3fd", "red onion-e424e8132fcc9536", "one half of brown onion-f703a1656bb28e95", "second half of brown onion-52545e711115ce73", "kettle-74a8b2a3ae189d8c", "fork-ae2d15829662b135", "phone-5dd4814328600726", "plastic packaging-bc1141ba01e9ba1e", "hummus-8da9a2fc6a7016a9", "jar of garlic-8bbd832082eb48c8", "small bowl-501a63422be3aaa4", "spoon-9183de8b5fcb802d", "clining film2-5b717f132de3399a", "lid-6ea5473e550a474c", "mushrooms packet1-00f6b4fca54b3c58", "mushrooms packet2-86d6e68560326de2", "mushrooms packet3-c3a2365d43481e80", "plastic packaging2-9fa24f4f9ce677f4", "plastic packaging3-8100bb72f8405ad5", "mushroom-4d76082e7b9ec768", "colander-8eb4057a30bdcc55", "baking tray-89ef6f170cea594f", "mushroom2-f9dcaf99333e2b39", "mushroom3-b08b1d06bdbbdef2", "mushroom4-400cd6d015491fee", "mushroom5-1aa477bdaee31f06", "mushroom6-d35733740873c64d", "mushroom7-2d6e4e16444edb63", "mushroom8-be20e4ae1a96e8b5", "mushroom9-c414fc887ed49a13", "mushroom10-946ede5d16786e7d", "mushroom11-4b284c2d842d3e50", "mushroom12-d5e621787be67433", "mushroom13-27243fb877c60087", "tea towel-0c0d9b68216d515a", "mushroom14-d364ff7c7b325854", "plastic film2-fad40cfc20c2b737", "plastic film3-7bbcaf5c768a6d84", "lid of garlic jar-0f0f61129763916e", "saucepan-c67b477f737e7c26", "box of stock cubes-4e0c7cc7a0abce93", "stock cube-cd74c78656d35100", "another stock cube-9f0b56ed6905ca18", "circular metal hob extender-eadbd0fe20998fba", "large frying pan-b42ad6eb492130f5", "spoon1-17d37f7b53b21d74", "bottle of oil-e0d7e6dcf818d784", "wooden spatula-6584758d8278cd5f", "honey-af18e9af046893c4", "glass jar of rice-bb367015db827a88", "spoon2-2df81f56fd9fef2c", "rubbish-4aefab195305ce8a" ], "reasoning": "1. I look for a kettle, and I find one (kettle-74a8b2a3ae189d8c) on a station with buttons at the end of the kitchen, to the left of a roll of paper towels at 00:06.\n2. I notice the user turn the kettle (kettle-74a8b2a3ae189d8c) on at 00:08. When this happens, 2 buttons glow blue.\n3. At 00:09, she presses a button next to 1 of the glowing ones, and another blue panel lights up while the 1 next to it stops glowing.\n4. After 00:09, the user causes no more buttons to glow blue.\n4. I count 6 total buttons on the station.\n5. When I put 3 over 6 and turn it into a simplified fraction, I get 1/2.\n6. Therefore, the answer is 1/2.", "referenced_object_masks": [ "kettle-74a8b2a3ae189d8c" ] }, { "key": "P07-20240529-191007:2d58c0b5b36abed21baf4c028eb3e67ade12dfe8", "video_id": "P07-20240529-191007", "question": "If I take the total weight of all the fresh mushrooms cut up and divide this number by the total number of fresh mushrooms cut up and raise it to the power of the total number of packages of mushrooms used, what number would I get?", "answer_choice_0": "316.05", "answer_choice_1": "45.94", "answer_choice_2": "256", "answer_choice_3": "369.02", "answer_choice_4": "37.21", "answer_id": 3, "answer": "369.02", "question_type": [ "Action Recognition", "Counting", "Event Occurence", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "piece of paper-bf8f784b82d48324", "pen-371a37932dee14bf", "smaller glass jug-f7d2085883de312c", "glass jug-8c241d892f6411a6", "digital weighing scales-ef7ce660d85df84f", "packet of mushrooms-014b68c23ae5eabd", "bowl-2ab0508662a40f03", "whole brown onion-cfe6a2f23f89b97c", "mushrooms-98766dba5a7ce407", "second pack of mushrooms-23d01cdd7f7657a3", "packet2-78ab138400b040fc", "cling film-21c626694a46e9a8", "knife-6f748ed0203c9641", "knife2-6114591711a1ccb4", "knife3-5101882187cfd3fd", "red onion-e424e8132fcc9536", "one half of brown onion-f703a1656bb28e95", "second half of brown onion-52545e711115ce73", "kettle-74a8b2a3ae189d8c", "fork-ae2d15829662b135", "phone-5dd4814328600726", "plastic packaging-bc1141ba01e9ba1e", "hummus-8da9a2fc6a7016a9", "jar of garlic-8bbd832082eb48c8", "small bowl-501a63422be3aaa4", "spoon-9183de8b5fcb802d", "clining film2-5b717f132de3399a", "lid-6ea5473e550a474c", "mushrooms packet1-00f6b4fca54b3c58", "mushrooms packet2-86d6e68560326de2", "mushrooms packet3-c3a2365d43481e80", "plastic packaging2-9fa24f4f9ce677f4", "plastic packaging3-8100bb72f8405ad5", "mushroom-4d76082e7b9ec768", "colander-8eb4057a30bdcc55", "baking tray-89ef6f170cea594f", "mushroom2-f9dcaf99333e2b39", "mushroom3-b08b1d06bdbbdef2", "mushroom4-400cd6d015491fee", "mushroom5-1aa477bdaee31f06", "mushroom6-d35733740873c64d", "mushroom7-2d6e4e16444edb63", "mushroom8-be20e4ae1a96e8b5", "mushroom9-c414fc887ed49a13", "mushroom10-946ede5d16786e7d", "mushroom11-4b284c2d842d3e50", "mushroom12-d5e621787be67433", "mushroom13-27243fb877c60087", "tea towel-0c0d9b68216d515a", "mushroom14-d364ff7c7b325854", "plastic film2-fad40cfc20c2b737", "plastic film3-7bbcaf5c768a6d84", "lid of garlic jar-0f0f61129763916e", "saucepan-c67b477f737e7c26", "box of stock cubes-4e0c7cc7a0abce93", "stock cube-cd74c78656d35100", "another stock cube-9f0b56ed6905ca18", "circular metal hob extender-eadbd0fe20998fba", "large frying pan-b42ad6eb492130f5", "spoon1-17d37f7b53b21d74", "bottle of oil-e0d7e6dcf818d784", "wooden spatula-6584758d8278cd5f", "honey-af18e9af046893c4", "glass jar of rice-bb367015db827a88", "spoon2-2df81f56fd9fef2c", "rubbish-4aefab195305ce8a" ], "reasoning": "1. I watched the whole video and saw the user at 10:45 weigh the mushrooms from the second package of fresh mushrooms (mushrooms packet2-86d6e68560326de2). The weight was 320g.\n2. At 16:00, the user cut up the mushrooms from the third package of fresh mushrooms (mushrooms packet3-c3a2365d43481e80). The weight was 122g.\n3. The total weight of all the fresh mushrooms cut up is 442g.\n4. At 10:45, the user cut up 17 mushrooms from the second package of fresh mushrooms (mushrooms packet2-86d6e68560326de2).\n5. At 16:00, the user cut up 6 mushrooms from the third package of fresh mushrooms (mushrooms packet3-c3a2365d43481e80).\n6. The total number of mushrooms cut up from both packages is 17 + 6 = 23.\n7. So the weight of the total mushrooms , 442 divided by the total number of mushrooms cut up, 23, is 19.21.\n8. 19.21 raised to the second power (total number of fresh mushroom packages used is 369.02.", "referenced_object_masks": [ "mushrooms packet2-86d6e68560326de2", "mushrooms packet3-c3a2365d43481e80" ] }, { "key": "P07-20240529-191007:32298e8170eb8f0dc6da8369cfbb8aa44b0d5731", "video_id": "P07-20240529-191007", "question": "What is the total number of minutes and seconds that I stirred just the garlic and onions in the frying pan?", "answer_choice_0": "1:53", "answer_choice_1": "1:32", "answer_choice_2": "2:21", "answer_choice_3": "0:57", "answer_choice_4": "3:04", "answer_id": 0, "answer": "1:53", "question_type": [ "Action Recognition", "Event Occurence", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "piece of paper-bf8f784b82d48324", "pen-371a37932dee14bf", "smaller glass jug-f7d2085883de312c", "glass jug-8c241d892f6411a6", "digital weighing scales-ef7ce660d85df84f", "packet of mushrooms-014b68c23ae5eabd", "bowl-2ab0508662a40f03", "whole brown onion-cfe6a2f23f89b97c", "mushrooms-98766dba5a7ce407", "second pack of mushrooms-23d01cdd7f7657a3", "packet2-78ab138400b040fc", "cling film-21c626694a46e9a8", "knife-6f748ed0203c9641", "knife2-6114591711a1ccb4", "knife3-5101882187cfd3fd", "red onion-e424e8132fcc9536", "one half of brown onion-f703a1656bb28e95", "second half of brown onion-52545e711115ce73", "kettle-74a8b2a3ae189d8c", "fork-ae2d15829662b135", "phone-5dd4814328600726", "plastic packaging-bc1141ba01e9ba1e", "hummus-8da9a2fc6a7016a9", "jar of garlic-8bbd832082eb48c8", "small bowl-501a63422be3aaa4", "spoon-9183de8b5fcb802d", "clining film2-5b717f132de3399a", "lid-6ea5473e550a474c", "mushrooms packet1-00f6b4fca54b3c58", "mushrooms packet2-86d6e68560326de2", "mushrooms packet3-c3a2365d43481e80", "plastic packaging2-9fa24f4f9ce677f4", "plastic packaging3-8100bb72f8405ad5", "mushroom-4d76082e7b9ec768", "colander-8eb4057a30bdcc55", "baking tray-89ef6f170cea594f", "mushroom2-f9dcaf99333e2b39", "mushroom3-b08b1d06bdbbdef2", "mushroom4-400cd6d015491fee", "mushroom5-1aa477bdaee31f06", "mushroom6-d35733740873c64d", "mushroom7-2d6e4e16444edb63", "mushroom8-be20e4ae1a96e8b5", "mushroom9-c414fc887ed49a13", "mushroom10-946ede5d16786e7d", "mushroom11-4b284c2d842d3e50", "mushroom12-d5e621787be67433", "mushroom13-27243fb877c60087", "tea towel-0c0d9b68216d515a", "mushroom14-d364ff7c7b325854", "plastic film2-fad40cfc20c2b737", "plastic film3-7bbcaf5c768a6d84", "lid of garlic jar-0f0f61129763916e", "saucepan-c67b477f737e7c26", "box of stock cubes-4e0c7cc7a0abce93", "stock cube-cd74c78656d35100", "another stock cube-9f0b56ed6905ca18", "circular metal hob extender-eadbd0fe20998fba", "large frying pan-b42ad6eb492130f5", "spoon1-17d37f7b53b21d74", "bottle of oil-e0d7e6dcf818d784", "wooden spatula-6584758d8278cd5f", "honey-af18e9af046893c4", "glass jar of rice-bb367015db827a88", "spoon2-2df81f56fd9fef2c", "rubbish-4aefab195305ce8a" ], "reasoning": "1. I watched the whole video and saw the user dump the garlic into the pan (large frying pan-b42ad6eb492130f5) at 24:43 and then dumps the onions in the pan (large frying pan-b42ad6eb492130f5) at 24:55.\n2. Starting at 25:01, the user stirs the garlic and onions to 25:13, which is 12 seconds.\n3. Starting at 25:34, the user stirs the garlic and onions until 26:26, which is 52 seconds.\n4. Starting at 26:35 the user stirs the garlic and onions until 26:43, which is 8 seconds.\n5. Starting at 27:48, the user stirs the garlic and onions until 28:07, which is 19 seconds.\n6. Starting at 29:30, the user stirs the garlic and onions until 29:48, which is 18 seconds. \n7. Starting at 30:09, the user stirs the garlic and onions until 30:13, which is 4 seconds.\n8. Add up all the times the user spent stirring the garlic and onions : 12+52+8+19+18+4 is 113 seconds.\n9. Convert 113 seconds to minutes and seconds: 1 minutes and 53 seconds.", "referenced_object_masks": [ "large frying pan-b42ad6eb492130f5" ] }, { "key": "P07-20240529-191007:3f8c62c40b68df2c0099d695a4d8c5047dc5cc97", "video_id": "P07-20240529-191007", "question": "If I were to use the final total weight I wrote down for the raw, cut mushrooms I weighed, what is the average weight of each individual mushroom I cut when it is rounded to the nearest whole number?", "answer_choice_0": "19 units", "answer_choice_1": "10 units", "answer_choice_2": "23 units", "answer_choice_3": "26 units", "answer_choice_4": "14 units", "answer_id": 0, "answer": "19 units", "question_type": [ "Action Recognition", "Numerical Reasoning", "Object Recognition", "Counting", "Reading", "State Changes" ], "objects_of_interest": [ "piece of paper-bf8f784b82d48324", "pen-371a37932dee14bf", "smaller glass jug-f7d2085883de312c", "glass jug-8c241d892f6411a6", "digital weighing scales-ef7ce660d85df84f", "packet of mushrooms-014b68c23ae5eabd", "bowl-2ab0508662a40f03", "whole brown onion-cfe6a2f23f89b97c", "mushrooms-98766dba5a7ce407", "second pack of mushrooms-23d01cdd7f7657a3", "packet2-78ab138400b040fc", "cling film-21c626694a46e9a8", "knife-6f748ed0203c9641", "knife2-6114591711a1ccb4", "knife3-5101882187cfd3fd", "red onion-e424e8132fcc9536", "one half of brown onion-f703a1656bb28e95", "second half of brown onion-52545e711115ce73", "kettle-74a8b2a3ae189d8c", "fork-ae2d15829662b135", "phone-5dd4814328600726", "plastic packaging-bc1141ba01e9ba1e", "hummus-8da9a2fc6a7016a9", "jar of garlic-8bbd832082eb48c8", "small bowl-501a63422be3aaa4", "spoon-9183de8b5fcb802d", "clining film2-5b717f132de3399a", "lid-6ea5473e550a474c", "mushrooms packet1-00f6b4fca54b3c58", "mushrooms packet2-86d6e68560326de2", "mushrooms packet3-c3a2365d43481e80", "plastic packaging2-9fa24f4f9ce677f4", "plastic packaging3-8100bb72f8405ad5", "mushroom-4d76082e7b9ec768", "colander-8eb4057a30bdcc55", "baking tray-89ef6f170cea594f", "mushroom2-f9dcaf99333e2b39", "mushroom3-b08b1d06bdbbdef2", "mushroom4-400cd6d015491fee", "mushroom5-1aa477bdaee31f06", "mushroom6-d35733740873c64d", "mushroom7-2d6e4e16444edb63", "mushroom8-be20e4ae1a96e8b5", "mushroom9-c414fc887ed49a13", "mushroom10-946ede5d16786e7d", "mushroom11-4b284c2d842d3e50", "mushroom12-d5e621787be67433", "mushroom13-27243fb877c60087", "tea towel-0c0d9b68216d515a", "mushroom14-d364ff7c7b325854", "plastic film2-fad40cfc20c2b737", "plastic film3-7bbcaf5c768a6d84", "lid of garlic jar-0f0f61129763916e", "saucepan-c67b477f737e7c26", "box of stock cubes-4e0c7cc7a0abce93", "stock cube-cd74c78656d35100", "another stock cube-9f0b56ed6905ca18", "circular metal hob extender-eadbd0fe20998fba", "large frying pan-b42ad6eb492130f5", "spoon1-17d37f7b53b21d74", "bottle of oil-e0d7e6dcf818d784", "wooden spatula-6584758d8278cd5f", "honey-af18e9af046893c4", "glass jar of rice-bb367015db827a88", "spoon2-2df81f56fd9fef2c", "rubbish-4aefab195305ce8a" ], "reasoning": "1. First, I looked for the user to cut and weigh mushrooms on the scale (digital weighing scales-ef7ce660d85df84f). I saw them do this from 10:47-14:48.\n2. Next, I counted up the number of individual mushrooms the user weighs. The user cuts the first mushroom (mushroom14-d364ff7c7b325854) at 10:47, the second at 11:13, the third at 11:33, the fourth at 11:51, the fifth at 12:02, the sixth at 12:19, the seventh at 12:47, the eighth at 13:00, the ninth at 13:08, the tenth at 13:18, the eleventh at 13:43, the twelfth at 13:55, the thirteenth at 14:05, the fourteenth at 14:18, the fifteenth at 14:33, the sixteenth at 14:41, and the seventeenth at 14:48.\n3. Next, I looked for the user to write down a total final weight for the cut mushrooms . At 15:36, I saw that the user had placed all the mushrooms in the paper box (mushrooms packet3-c3a2365d43481e80) on the scale (digital weighing scales-ef7ce660d85df84f) and that the scale (digital weighing scales-ef7ce660d85df84f) read 320. I saw the user turn to write this number down on a notepad (piece of paper-bf8f784b82d48324) at 15:39. \n4. Using the number 320 as the final total weight for the cut mushrooms , I then divided 320 by 17 for a total of 18.8 units, which I rounded to the nearest whole number for a final answer of 19.", "referenced_object_masks": [ "piece of paper-bf8f784b82d48324", "mushroom14-d364ff7c7b325854", "mushrooms packet3-c3a2365d43481e80", "digital weighing scales-ef7ce660d85df84f" ] }, { "key": "P07-20240529-191007:70a4875d99cc97e37b5920d8051520a5eba1c55d", "video_id": "P07-20240529-191007", "question": "What do the animals on the microwave and the animal on the spaghetti container have in common?", "answer_choice_0": "They both have horns", "answer_choice_1": "They both can fly", "answer_choice_2": "They are both white", "answer_choice_3": "They are both mammals", "answer_choice_4": "They are both land animals", "answer_id": 4, "answer": "They are both land animals", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "piece of paper-bf8f784b82d48324", "pen-371a37932dee14bf", "smaller glass jug-f7d2085883de312c", "glass jug-8c241d892f6411a6", "digital weighing scales-ef7ce660d85df84f", "packet of mushrooms-014b68c23ae5eabd", "bowl-2ab0508662a40f03", "whole brown onion-cfe6a2f23f89b97c", "mushrooms-98766dba5a7ce407", "second pack of mushrooms-23d01cdd7f7657a3", "packet2-78ab138400b040fc", "cling film-21c626694a46e9a8", "knife-6f748ed0203c9641", "knife2-6114591711a1ccb4", "knife3-5101882187cfd3fd", "red onion-e424e8132fcc9536", "one half of brown onion-f703a1656bb28e95", "second half of brown onion-52545e711115ce73", "kettle-74a8b2a3ae189d8c", "fork-ae2d15829662b135", "phone-5dd4814328600726", "plastic packaging-bc1141ba01e9ba1e", "hummus-8da9a2fc6a7016a9", "jar of garlic-8bbd832082eb48c8", "small bowl-501a63422be3aaa4", "spoon-9183de8b5fcb802d", "clining film2-5b717f132de3399a", "lid-6ea5473e550a474c", "mushrooms packet1-00f6b4fca54b3c58", "mushrooms packet2-86d6e68560326de2", "mushrooms packet3-c3a2365d43481e80", "plastic packaging2-9fa24f4f9ce677f4", "plastic packaging3-8100bb72f8405ad5", "mushroom-4d76082e7b9ec768", "colander-8eb4057a30bdcc55", "baking tray-89ef6f170cea594f", "mushroom2-f9dcaf99333e2b39", "mushroom3-b08b1d06bdbbdef2", "mushroom4-400cd6d015491fee", "mushroom5-1aa477bdaee31f06", "mushroom6-d35733740873c64d", "mushroom7-2d6e4e16444edb63", "mushroom8-be20e4ae1a96e8b5", "mushroom9-c414fc887ed49a13", "mushroom10-946ede5d16786e7d", "mushroom11-4b284c2d842d3e50", "mushroom12-d5e621787be67433", "mushroom13-27243fb877c60087", "tea towel-0c0d9b68216d515a", "mushroom14-d364ff7c7b325854", "plastic film2-fad40cfc20c2b737", "plastic film3-7bbcaf5c768a6d84", "lid of garlic jar-0f0f61129763916e", "saucepan-c67b477f737e7c26", "box of stock cubes-4e0c7cc7a0abce93", "stock cube-cd74c78656d35100", "another stock cube-9f0b56ed6905ca18", "circular metal hob extender-eadbd0fe20998fba", "large frying pan-b42ad6eb492130f5", "spoon1-17d37f7b53b21d74", "bottle of oil-e0d7e6dcf818d784", "wooden spatula-6584758d8278cd5f", "honey-af18e9af046893c4", "glass jar of rice-bb367015db827a88", "spoon2-2df81f56fd9fef2c", "rubbish-4aefab195305ce8a" ], "reasoning": "1. I look for a decent view of the microwave , and I find it at 00:08, when the user looks left toward it.\n2. I notice 2 chickens on the microwave .\n3. I look for a decent view of the spaghetti container and find it at 00:06, before the user has looked left.\n4. I notice a small, long-necked dinosaur on the spaghetti container .\n5. The chickens and the long-necked dinosaur are both land animals, while also not both being mammals, white, able to fly, or growing horns.\n6. Therefore, land animals is the answer.", "referenced_object_masks": [] }, { "key": "P07-20240529-191007:a1b2f9ab95df8a7ac982d79c9701f2a972daa21c", "video_id": "P07-20240529-191007", "question": "What are the three textures in chronological order of the dried mushrooms from the pink package?", "answer_choice_0": "Soft and dry, soaked, softened by saut\u00e9ing", "answer_choice_1": "Dry and hard, soaked, softened by saut\u00e9ing", "answer_choice_2": "Soft and dry, soaked, dried again", "answer_choice_3": "Dry and hard, soaked, dried again", "answer_choice_4": "Dry and hard, softened by saut\u00e9ing, soaked", "answer_id": 1, "answer": "Dry and hard, soaked, softened by saut\u00e9ing", "question_type": [ "Action Recognition", "Event Occurence", "Object Recognition", "State Changes", "Temporal Reasoning" ], "objects_of_interest": [ "piece of paper-bf8f784b82d48324", "pen-371a37932dee14bf", "smaller glass jug-f7d2085883de312c", "glass jug-8c241d892f6411a6", "digital weighing scales-ef7ce660d85df84f", "packet of mushrooms-014b68c23ae5eabd", "bowl-2ab0508662a40f03", "whole brown onion-cfe6a2f23f89b97c", "mushrooms-98766dba5a7ce407", "second pack of mushrooms-23d01cdd7f7657a3", "packet2-78ab138400b040fc", "cling film-21c626694a46e9a8", "knife-6f748ed0203c9641", "knife2-6114591711a1ccb4", "knife3-5101882187cfd3fd", "red onion-e424e8132fcc9536", "one half of brown onion-f703a1656bb28e95", "second half of brown onion-52545e711115ce73", "kettle-74a8b2a3ae189d8c", "fork-ae2d15829662b135", "phone-5dd4814328600726", "plastic packaging-bc1141ba01e9ba1e", "hummus-8da9a2fc6a7016a9", "jar of garlic-8bbd832082eb48c8", "small bowl-501a63422be3aaa4", "spoon-9183de8b5fcb802d", "clining film2-5b717f132de3399a", "lid-6ea5473e550a474c", "mushrooms packet1-00f6b4fca54b3c58", "mushrooms packet2-86d6e68560326de2", "mushrooms packet3-c3a2365d43481e80", "plastic packaging2-9fa24f4f9ce677f4", "plastic packaging3-8100bb72f8405ad5", "mushroom-4d76082e7b9ec768", "colander-8eb4057a30bdcc55", "baking tray-89ef6f170cea594f", "mushroom2-f9dcaf99333e2b39", "mushroom3-b08b1d06bdbbdef2", "mushroom4-400cd6d015491fee", "mushroom5-1aa477bdaee31f06", "mushroom6-d35733740873c64d", "mushroom7-2d6e4e16444edb63", "mushroom8-be20e4ae1a96e8b5", "mushroom9-c414fc887ed49a13", "mushroom10-946ede5d16786e7d", "mushroom11-4b284c2d842d3e50", "mushroom12-d5e621787be67433", "mushroom13-27243fb877c60087", "tea towel-0c0d9b68216d515a", "mushroom14-d364ff7c7b325854", "plastic film2-fad40cfc20c2b737", "plastic film3-7bbcaf5c768a6d84", "lid of garlic jar-0f0f61129763916e", "saucepan-c67b477f737e7c26", "box of stock cubes-4e0c7cc7a0abce93", "stock cube-cd74c78656d35100", "another stock cube-9f0b56ed6905ca18", "circular metal hob extender-eadbd0fe20998fba", "large frying pan-b42ad6eb492130f5", "spoon1-17d37f7b53b21d74", "bottle of oil-e0d7e6dcf818d784", "wooden spatula-6584758d8278cd5f", "honey-af18e9af046893c4", "glass jar of rice-bb367015db827a88", "spoon2-2df81f56fd9fef2c", "rubbish-4aefab195305ce8a" ], "reasoning": "1. I watched the whole video and saw the user pour the dried mushrooms from a pink package (packet of mushrooms-014b68c23ae5eabd) into a glass measuring cup (glass jug-8c241d892f6411a6) at 01:07.\n2. At 01:20, the user put the glass measuring cup (glass jug-8c241d892f6411a6) on the counter to the left of the sink .\n3. At 05:50, the user poured hot water from a tea kettle (kettle-74a8b2a3ae189d8c) into the measuring cup (glass jug-8c241d892f6411a6) to soften the dried mushroom contents .\n4. At 22:01, the dried mushroom contents had soaked up all the water. The user began to press the mushrooms with a fork (fork-ae2d15829662b135) to ensure all the water had been absorbed.\n5. At 30:21, the user poured the mushrooms in the glass measuring cup (glass jug-8c241d892f6411a6) into the frying pan (large frying pan-b42ad6eb492130f5) pan where the garlic and onions were being saut\u00e9d. \n6. The three textures are dry and hard, softening by soaking up water, and softening by saut\u00e9ing.", "referenced_object_masks": [ "fork-ae2d15829662b135", "packet of mushrooms-014b68c23ae5eabd", "glass jug-8c241d892f6411a6", "kettle-74a8b2a3ae189d8c", "large frying pan-b42ad6eb492130f5" ] }, { "key": "P07-20240529-191007:b89bfd0c23727f3ddaa1e84bc7241a7571ab3c2f", "video_id": "P07-20240529-191007", "question": "If I wanted to throw the empty mixed mushroom packet into the trash and I was currently facing the last place I put it, where would the trash be located if I were to use clock bearing as a reference point?", "answer_choice_0": "1 o'clock", "answer_choice_1": "9 o'clock", "answer_choice_2": "12 o' clock", "answer_choice_3": "7 o'clock", "answer_choice_4": "4 o' clock", "answer_id": 4, "answer": "4 o' clock", "question_type": [ "Egocentric & Spatial Perception", "Situational Awareness", "Temporal Reasoning", "Object Recognition", "Counterfactual" ], "objects_of_interest": [ "piece of paper-bf8f784b82d48324", "pen-371a37932dee14bf", "smaller glass jug-f7d2085883de312c", "glass jug-8c241d892f6411a6", "digital weighing scales-ef7ce660d85df84f", "packet of mushrooms-014b68c23ae5eabd", "bowl-2ab0508662a40f03", "whole brown onion-cfe6a2f23f89b97c", "mushrooms-98766dba5a7ce407", "second pack of mushrooms-23d01cdd7f7657a3", "packet2-78ab138400b040fc", "cling film-21c626694a46e9a8", "knife-6f748ed0203c9641", "knife2-6114591711a1ccb4", "knife3-5101882187cfd3fd", "red onion-e424e8132fcc9536", "one half of brown onion-f703a1656bb28e95", "second half of brown onion-52545e711115ce73", "kettle-74a8b2a3ae189d8c", "fork-ae2d15829662b135", "phone-5dd4814328600726", "plastic packaging-bc1141ba01e9ba1e", "hummus-8da9a2fc6a7016a9", "jar of garlic-8bbd832082eb48c8", "small bowl-501a63422be3aaa4", "spoon-9183de8b5fcb802d", "clining film2-5b717f132de3399a", "lid-6ea5473e550a474c", "mushrooms packet1-00f6b4fca54b3c58", "mushrooms packet2-86d6e68560326de2", "mushrooms packet3-c3a2365d43481e80", "plastic packaging2-9fa24f4f9ce677f4", "plastic packaging3-8100bb72f8405ad5", "mushroom-4d76082e7b9ec768", "colander-8eb4057a30bdcc55", "baking tray-89ef6f170cea594f", "mushroom2-f9dcaf99333e2b39", "mushroom3-b08b1d06bdbbdef2", "mushroom4-400cd6d015491fee", "mushroom5-1aa477bdaee31f06", "mushroom6-d35733740873c64d", "mushroom7-2d6e4e16444edb63", "mushroom8-be20e4ae1a96e8b5", "mushroom9-c414fc887ed49a13", "mushroom10-946ede5d16786e7d", "mushroom11-4b284c2d842d3e50", "mushroom12-d5e621787be67433", "mushroom13-27243fb877c60087", "tea towel-0c0d9b68216d515a", "mushroom14-d364ff7c7b325854", "plastic film2-fad40cfc20c2b737", "plastic film3-7bbcaf5c768a6d84", "lid of garlic jar-0f0f61129763916e", "saucepan-c67b477f737e7c26", "box of stock cubes-4e0c7cc7a0abce93", "stock cube-cd74c78656d35100", "another stock cube-9f0b56ed6905ca18", "circular metal hob extender-eadbd0fe20998fba", "large frying pan-b42ad6eb492130f5", "spoon1-17d37f7b53b21d74", "bottle of oil-e0d7e6dcf818d784", "wooden spatula-6584758d8278cd5f", "honey-af18e9af046893c4", "glass jar of rice-bb367015db827a88", "spoon2-2df81f56fd9fef2c", "rubbish-4aefab195305ce8a" ], "reasoning": "1. I look for the mixed mushroom packet (packet of mushrooms-014b68c23ae5eabd) and I first find it at 00:30, when the user takes it (packet of mushrooms-014b68c23ae5eabd) out of a cupboard.\n2. The packet (packet of mushrooms-014b68c23ae5eabd) itself remains near or under the cutting board for most of the time the user spends cooking. At 29:14, it (packet of mushrooms-014b68c23ae5eabd) is still wedged under the cutting board .\n3. At 29:15, however, the user grabs the packet (packet of mushrooms-014b68c23ae5eabd), and at 29:21, the user places it (packet of mushrooms-014b68c23ae5eabd) in the top drawer on the kitchen island.\n4. After this, the packet (packet of mushrooms-014b68c23ae5eabd) is no longer seen.\n5. I look for the trash , and I find that it is a white bin beside the fridge at the very beginning of the video, at 00:00.\n6. I realize that, if I wanted to throw the mushroom packet (packet of mushrooms-014b68c23ae5eabd) away, I would need turn rotate right about 120 degrees to face the trash .\n7. As this corresponds to 4 o'clock in clock bearing, this is my answer.", "referenced_object_masks": [ "packet of mushrooms-014b68c23ae5eabd" ] }, { "key": "P07-20240529-191007:c3663aaea410d9c34804c846cb8599ce06b0ea3e", "video_id": "P07-20240529-191007", "question": "If I am at 12 o'clock at 17:55, what o'clock is the trash can?", "answer_choice_0": "6 o'clock", "answer_choice_1": "8 o'clock", "answer_choice_2": "5 o'clock", "answer_choice_3": "7 o'clock", "answer_choice_4": "4 o'clock", "answer_id": 0, "answer": "6 o'clock", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Event Occurence", "Numerical Reasoning", "Temporal Reasoning" ], "objects_of_interest": [ "piece of paper-bf8f784b82d48324", "pen-371a37932dee14bf", "smaller glass jug-f7d2085883de312c", "glass jug-8c241d892f6411a6", "digital weighing scales-ef7ce660d85df84f", "packet of mushrooms-014b68c23ae5eabd", "bowl-2ab0508662a40f03", "whole brown onion-cfe6a2f23f89b97c", "mushrooms-98766dba5a7ce407", "second pack of mushrooms-23d01cdd7f7657a3", "packet2-78ab138400b040fc", "cling film-21c626694a46e9a8", "knife-6f748ed0203c9641", "knife2-6114591711a1ccb4", "knife3-5101882187cfd3fd", "red onion-e424e8132fcc9536", "one half of brown onion-f703a1656bb28e95", "second half of brown onion-52545e711115ce73", "kettle-74a8b2a3ae189d8c", "fork-ae2d15829662b135", "phone-5dd4814328600726", "plastic packaging-bc1141ba01e9ba1e", "hummus-8da9a2fc6a7016a9", "jar of garlic-8bbd832082eb48c8", "small bowl-501a63422be3aaa4", "spoon-9183de8b5fcb802d", "clining film2-5b717f132de3399a", "lid-6ea5473e550a474c", "mushrooms packet1-00f6b4fca54b3c58", "mushrooms packet2-86d6e68560326de2", "mushrooms packet3-c3a2365d43481e80", "plastic packaging2-9fa24f4f9ce677f4", "plastic packaging3-8100bb72f8405ad5", "mushroom-4d76082e7b9ec768", "colander-8eb4057a30bdcc55", "baking tray-89ef6f170cea594f", "mushroom2-f9dcaf99333e2b39", "mushroom3-b08b1d06bdbbdef2", "mushroom4-400cd6d015491fee", "mushroom5-1aa477bdaee31f06", "mushroom6-d35733740873c64d", "mushroom7-2d6e4e16444edb63", "mushroom8-be20e4ae1a96e8b5", "mushroom9-c414fc887ed49a13", "mushroom10-946ede5d16786e7d", "mushroom11-4b284c2d842d3e50", "mushroom12-d5e621787be67433", "mushroom13-27243fb877c60087", "tea towel-0c0d9b68216d515a", "mushroom14-d364ff7c7b325854", "plastic film2-fad40cfc20c2b737", "plastic film3-7bbcaf5c768a6d84", "lid of garlic jar-0f0f61129763916e", "saucepan-c67b477f737e7c26", "box of stock cubes-4e0c7cc7a0abce93", "stock cube-cd74c78656d35100", "another stock cube-9f0b56ed6905ca18", "circular metal hob extender-eadbd0fe20998fba", "large frying pan-b42ad6eb492130f5", "spoon1-17d37f7b53b21d74", "bottle of oil-e0d7e6dcf818d784", "wooden spatula-6584758d8278cd5f", "honey-af18e9af046893c4", "glass jar of rice-bb367015db827a88", "spoon2-2df81f56fd9fef2c", "rubbish-4aefab195305ce8a" ], "reasoning": "1. At 17:55, the user is at the counter to the left of the sink updating the piece of paper (piece of paper-bf8f784b82d48324).\n2. At 18:05, the user turns around from the counter and walks straight back to the trash can (rubbish-4aefab195305ce8a).\n3. If the user is at 12 o'clock, and the user walks straight back to the trash can (rubbish-4aefab195305ce8a), the trash can (rubbish-4aefab195305ce8a) is at 6 o'clock", "referenced_object_masks": [ "rubbish-4aefab195305ce8a", "piece of paper-bf8f784b82d48324" ] }, { "key": "P07-20240529-191007:c5e00c3d1d0666eefece0232b3af63cfaeec8726", "video_id": "P07-20240529-191007", "question": "If I were to duplicate myself and stare out the window while my clone carried the old and wrinkled mushrooms and onion skins from its place by the stovetop to the compost bin, how would my clone pass me by?", "answer_choice_0": "From my left to my right", "answer_choice_1": "From my right to my left", "answer_choice_2": "From in front of me to behind me", "answer_choice_3": "From behind me to in front of me", "answer_choice_4": "The clone wouldn't cross my path", "answer_id": 0, "answer": "From my left to my right", "question_type": [ "Counterfactual", "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness", "Action Recognition" ], "objects_of_interest": [ "piece of paper-bf8f784b82d48324", "pen-371a37932dee14bf", "smaller glass jug-f7d2085883de312c", "glass jug-8c241d892f6411a6", "digital weighing scales-ef7ce660d85df84f", "packet of mushrooms-014b68c23ae5eabd", "bowl-2ab0508662a40f03", "whole brown onion-cfe6a2f23f89b97c", "mushrooms-98766dba5a7ce407", "second pack of mushrooms-23d01cdd7f7657a3", "packet2-78ab138400b040fc", "cling film-21c626694a46e9a8", "knife-6f748ed0203c9641", "knife2-6114591711a1ccb4", "knife3-5101882187cfd3fd", "red onion-e424e8132fcc9536", "one half of brown onion-f703a1656bb28e95", "second half of brown onion-52545e711115ce73", "kettle-74a8b2a3ae189d8c", "fork-ae2d15829662b135", "phone-5dd4814328600726", "plastic packaging-bc1141ba01e9ba1e", "hummus-8da9a2fc6a7016a9", "jar of garlic-8bbd832082eb48c8", "small bowl-501a63422be3aaa4", "spoon-9183de8b5fcb802d", "clining film2-5b717f132de3399a", "lid-6ea5473e550a474c", "mushrooms packet1-00f6b4fca54b3c58", "mushrooms packet2-86d6e68560326de2", "mushrooms packet3-c3a2365d43481e80", "plastic packaging2-9fa24f4f9ce677f4", "plastic packaging3-8100bb72f8405ad5", "mushroom-4d76082e7b9ec768", "colander-8eb4057a30bdcc55", "baking tray-89ef6f170cea594f", "mushroom2-f9dcaf99333e2b39", "mushroom3-b08b1d06bdbbdef2", "mushroom4-400cd6d015491fee", "mushroom5-1aa477bdaee31f06", "mushroom6-d35733740873c64d", "mushroom7-2d6e4e16444edb63", "mushroom8-be20e4ae1a96e8b5", "mushroom9-c414fc887ed49a13", "mushroom10-946ede5d16786e7d", "mushroom11-4b284c2d842d3e50", "mushroom12-d5e621787be67433", "mushroom13-27243fb877c60087", "tea towel-0c0d9b68216d515a", "mushroom14-d364ff7c7b325854", "plastic film2-fad40cfc20c2b737", "plastic film3-7bbcaf5c768a6d84", "lid of garlic jar-0f0f61129763916e", "saucepan-c67b477f737e7c26", "box of stock cubes-4e0c7cc7a0abce93", "stock cube-cd74c78656d35100", "another stock cube-9f0b56ed6905ca18", "circular metal hob extender-eadbd0fe20998fba", "large frying pan-b42ad6eb492130f5", "spoon1-17d37f7b53b21d74", "bottle of oil-e0d7e6dcf818d784", "wooden spatula-6584758d8278cd5f", "honey-af18e9af046893c4", "glass jar of rice-bb367015db827a88", "spoon2-2df81f56fd9fef2c", "rubbish-4aefab195305ce8a" ], "reasoning": "1. First, I look for the kitchen window. I see a good view of it at 00:03, when the user walks into the kitchen the first time. It is located at the end of the kitchen behind the sink and between white cupboards.\n2. Next, I try to find the time the user took expired mushrooms and some onion skins in the compost bin.\n3. From 08:38-08:42, the user peels plastic wrapping (plastic packaging2-9fa24f4f9ce677f4) off a box of mushrooms (mushrooms packet1-00f6b4fca54b3c58). I notice that this box of mushrooms (mushrooms packet1-00f6b4fca54b3c58) is close to the stovetop. From 08:43-8:56, the user takes the mushrooms out of the box (mushrooms packet1-00f6b4fca54b3c58) and examines them. The mushrooms look old and wrinkled. From 08:56-09:18, the user, still holding the old mushrooms , gathers up onion skins and turns right to throw them away in the compost bin . I realize this is the moment I am looking for.\n4. I notice the compost bin is to the right of the drying rack , which is to the right of the sink, and, as mentioned before, the window is behind the sink. Indeed, from 09:12-09:14, the user passes the window on the lefthand side to get to the bin .\n5. If the user were to duplicate themselves and stare out the kitchen window while the clone traveled from the stovetop to the compost bin , the clone would be starting their journey on the lefthand side of the user and moving to the righthand side.\n6. Therefore, the answer is from the left of the user to the right.", "referenced_object_masks": [ "mushrooms packet1-00f6b4fca54b3c58", "plastic packaging2-9fa24f4f9ce677f4" ] }, { "key": "P07-20240529-194518:060f3a1b79331f3e6eb6744d138d16b04f9d8646", "video_id": "P07-20240529-194518", "question": "What number would I get if I multiplied the number of plant pots behind the sink by the number of times I rotate the wine bottle with my right hand to open the lid?", "answer_choice_0": "18", "answer_choice_1": "24", "answer_choice_2": "12", "answer_choice_3": "3", "answer_choice_4": "15", "answer_id": 2, "answer": "12", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counting", "Numerical Reasoning" ], "objects_of_interest": [ "wooden spatula-6a45cdf23bdd86d3", "frying pan-e144761e0ca28fbd", "saucepan-27b411bed31173de", "salt pig-7d76653044d55e3a", "pepper grinder-6c10b1dd9e13fdd5", "glass jar-dca3a59f4d09da0e", "bottle of wine-a5fc00bc00b59182", "lid of wine bottle-809f9516dc8991a8", "bowl-875ab2ef3779249c", "small bowl-a6b386e6bab2cc3b", "bulb-3c2ffffcc59866ef", "small object-9c5b32caf41ea7b5", "some object2-ae2b47e8e2dc1c74", "obj3-73f440c4ab148e8f", "glass jug2-73be4fae1e24e333", "spoon-633d244e722330bd", "fork-0fd2505f0727337d", "cheese block-c51aedd3e2b3e165", "digital scales-f73155d16b30fe0c", "scissors-f798ee0fa140ca95", "kitchen roll-086d32975371ef2f", "piece of kitchen roll-4643e3df23416793", "small block of cheese-c84866fd736372f8", "another very small piece of cheese-005b510257a37712", "pen-ee28a0d8efa50489", "piece of cheese3-3434adfcda47303d", "thin slice of cheese-2706430eeb82d855", "thin slice2-e5d7eeba01287985", "cheese piece5-4f55ec4bdb987d42", "empty cheese packaging-b933473a293976e0", "tea towel-36f36c206bbaa0a0", "cheese grater-ffd963d630712759", "bowl2-76300137c657f7a3", "bottle of oil-6a5d6e233495046b", "piece of paper-61a7cf79e6a59550", "tupperware container-c42614210387dc0c", "piece of plastic packaging-1a1fb9e272d46248", "piece of kitchen roll2-c5e0f2b43efa74a2", "glass of water-40dcf7749a8c8a15", "colander-559f5587ae532369", "dish washing liquid-a06c01a8dc14c1d2", "brush-6bec578509d79feb", "knife-b23d235095c6e134", "baking tray-51f814dae4bf8486", "can opener-60170d705d267c35", "measuring spoons-ba74de7b24c6307b", "cloth-3d3cecdbf2b9490c", "phone-78c9ee47cab95cb1", "vacuum cleaner-c5c51437c3e8423f", "bowl3-703f3de39d983585", "knife2-82be27931bf65f28", "cover of butter dish-dd025665fe270be9", "lettuce bag-53344fda660944d9", "small bowl2-db3ecb3538ad5811", "spoon2-c9079f3e7df98043", "knife3-9827aa537ed14927", "teaspoon-6a23ed225f92fd9b" ], "reasoning": "1. I locate the sink across the room at 00:00.\n2. At 02:08, I count 6 plant pots in front of the sink on the counter in front of the user. \n3. At 02:20, I observe the user take a wine bottle (bottle of wine-a5fc00bc00b59182) from the fridge .\n4. From 02:24 to 02:27, I observe the user open the wine bottle (bottle of wine-a5fc00bc00b59182) and remove the lid (lid of wine bottle-809f9516dc8991a8).\n5. From 02:24 to 02:27, I observe the user rotate the wine bottle (bottle of wine-a5fc00bc00b59182) 2 times with their right hand.\n6. I multiply 6 by 2 and get 12.", "referenced_object_masks": [ "lid of wine bottle-809f9516dc8991a8", "bottle of wine-a5fc00bc00b59182" ] }, { "key": "P07-20240529-194518:19bae3650e3db8b3f663fe381adfcc7354c47955", "video_id": "P07-20240529-194518", "question": "Before weighing the cheese on the digital scale for the first time, how many more times do I press a button with my right hand than with my left hand?", "answer_choice_0": "3", "answer_choice_1": "5", "answer_choice_2": "4", "answer_choice_3": "1", "answer_choice_4": "2", "answer_id": 0, "answer": "3", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counting", "Numerical Reasoning" ], "objects_of_interest": [ "wooden spatula-6a45cdf23bdd86d3", "frying pan-e144761e0ca28fbd", "saucepan-27b411bed31173de", "salt pig-7d76653044d55e3a", "pepper grinder-6c10b1dd9e13fdd5", "glass jar-dca3a59f4d09da0e", "bottle of wine-a5fc00bc00b59182", "lid of wine bottle-809f9516dc8991a8", "bowl-875ab2ef3779249c", "small bowl-a6b386e6bab2cc3b", "bulb-3c2ffffcc59866ef", "small object-9c5b32caf41ea7b5", "some object2-ae2b47e8e2dc1c74", "obj3-73f440c4ab148e8f", "glass jug2-73be4fae1e24e333", "spoon-633d244e722330bd", "fork-0fd2505f0727337d", "cheese block-c51aedd3e2b3e165", "digital scales-f73155d16b30fe0c", "scissors-f798ee0fa140ca95", "kitchen roll-086d32975371ef2f", "piece of kitchen roll-4643e3df23416793", "small block of cheese-c84866fd736372f8", "another very small piece of cheese-005b510257a37712", "pen-ee28a0d8efa50489", "piece of cheese3-3434adfcda47303d", "thin slice of cheese-2706430eeb82d855", "thin slice2-e5d7eeba01287985", "cheese piece5-4f55ec4bdb987d42", "empty cheese packaging-b933473a293976e0", "tea towel-36f36c206bbaa0a0", "cheese grater-ffd963d630712759", "bowl2-76300137c657f7a3", "bottle of oil-6a5d6e233495046b", "piece of paper-61a7cf79e6a59550", "tupperware container-c42614210387dc0c", "piece of plastic packaging-1a1fb9e272d46248", "piece of kitchen roll2-c5e0f2b43efa74a2", "glass of water-40dcf7749a8c8a15", "colander-559f5587ae532369", "dish washing liquid-a06c01a8dc14c1d2", "brush-6bec578509d79feb", "knife-b23d235095c6e134", "baking tray-51f814dae4bf8486", "can opener-60170d705d267c35", "measuring spoons-ba74de7b24c6307b", "cloth-3d3cecdbf2b9490c", "phone-78c9ee47cab95cb1", "vacuum cleaner-c5c51437c3e8423f", "bowl3-703f3de39d983585", "knife2-82be27931bf65f28", "cover of butter dish-dd025665fe270be9", "lettuce bag-53344fda660944d9", "small bowl2-db3ecb3538ad5811", "spoon2-c9079f3e7df98043", "knife3-9827aa537ed14927", "teaspoon-6a23ed225f92fd9b" ], "reasoning": "1. At 08:17, I observe the user removing a block of cheese (cheese block-c51aedd3e2b3e165) from the fridge . \n2. At 08:20, I observe the user picking up the digital scale (digital scales-f73155d16b30fe0c) from the counter in front of them.\n3. At 08:22, I observe the user placing the digital scale (digital scales-f73155d16b30fe0c) to the left of the piece of paper (piece of paper-61a7cf79e6a59550) on the counter.\n4. At 08:22, I observe the user turning on the digital scale (digital scales-f73155d16b30fe0c) by pressing a button with their right pointer finger 1 time.\n5. Between 09:02 and 09:06, I observe the user press a button on the digital scale (digital scales-f73155d16b30fe0c) with their right pointer finger 4 times.\n6. Between 09:07 and 09:10, I observe the user press a button on the digital scale (digital scales-f73155d16b30fe0c) with their left pointer finger 2 times.\n7. At 09:11, I observe the user place a piece of cheese (small block of cheese-c84866fd736372f8) on the digital scale (digital scales-f73155d16b30fe0c) to weigh it. \n8. I determine the user pressed a button on the digital scale (digital scales-f73155d16b30fe0c) with their right pointer finger 5 times.\n9. I determine the user pressed a button on the digital scale (digital scales-f73155d16b30fe0c) with their left pointer finger 2 times.\n10. I subtract 2 from 5 to get 3.", "referenced_object_masks": [ "small block of cheese-c84866fd736372f8", "cheese block-c51aedd3e2b3e165", "piece of paper-61a7cf79e6a59550", "digital scales-f73155d16b30fe0c" ] }, { "key": "P07-20240529-194518:343ce0e8bf16acab742f1d195b777cde5c99be97", "video_id": "P07-20240529-194518", "question": "What is the last object I touch before opening the fridge for the first time?", "answer_choice_0": "Glass jar", "answer_choice_1": "Pepper grinder", "answer_choice_2": "Wine bottle", "answer_choice_3": "Saucepan", "answer_choice_4": "Salt pig", "answer_id": 0, "answer": "Glass jar", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "wooden spatula-6a45cdf23bdd86d3", "frying pan-e144761e0ca28fbd", "saucepan-27b411bed31173de", "salt pig-7d76653044d55e3a", "pepper grinder-6c10b1dd9e13fdd5", "glass jar-dca3a59f4d09da0e", "bottle of wine-a5fc00bc00b59182", "lid of wine bottle-809f9516dc8991a8", "bowl-875ab2ef3779249c", "small bowl-a6b386e6bab2cc3b", "bulb-3c2ffffcc59866ef", "small object-9c5b32caf41ea7b5", "some object2-ae2b47e8e2dc1c74", "obj3-73f440c4ab148e8f", "glass jug2-73be4fae1e24e333", "spoon-633d244e722330bd", "fork-0fd2505f0727337d", "cheese block-c51aedd3e2b3e165", "digital scales-f73155d16b30fe0c", "scissors-f798ee0fa140ca95", "kitchen roll-086d32975371ef2f", "piece of kitchen roll-4643e3df23416793", "small block of cheese-c84866fd736372f8", "another very small piece of cheese-005b510257a37712", "pen-ee28a0d8efa50489", "piece of cheese3-3434adfcda47303d", "thin slice of cheese-2706430eeb82d855", "thin slice2-e5d7eeba01287985", "cheese piece5-4f55ec4bdb987d42", "empty cheese packaging-b933473a293976e0", "tea towel-36f36c206bbaa0a0", "cheese grater-ffd963d630712759", "bowl2-76300137c657f7a3", "bottle of oil-6a5d6e233495046b", "piece of paper-61a7cf79e6a59550", "tupperware container-c42614210387dc0c", "piece of plastic packaging-1a1fb9e272d46248", "piece of kitchen roll2-c5e0f2b43efa74a2", "glass of water-40dcf7749a8c8a15", "colander-559f5587ae532369", "dish washing liquid-a06c01a8dc14c1d2", "brush-6bec578509d79feb", "knife-b23d235095c6e134", "baking tray-51f814dae4bf8486", "can opener-60170d705d267c35", "measuring spoons-ba74de7b24c6307b", "cloth-3d3cecdbf2b9490c", "phone-78c9ee47cab95cb1", "vacuum cleaner-c5c51437c3e8423f", "bowl3-703f3de39d983585", "knife2-82be27931bf65f28", "cover of butter dish-dd025665fe270be9", "lettuce bag-53344fda660944d9", "small bowl2-db3ecb3538ad5811", "spoon2-c9079f3e7df98043", "knife3-9827aa537ed14927", "teaspoon-6a23ed225f92fd9b" ], "reasoning": "1. I locate the fridge at 02:11 on the right side of the room in front of the user. \n2. At 02:18, I observe the user open the fridge.\n3. I rewind from 02:18 until I observe the user touch the glass jar (glass jar-dca3a59f4d09da0e) at 02:16.\n4. I determine that the last object the user touches before opening the fridge is the glass jar (glass jar-dca3a59f4d09da0e).", "referenced_object_masks": [ "glass jar-dca3a59f4d09da0e" ] }, { "key": "P07-20240529-194518:38d863e1c2cd4c37cc53ae30d7a34619b0ea6b60", "video_id": "P07-20240529-194518", "question": "After opening the wine bottle, what is the third thing I touch with my left hand, not counting the wine bottle lid?", "answer_choice_0": "Countertop", "answer_choice_1": "Wine bottle", "answer_choice_2": "Pepper grinder", "answer_choice_3": "Saucepan", "answer_choice_4": "Fridge handle", "answer_id": 4, "answer": "Fridge handle", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counting" ], "objects_of_interest": [ "wooden spatula-6a45cdf23bdd86d3", "frying pan-e144761e0ca28fbd", "saucepan-27b411bed31173de", "salt pig-7d76653044d55e3a", "pepper grinder-6c10b1dd9e13fdd5", "glass jar-dca3a59f4d09da0e", "bottle of wine-a5fc00bc00b59182", "lid of wine bottle-809f9516dc8991a8", "bowl-875ab2ef3779249c", "small bowl-a6b386e6bab2cc3b", "bulb-3c2ffffcc59866ef", "small object-9c5b32caf41ea7b5", "some object2-ae2b47e8e2dc1c74", "obj3-73f440c4ab148e8f", "glass jug2-73be4fae1e24e333", "spoon-633d244e722330bd", "fork-0fd2505f0727337d", "cheese block-c51aedd3e2b3e165", "digital scales-f73155d16b30fe0c", "scissors-f798ee0fa140ca95", "kitchen roll-086d32975371ef2f", "piece of kitchen roll-4643e3df23416793", "small block of cheese-c84866fd736372f8", "another very small piece of cheese-005b510257a37712", "pen-ee28a0d8efa50489", "piece of cheese3-3434adfcda47303d", "thin slice of cheese-2706430eeb82d855", "thin slice2-e5d7eeba01287985", "cheese piece5-4f55ec4bdb987d42", "empty cheese packaging-b933473a293976e0", "tea towel-36f36c206bbaa0a0", "cheese grater-ffd963d630712759", "bowl2-76300137c657f7a3", "bottle of oil-6a5d6e233495046b", "piece of paper-61a7cf79e6a59550", "tupperware container-c42614210387dc0c", "piece of plastic packaging-1a1fb9e272d46248", "piece of kitchen roll2-c5e0f2b43efa74a2", "glass of water-40dcf7749a8c8a15", "colander-559f5587ae532369", "dish washing liquid-a06c01a8dc14c1d2", "brush-6bec578509d79feb", "knife-b23d235095c6e134", "baking tray-51f814dae4bf8486", "can opener-60170d705d267c35", "measuring spoons-ba74de7b24c6307b", "cloth-3d3cecdbf2b9490c", "phone-78c9ee47cab95cb1", "vacuum cleaner-c5c51437c3e8423f", "bowl3-703f3de39d983585", "knife2-82be27931bf65f28", "cover of butter dish-dd025665fe270be9", "lettuce bag-53344fda660944d9", "small bowl2-db3ecb3538ad5811", "spoon2-c9079f3e7df98043", "knife3-9827aa537ed14927", "teaspoon-6a23ed225f92fd9b" ], "reasoning": "1. At 02:20, I observed the user take a wine bottle (bottle of wine-a5fc00bc00b59182) from the fridge.\n2. From 02:24 to 02:27, I observed the user open the wine bottle (bottle of wine-a5fc00bc00b59182) and remove the lid (lid of wine bottle-809f9516dc8991a8).\n3. At 02:27, I observed the user place their left hand on top of the counter and counted this as the first thing they touch. \n4: At 02:40, I observed the user pick up the wine bottle lid (lid of wine bottle-809f9516dc8991a8) with their left hand, which I did not count.\n5. At 02:41, I observed the user grab the wine bottle (bottle of wine-a5fc00bc00b59182) with their left hand, and counted this as the second thing they touched other than the wine bottle lid (lid of wine bottle-809f9516dc8991a8).\n6. At 02:44, I observed the user touch the fridge handle with their left hand, and counted this as the third thing they touched other than the wine bottle lid (lid of wine bottle-809f9516dc8991a8) after opening the bottle of wine (bottle of wine-a5fc00bc00b59182).", "referenced_object_masks": [ "lid of wine bottle-809f9516dc8991a8", "bottle of wine-a5fc00bc00b59182" ] }, { "key": "P07-20240529-194518:3bbe9d2a4968198a990936035c0b3cbdcf7a8dc1", "video_id": "P07-20240529-194518", "question": "What number would I get if I multiplied the number of pinches of salt from the salt pig that I add to the frying pan by the number of framed paintings in the corner by the table?", "answer_choice_0": "2", "answer_choice_1": "6", "answer_choice_2": "3", "answer_choice_3": "5", "answer_choice_4": "4", "answer_id": 1, "answer": "6", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning", "Numerical Reasoning", "Counting" ], "objects_of_interest": [ "wooden spatula-6a45cdf23bdd86d3", "frying pan-e144761e0ca28fbd", "saucepan-27b411bed31173de", "salt pig-7d76653044d55e3a", "pepper grinder-6c10b1dd9e13fdd5", "glass jar-dca3a59f4d09da0e", "bottle of wine-a5fc00bc00b59182", "lid of wine bottle-809f9516dc8991a8", "bowl-875ab2ef3779249c", "small bowl-a6b386e6bab2cc3b", "bulb-3c2ffffcc59866ef", "small object-9c5b32caf41ea7b5", "some object2-ae2b47e8e2dc1c74", "obj3-73f440c4ab148e8f", "glass jug2-73be4fae1e24e333", "spoon-633d244e722330bd", "fork-0fd2505f0727337d", "cheese block-c51aedd3e2b3e165", "digital scales-f73155d16b30fe0c", "scissors-f798ee0fa140ca95", "kitchen roll-086d32975371ef2f", "piece of kitchen roll-4643e3df23416793", "small block of cheese-c84866fd736372f8", "another very small piece of cheese-005b510257a37712", "pen-ee28a0d8efa50489", "piece of cheese3-3434adfcda47303d", "thin slice of cheese-2706430eeb82d855", "thin slice2-e5d7eeba01287985", "cheese piece5-4f55ec4bdb987d42", "empty cheese packaging-b933473a293976e0", "tea towel-36f36c206bbaa0a0", "cheese grater-ffd963d630712759", "bowl2-76300137c657f7a3", "bottle of oil-6a5d6e233495046b", "piece of paper-61a7cf79e6a59550", "tupperware container-c42614210387dc0c", "piece of plastic packaging-1a1fb9e272d46248", "piece of kitchen roll2-c5e0f2b43efa74a2", "glass of water-40dcf7749a8c8a15", "colander-559f5587ae532369", "dish washing liquid-a06c01a8dc14c1d2", "brush-6bec578509d79feb", "knife-b23d235095c6e134", "baking tray-51f814dae4bf8486", "can opener-60170d705d267c35", "measuring spoons-ba74de7b24c6307b", "cloth-3d3cecdbf2b9490c", "phone-78c9ee47cab95cb1", "vacuum cleaner-c5c51437c3e8423f", "bowl3-703f3de39d983585", "knife2-82be27931bf65f28", "cover of butter dish-dd025665fe270be9", "lettuce bag-53344fda660944d9", "small bowl2-db3ecb3538ad5811", "spoon2-c9079f3e7df98043", "knife3-9827aa537ed14927", "teaspoon-6a23ed225f92fd9b" ], "reasoning": "1. I locate the frying pan (frying pan-e144761e0ca28fbd) at 00:02 on the right side of the stovetop in front of the user.\n2. I locate the salt pig (salt pig-7d76653044d55e3a) at 00:02 on the counter to the left of the user. \n3. From 01:03 to 01:11, I observe the user add two pinches of salt from the salt pig (salt pig-7d76653044d55e3a) into the frying pan (frying pan-e144761e0ca28fbd).\n4. At 02:10, I locate the table in the corner on the far side of the room, surrounded by 3 framed paintings .\n5. I multiply 2 and 3 to get 6.", "referenced_object_masks": [ "salt pig-7d76653044d55e3a", "frying pan-e144761e0ca28fbd" ] }, { "key": "P07-20240529-194518:9dcc6e7a54f74f9f01e81b41ed42795a6f68ed79", "video_id": "P07-20240529-194518", "question": "Where is the microwave, in relation to myself, as I write on the piece of paper for the second time?", "answer_choice_0": "In front and to the left", "answer_choice_1": "Directly in front", "answer_choice_2": "Behind and to the left", "answer_choice_3": "Behind and to the left", "answer_choice_4": "In front and to the right", "answer_id": 0, "answer": "In front and to the left", "question_type": [ "Object Recognition", "Egocentric & Spatial Perception", "Counting", "Event Occurence" ], "objects_of_interest": [ "wooden spatula-6a45cdf23bdd86d3", "frying pan-e144761e0ca28fbd", "saucepan-27b411bed31173de", "salt pig-7d76653044d55e3a", "pepper grinder-6c10b1dd9e13fdd5", "glass jar-dca3a59f4d09da0e", "bottle of wine-a5fc00bc00b59182", "lid of wine bottle-809f9516dc8991a8", "bowl-875ab2ef3779249c", "small bowl-a6b386e6bab2cc3b", "bulb-3c2ffffcc59866ef", "small object-9c5b32caf41ea7b5", "some object2-ae2b47e8e2dc1c74", "obj3-73f440c4ab148e8f", "glass jug2-73be4fae1e24e333", "spoon-633d244e722330bd", "fork-0fd2505f0727337d", "cheese block-c51aedd3e2b3e165", "digital scales-f73155d16b30fe0c", "scissors-f798ee0fa140ca95", "kitchen roll-086d32975371ef2f", "piece of kitchen roll-4643e3df23416793", "small block of cheese-c84866fd736372f8", "another very small piece of cheese-005b510257a37712", "pen-ee28a0d8efa50489", "piece of cheese3-3434adfcda47303d", "thin slice of cheese-2706430eeb82d855", "thin slice2-e5d7eeba01287985", "cheese piece5-4f55ec4bdb987d42", "empty cheese packaging-b933473a293976e0", "tea towel-36f36c206bbaa0a0", "cheese grater-ffd963d630712759", "bowl2-76300137c657f7a3", "bottle of oil-6a5d6e233495046b", "piece of paper-61a7cf79e6a59550", "tupperware container-c42614210387dc0c", "piece of plastic packaging-1a1fb9e272d46248", "piece of kitchen roll2-c5e0f2b43efa74a2", "glass of water-40dcf7749a8c8a15", "colander-559f5587ae532369", "dish washing liquid-a06c01a8dc14c1d2", "brush-6bec578509d79feb", "knife-b23d235095c6e134", "baking tray-51f814dae4bf8486", "can opener-60170d705d267c35", "measuring spoons-ba74de7b24c6307b", "cloth-3d3cecdbf2b9490c", "phone-78c9ee47cab95cb1", "vacuum cleaner-c5c51437c3e8423f", "bowl3-703f3de39d983585", "knife2-82be27931bf65f28", "cover of butter dish-dd025665fe270be9", "lettuce bag-53344fda660944d9", "small bowl2-db3ecb3538ad5811", "spoon2-c9079f3e7df98043", "knife3-9827aa537ed14927", "teaspoon-6a23ed225f92fd9b" ], "reasoning": "1. To find this answer, I watch the video to determine when the user first writes on the piece of paper, and observe the first instance from 09:19-09:24, when the user uses the pen (pen-ee28a0d8efa50489) to write on the paper .\n2. I continue watching until I observe the second instance of the user writing on the piece of paper , which I observe from 09:53-09:59.\n3. Having identified the target timespan as 09:53-09:59, I then look for the microwave and observe it at the back left corner of the cabinet, relative to the user. This places it in front of the user and to the user's left, aligning with answer choice D.", "referenced_object_masks": [ "pen-ee28a0d8efa50489" ] }, { "key": "P07-20240529-194518:e02e82a46c99fff05038707d7833749dbf3deda6", "video_id": "P07-20240529-194518", "question": "After I pour broth from the saucepan into the large frying pan for the third time, and set the saucepan down, what is the material of the second item I touch with my left hand?", "answer_choice_0": "Wood", "answer_choice_1": "Cloth", "answer_choice_2": "Plastic", "answer_choice_3": "Silicone", "answer_choice_4": "Metal", "answer_id": 1, "answer": "Cloth", "question_type": [ "Counting", "Event Occurence", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "wooden spatula-6a45cdf23bdd86d3", "frying pan-e144761e0ca28fbd", "saucepan-27b411bed31173de", "salt pig-7d76653044d55e3a", "pepper grinder-6c10b1dd9e13fdd5", "glass jar-dca3a59f4d09da0e", "bottle of wine-a5fc00bc00b59182", "lid of wine bottle-809f9516dc8991a8", "bowl-875ab2ef3779249c", "small bowl-a6b386e6bab2cc3b", "bulb-3c2ffffcc59866ef", "small object-9c5b32caf41ea7b5", "some object2-ae2b47e8e2dc1c74", "obj3-73f440c4ab148e8f", "glass jug2-73be4fae1e24e333", "spoon-633d244e722330bd", "fork-0fd2505f0727337d", "cheese block-c51aedd3e2b3e165", "digital scales-f73155d16b30fe0c", "scissors-f798ee0fa140ca95", "kitchen roll-086d32975371ef2f", "piece of kitchen roll-4643e3df23416793", "small block of cheese-c84866fd736372f8", "another very small piece of cheese-005b510257a37712", "pen-ee28a0d8efa50489", "piece of cheese3-3434adfcda47303d", "thin slice of cheese-2706430eeb82d855", "thin slice2-e5d7eeba01287985", "cheese piece5-4f55ec4bdb987d42", "empty cheese packaging-b933473a293976e0", "tea towel-36f36c206bbaa0a0", "cheese grater-ffd963d630712759", "bowl2-76300137c657f7a3", "bottle of oil-6a5d6e233495046b", "piece of paper-61a7cf79e6a59550", "tupperware container-c42614210387dc0c", "piece of plastic packaging-1a1fb9e272d46248", "piece of kitchen roll2-c5e0f2b43efa74a2", "glass of water-40dcf7749a8c8a15", "colander-559f5587ae532369", "dish washing liquid-a06c01a8dc14c1d2", "brush-6bec578509d79feb", "knife-b23d235095c6e134", "baking tray-51f814dae4bf8486", "can opener-60170d705d267c35", "measuring spoons-ba74de7b24c6307b", "cloth-3d3cecdbf2b9490c", "phone-78c9ee47cab95cb1", "vacuum cleaner-c5c51437c3e8423f", "bowl3-703f3de39d983585", "knife2-82be27931bf65f28", "cover of butter dish-dd025665fe270be9", "lettuce bag-53344fda660944d9", "small bowl2-db3ecb3538ad5811", "spoon2-c9079f3e7df98043", "knife3-9827aa537ed14927", "teaspoon-6a23ed225f92fd9b" ], "reasoning": "1. I watch the video to observe and count the instances in which the user pours broth from the saucepan (saucepan-27b411bed31173de) into the large frying pan (frying pan-e144761e0ca28fbd). I identify the first pour from 07:46-07:50.\n2. I continue watching the video to identify and count the second pour of broth from the saucepan (saucepan-27b411bed31173de) into the frying pan (frying pan-e144761e0ca28fbd), which I observe from 10:15-10:20.\n3. I continue watching the video to identify and count the third pour of broth from the saucepan (saucepan-27b411bed31173de) into the frying pan (frying pan-e144761e0ca28fbd), which I observe from 19:08-19:20. I then observe the user set the saucepan (saucepan-27b411bed31173de) down at 19:21.\n4. From this point on, I observe what the user does with their left hand - I observe that they touch nothing with their hand until approximately 19:45, after picking up the saucepan (saucepan-27b411bed31173de) with their right hand at 19:44, taking it to the sink, and transferring it to their left hand at 19:45 on the way to the sink. The user grips the handle of the saucepan (saucepan-27b411bed31173de) with their left hand - this handle is plastic, and I count it as the first instance of the user touching something with their left hand after the third broth pour.\n5. I continue watching until the user touches something new with their left hand - this occurs from 20:11-20:14, when the user dries both of their hands with the towel (tea towel-36f36c206bbaa0a0) hanging on the kitchen island. I observe that this towel (tea towel-36f36c206bbaa0a0) is made of cloth, aligning with answer choice C.", "referenced_object_masks": [ "tea towel-36f36c206bbaa0a0", "saucepan-27b411bed31173de", "frying pan-e144761e0ca28fbd" ] }, { "key": "P07-20240529-194518:e0d5fb8ccf7aabcbad0c9ef06a4983479d64d765", "video_id": "P07-20240529-194518", "question": "Where do I place the pepper grinder after using it to season the food in the frying pan?", "answer_choice_0": "In front of the salt container and the toaster", "answer_choice_1": "Behind the salt container and to the right of the toaster", "answer_choice_2": "In front of the salt container and to the right of the toaster", "answer_choice_3": "In front of the salt container and to the left of the toaster", "answer_choice_4": "Behind the salt container and to the left of the toaster", "answer_id": 1, "answer": "Behind the salt container and to the right of the toaster", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Event Occurence" ], "objects_of_interest": [ "wooden spatula-6a45cdf23bdd86d3", "frying pan-e144761e0ca28fbd", "saucepan-27b411bed31173de", "salt pig-7d76653044d55e3a", "pepper grinder-6c10b1dd9e13fdd5", "glass jar-dca3a59f4d09da0e", "bottle of wine-a5fc00bc00b59182", "lid of wine bottle-809f9516dc8991a8", "bowl-875ab2ef3779249c", "small bowl-a6b386e6bab2cc3b", "bulb-3c2ffffcc59866ef", "small object-9c5b32caf41ea7b5", "some object2-ae2b47e8e2dc1c74", "obj3-73f440c4ab148e8f", "glass jug2-73be4fae1e24e333", "spoon-633d244e722330bd", "fork-0fd2505f0727337d", "cheese block-c51aedd3e2b3e165", "digital scales-f73155d16b30fe0c", "scissors-f798ee0fa140ca95", "kitchen roll-086d32975371ef2f", "piece of kitchen roll-4643e3df23416793", "small block of cheese-c84866fd736372f8", "another very small piece of cheese-005b510257a37712", "pen-ee28a0d8efa50489", "piece of cheese3-3434adfcda47303d", "thin slice of cheese-2706430eeb82d855", "thin slice2-e5d7eeba01287985", "cheese piece5-4f55ec4bdb987d42", "empty cheese packaging-b933473a293976e0", "tea towel-36f36c206bbaa0a0", "cheese grater-ffd963d630712759", "bowl2-76300137c657f7a3", "bottle of oil-6a5d6e233495046b", "piece of paper-61a7cf79e6a59550", "tupperware container-c42614210387dc0c", "piece of plastic packaging-1a1fb9e272d46248", "piece of kitchen roll2-c5e0f2b43efa74a2", "glass of water-40dcf7749a8c8a15", "colander-559f5587ae532369", "dish washing liquid-a06c01a8dc14c1d2", "brush-6bec578509d79feb", "knife-b23d235095c6e134", "baking tray-51f814dae4bf8486", "can opener-60170d705d267c35", "measuring spoons-ba74de7b24c6307b", "cloth-3d3cecdbf2b9490c", "phone-78c9ee47cab95cb1", "vacuum cleaner-c5c51437c3e8423f", "bowl3-703f3de39d983585", "knife2-82be27931bf65f28", "cover of butter dish-dd025665fe270be9", "lettuce bag-53344fda660944d9", "small bowl2-db3ecb3538ad5811", "spoon2-c9079f3e7df98043", "knife3-9827aa537ed14927", "teaspoon-6a23ed225f92fd9b" ], "reasoning": "1. I watch the video to determine when the user seasons the food in the frying pan with pepper - this occurs from 01:12-01:25, when the user grinds pepper from the pepper mill (pepper grinder-6c10b1dd9e13fdd5) over the large frying pan (frying pan-e144761e0ca28fbd).\n2. Next, I watch the user put the pepper grinder (pepper grinder-6c10b1dd9e13fdd5) down at 01:25, directly behind the salt container (salt pig-7d76653044d55e3a). I recognize this as the salt container (salt pig-7d76653044d55e3a) as such after watching the user sprinkle salt over the frying pan from it (frying pan-e144761e0ca28fbd) from 01:03-01:11.\n3. Next, I look for the location of the toaster in relation to the pepper grinder (pepper grinder-6c10b1dd9e13fdd5) and observe it to the left of the pepper grinder (pepper grinder-6c10b1dd9e13fdd5) at 01:25. I compare the pepper grinder's (pepper grinder-6c10b1dd9e13fdd5) relative location to the two items to the answer choices in the answer bank, and determine that E matches the locations observed at 01:25.", "referenced_object_masks": [ "salt pig-7d76653044d55e3a", "frying pan-e144761e0ca28fbd", "pepper grinder-6c10b1dd9e13fdd5" ] }, { "key": "P07-20240529-194518:e2a88b1e689c6e31249b6cf7886e70b1b803113c", "video_id": "P07-20240529-194518", "question": "What is the difference between the first weight reading of the second-to-last food item I weigh on the food scale, and the final weight reading of the last food item I weigh on the food scale?", "answer_choice_0": "29", "answer_choice_1": "10", "answer_choice_2": "49", "answer_choice_3": "37", "answer_choice_4": "39", "answer_id": 4, "answer": "39", "question_type": [ "Reading", "Temporal Reasoning", "Numerical Reasoning", "Action Recognition" ], "objects_of_interest": [ "wooden spatula-6a45cdf23bdd86d3", "frying pan-e144761e0ca28fbd", "saucepan-27b411bed31173de", "salt pig-7d76653044d55e3a", "pepper grinder-6c10b1dd9e13fdd5", "glass jar-dca3a59f4d09da0e", "bottle of wine-a5fc00bc00b59182", "lid of wine bottle-809f9516dc8991a8", "bowl-875ab2ef3779249c", "small bowl-a6b386e6bab2cc3b", "bulb-3c2ffffcc59866ef", "small object-9c5b32caf41ea7b5", "some object2-ae2b47e8e2dc1c74", "obj3-73f440c4ab148e8f", "glass jug2-73be4fae1e24e333", "spoon-633d244e722330bd", "fork-0fd2505f0727337d", "cheese block-c51aedd3e2b3e165", "digital scales-f73155d16b30fe0c", "scissors-f798ee0fa140ca95", "kitchen roll-086d32975371ef2f", "piece of kitchen roll-4643e3df23416793", "small block of cheese-c84866fd736372f8", "another very small piece of cheese-005b510257a37712", "pen-ee28a0d8efa50489", "piece of cheese3-3434adfcda47303d", "thin slice of cheese-2706430eeb82d855", "thin slice2-e5d7eeba01287985", "cheese piece5-4f55ec4bdb987d42", "empty cheese packaging-b933473a293976e0", "tea towel-36f36c206bbaa0a0", "cheese grater-ffd963d630712759", "bowl2-76300137c657f7a3", "bottle of oil-6a5d6e233495046b", "piece of paper-61a7cf79e6a59550", "tupperware container-c42614210387dc0c", "piece of plastic packaging-1a1fb9e272d46248", "piece of kitchen roll2-c5e0f2b43efa74a2", "glass of water-40dcf7749a8c8a15", "colander-559f5587ae532369", "dish washing liquid-a06c01a8dc14c1d2", "brush-6bec578509d79feb", "knife-b23d235095c6e134", "baking tray-51f814dae4bf8486", "can opener-60170d705d267c35", "measuring spoons-ba74de7b24c6307b", "cloth-3d3cecdbf2b9490c", "phone-78c9ee47cab95cb1", "vacuum cleaner-c5c51437c3e8423f", "bowl3-703f3de39d983585", "knife2-82be27931bf65f28", "cover of butter dish-dd025665fe270be9", "lettuce bag-53344fda660944d9", "small bowl2-db3ecb3538ad5811", "spoon2-c9079f3e7df98043", "knife3-9827aa537ed14927", "teaspoon-6a23ed225f92fd9b" ], "reasoning": "1. I watched the video and observed the final item that was weighed on the food scale (digital scales-f73155d16b30fe0c) was lettuce from the lettuce bag (lettuce bag-53344fda660944d9) from 23:51-24:05. I watched this sequence to the end, and read the final weight that I was able to see on the scale - this weight gave a value of 10 at 24:05.\n2. Having established the final food item as the lettuce from the bag of lettuce (lettuce bag-53344fda660944d9), I watch the video to determine which food item was weighed directly before this item - I find that it was the pat of butter , weighed on the food scale (digital scales-f73155d16b30fe0c) from 22:05-22:20. When the butter pat is first dropped onto the scale (digital scales-f73155d16b30fe0c), I read a value of 49.\n3. To find the difference between these two numbers, I do the equation 49 - 10 = 39, making B the correct answer.", "referenced_object_masks": [ "lettuce bag-53344fda660944d9", "digital scales-f73155d16b30fe0c" ] }, { "key": "P07-20240529-194518:eb803c3b15f34d947c4236f6c7352dcf8394a1c5", "video_id": "P07-20240529-194518", "question": "During the part of the video in which I perform the following actions in close succession - take out the cheese grater, sip water, open the cabinet, touch the piece of paper, dry hands - which action would be performed fourth if the actions were considered in reverse chronological order?", "answer_choice_0": "Open the cabinet", "answer_choice_1": "Sip water", "answer_choice_2": "Dry hands", "answer_choice_3": "Touch the piece of paper", "answer_choice_4": "Take out the cheese grater", "answer_id": 1, "answer": "Sip water", "question_type": [ "Action Recognition", "Temporal Reasoning", "Counterfactual", "Object Recognition" ], "objects_of_interest": [ "wooden spatula-6a45cdf23bdd86d3", "frying pan-e144761e0ca28fbd", "saucepan-27b411bed31173de", "salt pig-7d76653044d55e3a", "pepper grinder-6c10b1dd9e13fdd5", "glass jar-dca3a59f4d09da0e", "bottle of wine-a5fc00bc00b59182", "lid of wine bottle-809f9516dc8991a8", "bowl-875ab2ef3779249c", "small bowl-a6b386e6bab2cc3b", "bulb-3c2ffffcc59866ef", "small object-9c5b32caf41ea7b5", "some object2-ae2b47e8e2dc1c74", "obj3-73f440c4ab148e8f", "glass jug2-73be4fae1e24e333", "spoon-633d244e722330bd", "fork-0fd2505f0727337d", "cheese block-c51aedd3e2b3e165", "digital scales-f73155d16b30fe0c", "scissors-f798ee0fa140ca95", "kitchen roll-086d32975371ef2f", "piece of kitchen roll-4643e3df23416793", "small block of cheese-c84866fd736372f8", "another very small piece of cheese-005b510257a37712", "pen-ee28a0d8efa50489", "piece of cheese3-3434adfcda47303d", "thin slice of cheese-2706430eeb82d855", "thin slice2-e5d7eeba01287985", "cheese piece5-4f55ec4bdb987d42", "empty cheese packaging-b933473a293976e0", "tea towel-36f36c206bbaa0a0", "cheese grater-ffd963d630712759", "bowl2-76300137c657f7a3", "bottle of oil-6a5d6e233495046b", "piece of paper-61a7cf79e6a59550", "tupperware container-c42614210387dc0c", "piece of plastic packaging-1a1fb9e272d46248", "piece of kitchen roll2-c5e0f2b43efa74a2", "glass of water-40dcf7749a8c8a15", "colander-559f5587ae532369", "dish washing liquid-a06c01a8dc14c1d2", "brush-6bec578509d79feb", "knife-b23d235095c6e134", "baking tray-51f814dae4bf8486", "can opener-60170d705d267c35", "measuring spoons-ba74de7b24c6307b", "cloth-3d3cecdbf2b9490c", "phone-78c9ee47cab95cb1", "vacuum cleaner-c5c51437c3e8423f", "bowl3-703f3de39d983585", "knife2-82be27931bf65f28", "cover of butter dish-dd025665fe270be9", "lettuce bag-53344fda660944d9", "small bowl2-db3ecb3538ad5811", "spoon2-c9079f3e7df98043", "knife3-9827aa537ed14927", "teaspoon-6a23ed225f92fd9b" ], "reasoning": "1. I watch the video in search of the actions listed in the question - I spot the first from 11:06-11:09, when I observe the user dry their hands on a towel (tea towel-36f36c206bbaa0a0) which hangs on the side of the kitchen island.\n2. I continue watching for another chronologically close item in the sequence, to ensure that I'm in the right timespan, and confirm this from 11:11-11:13, when I observe the user pick up a glass of water (glass of water-40dcf7749a8c8a15) with their right hand and lifts it to their face to take a sip.\n3. Having confirmed I'm in the right timespan, I continue watching for the next of the listed actions in the sequence - I observe it from 11:17-11:19, when the user touches the piece of paper on the counter by gliding their index finger across it.\n4. I continue watching and observe the next action in the sequence, which occurs from 11:23-11:24, when the user opens a drawer and takes out a handheld cheese grater (cheese grater-ffd963d630712759).\n5. I observe the final action in the sequence from 11:26-11:27, when the user opens an upper cabinet door with their left hand.\n6. Next, I counterfactually imagine these actions occurring in reverse chronological order, and thus obtain an order of: open cabinet door, take out cheese grater, touch piece of paper, sip water, and dry hands. Thus, I identify \"sip water\" as the fourth item in the sequence and obtain A as the correct answer.", "referenced_object_masks": [ "tea towel-36f36c206bbaa0a0", "cheese grater-ffd963d630712759", "glass of water-40dcf7749a8c8a15" ] }, { "key": "P07-20240530-102543:5d071bedca96e12b97481a9a5ad350f19943f130", "video_id": "P07-20240530-102543", "question": "When loading the top rack of the dishwasher, how many items do I touch with my left hand?", "answer_choice_0": "4", "answer_choice_1": "1", "answer_choice_2": "2", "answer_choice_3": "3", "answer_choice_4": ", 0", "answer_id": 1, "answer": "1", "question_type": [ "Object Recognition", "Counting", "Action Recognition", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "piece of paper-e3c4dd5595274b83", "pen-1d1bedb0692b222b", "small cheese grater-3880df074397175b", "glass-4a4dfe5f7f4d15b5", "glass2-dcf718dfba2d6c1e", "glass3-dc8815d66dc23e97", "glass4-710519b3c22aa5a4", "mug-c2160817972a2375", "mug2-884a6d20d120dd9b", "bowl-1f2a19a60439d92e", "bowl2-0a125aca643e1b8e", "bowl3-f5d0cc006f25122e", "wooden spoon-288bfd3197a48a8d", "knife-faa9d7c1a68cb98c", "glass jug-0208d781dfddc260", "glass jug2-6be21e0d05a66b24", "bowl4-2671372265986d18", "dish-4eb2321eef3a9ea8", "dish2-8750ca603b078a3e", "dish3-72445a6842983782", "dish4-fae14162ec1502c9", "dish5-d0066438fa9d0764", "dish6-d0531b05732a4ed2", "plate-2f744d8182f02eac", "plastic tub-27783f8e52eeada1", "plastic box2-38f70b13f90caa87", "mug3-2e016a8533b887cd", "mug4-6ba48a851f8de918", "cutlery organiser-d2f57b83ab5a048c", "cutlery-69547d0e85c016a7", "cutlery2-e9039880cc5ae9a2", "cutlery3-1e46994a38e42af8", "cutlery4-bbaf0ef0f5e2d0f5", "cutlery5-79da77cef673ba14", "forks-2d9550fdb651c17b", "forks2-728016f98db2b29d", "cutlery6-25ac101e20948004", "cutlery7-f114212e23983fb4", "cutlery8-b1460df52355e712", "cutlery9-b722a9001f288c68", "cutlery10-b58fbfc63703903d", "cutlery11-9a6dffed8bf661bd", "knives-4e10802612bd2112", "cutlery12-61a6b90d19179483", "cutlery13-d9622eba701c3eb0", "cutlery14-3140d919d08c2a44", "spoon-187843e083891950", "wooden spoon2-3f392d8e7ca9baa9", "glass5-ad10e5acdb7bf5cc", "bowl5-a1811a30c0b4c95a", "glass6-d41100715e7bfa0e", "mug5-98164392d067525b", "bowl6-22948cff3c9a735a", "digital weighing scale-d18eaa1e2759e89e", "butter pack-79f71b2222e02f6f", "knife2-65eb37cb61e079d4", "phone-99b7048ce196acdf", "bottle of dishwashing liquid-e46c8a5a6242974e", "fork3-f61478996797cd4a", "large silver bowl-1e7813e6e613a8c2", "packet of chocolate chips-0dfd92cec40be341", "packet of sugar-3b15c481102a833f", "another packet of sugar-f3cfce63317849c4", "frozen banana-e1dcd9734165a98e", "small bowl-edc59345ebbbf00d", "stool-746961e147f9c80b", "eggshell halves-1f61d7324317ade6", "egg-0ea47b75fd5e54e0", "dishcloth-55b4dad939e268fe", "food colouring-25f4b62e069fb91a", "bottle of vanilla extract-c74628b4f0c44823", "different bottle of vanilla extract-6faaa4e7ab87aa43", "cap of vanilla extract-9b0a2df70b4af5c9", "measuring spoon-79aa4b829b720cae", "self-raising flour-b22f670b3d29fdf5", "jar of cinnamon-2c78c72e1436808e", "lid-87bef665f302d4dd", "bundle of measuring spoons-e7547c0cd4e140c1", "wooden spoon3-25be19ed35e79e70", "flour2-ce8eebab899cb76a", "teaspoon-21a63251a249bdb1", "tea towel-1d6b81273de3c112", "baking tray-2c38c6880ed32299", "another tray-08d98dbb7a2290e7", "baking tray3-f267fe8668a0e669", "baking tray2-c8dacad37c5d99f6", "large frying pan-8f165e52c0a1907b", "saucepan-d77c6538c765b529", "colander-b6dc83a8108d1092", "saucepan2-5bcb83c6b05845fe", "stack of pans-dc779649831e4d7d", "lid2-be6c135443054d64", "spoon2-70ca298babbde1e0", "orange sponge-91f0584e90d28efb", "cloth-42233e44c039ef1e", "scrubbing brush-e6b13dee9d26a23e", "pair of tongs-67a14eff4502fbf6", "wooden spoon4-64492f46be762100", "small bowl2-f7e1155f875d754a", "knife3-50ca003f99bdc912", "knife4-d668923b0458ec0a", "torn off piece of baking paper-51e59bd8cf995835", "baking paper-575d2b02b73b0734", "tea towel2-d839e5738570e458", "teaspoon2-387d4b2deae96246", "teaspoon3-1d4e3d773bbfb682", "teaspoon4-920a37a312e4b45d", "coffee basket-13a2d19317e7799b", "drawer of coffee grinder-6dfb938c5e8e96c5", "coffee grinder-0aefaa1990908ac5", "coffee beans package-9e75b0d00284cea1", "lid of coffee grinder-a446ab50984202f1", "coffee spoon-15cd40a172d1b8f3", "lid of drawer of coffee grinder-20efd78f1cb1701d", "mug6-63788c45b999cee4", "drip tray-a7a99e98d6075642", "spoon3-4cc6fcca892d5ec4", "spoon4-26d9a1e27ce73a8d", "baking paper2-7dd10eaaedeb9b03" ], "reasoning": "1. I watched the video and saw that the user begins loading dishes into the dishwasher at 02:51, when they put a dirty spoon (spoon-187843e083891950) in the cutlery organizer (cutlery organiser-d2f57b83ab5a048c).\n2. From here, I continued to watch as the user loaded dishes into the bottom rack of the dishwasher .\n3. This continued until 03:11-03:12, when the user pulls out the top rack of the dishwasher .\n4. From 03:12-03:35 the user loads the follow dishes at the specified times: 03:12-03:13 cup 1 (glass5-ad10e5acdb7bf5cc), 03:15-03:17 bowl 1 (bowl5-a1811a30c0b4c95a), 03:18-03:20 cup 2 (glass6-d41100715e7bfa0e), mug 1 from 03:22-03:25 (mug5-98164392d067525b), mug 2 from 03:26-03:28 (bowl6-22948cff3c9a735a), and bowl 2 from 03:30-03:33 .\n5. During this time span, I watched to see if the user touched any items with their left hand, observing from 03:19-03:21 that they touch cup 1 (glass5-ad10e5acdb7bf5cc) with their left hand.\n6. Thus, only one item is touched with the left hand when loading the top rack of the dishwasher .", "referenced_object_masks": [ "bowl5-a1811a30c0b4c95a", "glass6-d41100715e7bfa0e", "bowl6-22948cff3c9a735a", "cutlery organiser-d2f57b83ab5a048c", "spoon-187843e083891950", "mug5-98164392d067525b", "glass5-ad10e5acdb7bf5cc" ] }, { "key": "P07-20240530-102543:718a1f7eaa018b6032cfda1651feace07e38992d", "video_id": "P07-20240530-102543", "question": "Which item do I grab from the dishwasher second?", "answer_choice_0": "Cheese grater", "answer_choice_1": "Cup", "answer_choice_2": "Knife", "answer_choice_3": "Glass", "answer_choice_4": "Pan", "answer_id": 3, "answer": "Glass", "question_type": [ "Action Recognition", "Object Recognition", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "piece of paper-e3c4dd5595274b83", "pen-1d1bedb0692b222b", "small cheese grater-3880df074397175b", "glass-4a4dfe5f7f4d15b5", "glass2-dcf718dfba2d6c1e", "glass3-dc8815d66dc23e97", "glass4-710519b3c22aa5a4", "mug-c2160817972a2375", "mug2-884a6d20d120dd9b", "bowl-1f2a19a60439d92e", "bowl2-0a125aca643e1b8e", "bowl3-f5d0cc006f25122e", "wooden spoon-288bfd3197a48a8d", "knife-faa9d7c1a68cb98c", "glass jug-0208d781dfddc260", "glass jug2-6be21e0d05a66b24", "bowl4-2671372265986d18", "dish-4eb2321eef3a9ea8", "dish2-8750ca603b078a3e", "dish3-72445a6842983782", "dish4-fae14162ec1502c9", "dish5-d0066438fa9d0764", "dish6-d0531b05732a4ed2", "plate-2f744d8182f02eac", "plastic tub-27783f8e52eeada1", "plastic box2-38f70b13f90caa87", "mug3-2e016a8533b887cd", "mug4-6ba48a851f8de918", "cutlery organiser-d2f57b83ab5a048c", "cutlery-69547d0e85c016a7", "cutlery2-e9039880cc5ae9a2", "cutlery3-1e46994a38e42af8", "cutlery4-bbaf0ef0f5e2d0f5", "cutlery5-79da77cef673ba14", "forks-2d9550fdb651c17b", "forks2-728016f98db2b29d", "cutlery6-25ac101e20948004", "cutlery7-f114212e23983fb4", "cutlery8-b1460df52355e712", "cutlery9-b722a9001f288c68", "cutlery10-b58fbfc63703903d", "cutlery11-9a6dffed8bf661bd", "knives-4e10802612bd2112", "cutlery12-61a6b90d19179483", "cutlery13-d9622eba701c3eb0", "cutlery14-3140d919d08c2a44", "spoon-187843e083891950", "wooden spoon2-3f392d8e7ca9baa9", "glass5-ad10e5acdb7bf5cc", "bowl5-a1811a30c0b4c95a", "glass6-d41100715e7bfa0e", "mug5-98164392d067525b", "bowl6-22948cff3c9a735a", "digital weighing scale-d18eaa1e2759e89e", "butter pack-79f71b2222e02f6f", "knife2-65eb37cb61e079d4", "phone-99b7048ce196acdf", "bottle of dishwashing liquid-e46c8a5a6242974e", "fork3-f61478996797cd4a", "large silver bowl-1e7813e6e613a8c2", "packet of chocolate chips-0dfd92cec40be341", "packet of sugar-3b15c481102a833f", "another packet of sugar-f3cfce63317849c4", "frozen banana-e1dcd9734165a98e", "small bowl-edc59345ebbbf00d", "stool-746961e147f9c80b", "eggshell halves-1f61d7324317ade6", "egg-0ea47b75fd5e54e0", "dishcloth-55b4dad939e268fe", "food colouring-25f4b62e069fb91a", "bottle of vanilla extract-c74628b4f0c44823", "different bottle of vanilla extract-6faaa4e7ab87aa43", "cap of vanilla extract-9b0a2df70b4af5c9", "measuring spoon-79aa4b829b720cae", "self-raising flour-b22f670b3d29fdf5", "jar of cinnamon-2c78c72e1436808e", "lid-87bef665f302d4dd", "bundle of measuring spoons-e7547c0cd4e140c1", "wooden spoon3-25be19ed35e79e70", "flour2-ce8eebab899cb76a", "teaspoon-21a63251a249bdb1", "tea towel-1d6b81273de3c112", "baking tray-2c38c6880ed32299", "another tray-08d98dbb7a2290e7", "baking tray3-f267fe8668a0e669", "baking tray2-c8dacad37c5d99f6", "large frying pan-8f165e52c0a1907b", "saucepan-d77c6538c765b529", "colander-b6dc83a8108d1092", "saucepan2-5bcb83c6b05845fe", "stack of pans-dc779649831e4d7d", "lid2-be6c135443054d64", "spoon2-70ca298babbde1e0", "orange sponge-91f0584e90d28efb", "cloth-42233e44c039ef1e", "scrubbing brush-e6b13dee9d26a23e", "pair of tongs-67a14eff4502fbf6", "wooden spoon4-64492f46be762100", "small bowl2-f7e1155f875d754a", "knife3-50ca003f99bdc912", "knife4-d668923b0458ec0a", "torn off piece of baking paper-51e59bd8cf995835", "baking paper-575d2b02b73b0734", "tea towel2-d839e5738570e458", "teaspoon2-387d4b2deae96246", "teaspoon3-1d4e3d773bbfb682", "teaspoon4-920a37a312e4b45d", "coffee basket-13a2d19317e7799b", "drawer of coffee grinder-6dfb938c5e8e96c5", "coffee grinder-0aefaa1990908ac5", "coffee beans package-9e75b0d00284cea1", "lid of coffee grinder-a446ab50984202f1", "coffee spoon-15cd40a172d1b8f3", "lid of drawer of coffee grinder-20efd78f1cb1701d", "mug6-63788c45b999cee4", "drip tray-a7a99e98d6075642", "spoon3-4cc6fcca892d5ec4", "spoon4-26d9a1e27ce73a8d", "baking paper2-7dd10eaaedeb9b03" ], "reasoning": "1. I watched the video and saw the user open the dishwasher at 00:10.\n2. Then, I observed the user grab the cheese grater (small cheese grater-3880df074397175b) and put it away from 00:14-00:15. \n3. Immediately after this, the user grabs a glass (glass-4a4dfe5f7f4d15b5), making this the second item they take out of the dishwasher.", "referenced_object_masks": [ "glass-4a4dfe5f7f4d15b5", "small cheese grater-3880df074397175b" ] }, { "key": "P07-20240530-102543:741283287dd035a3453fc998b803ab50822eaa3c", "video_id": "P07-20240530-102543", "question": "How many times is the folding stool folded or unfolded over the course of the video?", "answer_choice_0": "4", "answer_choice_1": "1", "answer_choice_2": "3", "answer_choice_3": "0", "answer_choice_4": "2", "answer_id": 2, "answer": "3", "question_type": [ "Counting", "Action Recognition", "Object Recognition", "Audio Reasoning", "State Changes" ], "objects_of_interest": [ "piece of paper-e3c4dd5595274b83", "pen-1d1bedb0692b222b", "small cheese grater-3880df074397175b", "glass-4a4dfe5f7f4d15b5", "glass2-dcf718dfba2d6c1e", "glass3-dc8815d66dc23e97", "glass4-710519b3c22aa5a4", "mug-c2160817972a2375", "mug2-884a6d20d120dd9b", "bowl-1f2a19a60439d92e", "bowl2-0a125aca643e1b8e", "bowl3-f5d0cc006f25122e", "wooden spoon-288bfd3197a48a8d", "knife-faa9d7c1a68cb98c", "glass jug-0208d781dfddc260", "glass jug2-6be21e0d05a66b24", "bowl4-2671372265986d18", "dish-4eb2321eef3a9ea8", "dish2-8750ca603b078a3e", "dish3-72445a6842983782", "dish4-fae14162ec1502c9", "dish5-d0066438fa9d0764", "dish6-d0531b05732a4ed2", "plate-2f744d8182f02eac", "plastic tub-27783f8e52eeada1", "plastic box2-38f70b13f90caa87", "mug3-2e016a8533b887cd", "mug4-6ba48a851f8de918", "cutlery organiser-d2f57b83ab5a048c", "cutlery-69547d0e85c016a7", "cutlery2-e9039880cc5ae9a2", "cutlery3-1e46994a38e42af8", "cutlery4-bbaf0ef0f5e2d0f5", "cutlery5-79da77cef673ba14", "forks-2d9550fdb651c17b", "forks2-728016f98db2b29d", "cutlery6-25ac101e20948004", "cutlery7-f114212e23983fb4", "cutlery8-b1460df52355e712", "cutlery9-b722a9001f288c68", "cutlery10-b58fbfc63703903d", "cutlery11-9a6dffed8bf661bd", "knives-4e10802612bd2112", "cutlery12-61a6b90d19179483", "cutlery13-d9622eba701c3eb0", "cutlery14-3140d919d08c2a44", "spoon-187843e083891950", "wooden spoon2-3f392d8e7ca9baa9", "glass5-ad10e5acdb7bf5cc", "bowl5-a1811a30c0b4c95a", "glass6-d41100715e7bfa0e", "mug5-98164392d067525b", "bowl6-22948cff3c9a735a", "digital weighing scale-d18eaa1e2759e89e", "butter pack-79f71b2222e02f6f", "knife2-65eb37cb61e079d4", "phone-99b7048ce196acdf", "bottle of dishwashing liquid-e46c8a5a6242974e", "fork3-f61478996797cd4a", "large silver bowl-1e7813e6e613a8c2", "packet of chocolate chips-0dfd92cec40be341", "packet of sugar-3b15c481102a833f", "another packet of sugar-f3cfce63317849c4", "frozen banana-e1dcd9734165a98e", "small bowl-edc59345ebbbf00d", "stool-746961e147f9c80b", "eggshell halves-1f61d7324317ade6", "egg-0ea47b75fd5e54e0", "dishcloth-55b4dad939e268fe", "food colouring-25f4b62e069fb91a", "bottle of vanilla extract-c74628b4f0c44823", "different bottle of vanilla extract-6faaa4e7ab87aa43", "cap of vanilla extract-9b0a2df70b4af5c9", "measuring spoon-79aa4b829b720cae", "self-raising flour-b22f670b3d29fdf5", "jar of cinnamon-2c78c72e1436808e", "lid-87bef665f302d4dd", "bundle of measuring spoons-e7547c0cd4e140c1", "wooden spoon3-25be19ed35e79e70", "flour2-ce8eebab899cb76a", "teaspoon-21a63251a249bdb1", "tea towel-1d6b81273de3c112", "baking tray-2c38c6880ed32299", "another tray-08d98dbb7a2290e7", "baking tray3-f267fe8668a0e669", "baking tray2-c8dacad37c5d99f6", "large frying pan-8f165e52c0a1907b", "saucepan-d77c6538c765b529", "colander-b6dc83a8108d1092", "saucepan2-5bcb83c6b05845fe", "stack of pans-dc779649831e4d7d", "lid2-be6c135443054d64", "spoon2-70ca298babbde1e0", "orange sponge-91f0584e90d28efb", "cloth-42233e44c039ef1e", "scrubbing brush-e6b13dee9d26a23e", "pair of tongs-67a14eff4502fbf6", "wooden spoon4-64492f46be762100", "small bowl2-f7e1155f875d754a", "knife3-50ca003f99bdc912", "knife4-d668923b0458ec0a", "torn off piece of baking paper-51e59bd8cf995835", "baking paper-575d2b02b73b0734", "tea towel2-d839e5738570e458", "teaspoon2-387d4b2deae96246", "teaspoon3-1d4e3d773bbfb682", "teaspoon4-920a37a312e4b45d", "coffee basket-13a2d19317e7799b", "drawer of coffee grinder-6dfb938c5e8e96c5", "coffee grinder-0aefaa1990908ac5", "coffee beans package-9e75b0d00284cea1", "lid of coffee grinder-a446ab50984202f1", "coffee spoon-15cd40a172d1b8f3", "lid of drawer of coffee grinder-20efd78f1cb1701d", "mug6-63788c45b999cee4", "drip tray-a7a99e98d6075642", "spoon3-4cc6fcca892d5ec4", "spoon4-26d9a1e27ce73a8d", "baking paper2-7dd10eaaedeb9b03" ], "reasoning": "1. I look for the first time the small folding stool (stool-746961e147f9c80b) is folded, and this happens at 00:05. Though I cannot see it fold, I can hear it.\n2. The folding stool (stool-746961e147f9c80b) is unfolded later at 11:45-11:46, after which the user stands on it and uses it to reach more ingredients.\n3. Finally, the folding stool (stool-746961e147f9c80b) is folded back at 20:09. The folding of the stool (stool-746961e147f9c80b) is heard more than seen.\n4. From 20:11-20:12, the user puts the stool (stool-746961e147f9c80b) by a rack on the wall to the right of the sink and in front of the coffee machine .\n5. After this time, the user never interacts with it, so I know here that the user has put it away.\n6. I count up the times the folding stool (stool-746961e147f9c80b) has been folded and unfolded and I come to a total of 3 times.", "referenced_object_masks": [ "stool-746961e147f9c80b" ] }, { "key": "P07-20240530-102543:77e33dc4845e267a1ac273a69741f611f5526e6b", "video_id": "P07-20240530-102543", "question": "What is the third dirty dish I load in the dishwasher?", "answer_choice_0": "Knife", "answer_choice_1": "Spoon", "answer_choice_2": "Bowl", "answer_choice_3": "Pan", "answer_choice_4": "Mug", "answer_id": 2, "answer": "Bowl", "question_type": [ "Object Recognition", "Action Recognition", "Egocentric & Spatial Perception", "Situational Awareness" ], "objects_of_interest": [ "piece of paper-e3c4dd5595274b83", "pen-1d1bedb0692b222b", "small cheese grater-3880df074397175b", "glass-4a4dfe5f7f4d15b5", "glass2-dcf718dfba2d6c1e", "glass3-dc8815d66dc23e97", "glass4-710519b3c22aa5a4", "mug-c2160817972a2375", "mug2-884a6d20d120dd9b", "bowl-1f2a19a60439d92e", "bowl2-0a125aca643e1b8e", "bowl3-f5d0cc006f25122e", "wooden spoon-288bfd3197a48a8d", "knife-faa9d7c1a68cb98c", "glass jug-0208d781dfddc260", "glass jug2-6be21e0d05a66b24", "bowl4-2671372265986d18", "dish-4eb2321eef3a9ea8", "dish2-8750ca603b078a3e", "dish3-72445a6842983782", "dish4-fae14162ec1502c9", "dish5-d0066438fa9d0764", "dish6-d0531b05732a4ed2", "plate-2f744d8182f02eac", "plastic tub-27783f8e52eeada1", "plastic box2-38f70b13f90caa87", "mug3-2e016a8533b887cd", "mug4-6ba48a851f8de918", "cutlery organiser-d2f57b83ab5a048c", "cutlery-69547d0e85c016a7", "cutlery2-e9039880cc5ae9a2", "cutlery3-1e46994a38e42af8", "cutlery4-bbaf0ef0f5e2d0f5", "cutlery5-79da77cef673ba14", "forks-2d9550fdb651c17b", "forks2-728016f98db2b29d", "cutlery6-25ac101e20948004", "cutlery7-f114212e23983fb4", "cutlery8-b1460df52355e712", "cutlery9-b722a9001f288c68", "cutlery10-b58fbfc63703903d", "cutlery11-9a6dffed8bf661bd", "knives-4e10802612bd2112", "cutlery12-61a6b90d19179483", "cutlery13-d9622eba701c3eb0", "cutlery14-3140d919d08c2a44", "spoon-187843e083891950", "wooden spoon2-3f392d8e7ca9baa9", "glass5-ad10e5acdb7bf5cc", "bowl5-a1811a30c0b4c95a", "glass6-d41100715e7bfa0e", "mug5-98164392d067525b", "bowl6-22948cff3c9a735a", "digital weighing scale-d18eaa1e2759e89e", "butter pack-79f71b2222e02f6f", "knife2-65eb37cb61e079d4", "phone-99b7048ce196acdf", "bottle of dishwashing liquid-e46c8a5a6242974e", "fork3-f61478996797cd4a", "large silver bowl-1e7813e6e613a8c2", "packet of chocolate chips-0dfd92cec40be341", "packet of sugar-3b15c481102a833f", "another packet of sugar-f3cfce63317849c4", "frozen banana-e1dcd9734165a98e", "small bowl-edc59345ebbbf00d", "stool-746961e147f9c80b", "eggshell halves-1f61d7324317ade6", "egg-0ea47b75fd5e54e0", "dishcloth-55b4dad939e268fe", "food colouring-25f4b62e069fb91a", "bottle of vanilla extract-c74628b4f0c44823", "different bottle of vanilla extract-6faaa4e7ab87aa43", "cap of vanilla extract-9b0a2df70b4af5c9", "measuring spoon-79aa4b829b720cae", "self-raising flour-b22f670b3d29fdf5", "jar of cinnamon-2c78c72e1436808e", "lid-87bef665f302d4dd", "bundle of measuring spoons-e7547c0cd4e140c1", "wooden spoon3-25be19ed35e79e70", "flour2-ce8eebab899cb76a", "teaspoon-21a63251a249bdb1", "tea towel-1d6b81273de3c112", "baking tray-2c38c6880ed32299", "another tray-08d98dbb7a2290e7", "baking tray3-f267fe8668a0e669", "baking tray2-c8dacad37c5d99f6", "large frying pan-8f165e52c0a1907b", "saucepan-d77c6538c765b529", "colander-b6dc83a8108d1092", "saucepan2-5bcb83c6b05845fe", "stack of pans-dc779649831e4d7d", "lid2-be6c135443054d64", "spoon2-70ca298babbde1e0", "orange sponge-91f0584e90d28efb", "cloth-42233e44c039ef1e", "scrubbing brush-e6b13dee9d26a23e", "pair of tongs-67a14eff4502fbf6", "wooden spoon4-64492f46be762100", "small bowl2-f7e1155f875d754a", "knife3-50ca003f99bdc912", "knife4-d668923b0458ec0a", "torn off piece of baking paper-51e59bd8cf995835", "baking paper-575d2b02b73b0734", "tea towel2-d839e5738570e458", "teaspoon2-387d4b2deae96246", "teaspoon3-1d4e3d773bbfb682", "teaspoon4-920a37a312e4b45d", "coffee basket-13a2d19317e7799b", "drawer of coffee grinder-6dfb938c5e8e96c5", "coffee grinder-0aefaa1990908ac5", "coffee beans package-9e75b0d00284cea1", "lid of coffee grinder-a446ab50984202f1", "coffee spoon-15cd40a172d1b8f3", "lid of drawer of coffee grinder-20efd78f1cb1701d", "mug6-63788c45b999cee4", "drip tray-a7a99e98d6075642", "spoon3-4cc6fcca892d5ec4", "spoon4-26d9a1e27ce73a8d", "baking paper2-7dd10eaaedeb9b03" ], "reasoning": "1. As I watched the video, I observed the user open the dishwasher at 00:10.\n2. Next, I saw the user unload the dishwasher from 00:10-02:50.\n3. After removing the last spoons (cutlery14-3140d919d08c2a44) from the utensil holder (cutlery organiser-d2f57b83ab5a048c), I then saw the user grab a dirty spoon (spoon-187843e083891950) at 02:51-02:53 and load it into the utensil holder. This is the first item loaded into the dishwasher .\n4. Next, from 03:09-03:12, the user grabs the glass (glass-4a4dfe5f7f4d15b5) and puts it in the top rack. This is the second item loaded into the dishwasher .\n5. I then observed the user grab the bowl (bowl5-a1811a30c0b4c95a) and load it into the dishwasher from 03:15-03:17.\n6. Thus, the bowl (bowl5-a1811a30c0b4c95a) is the third dirty dish put into the dishwasher .", "referenced_object_masks": [ "bowl5-a1811a30c0b4c95a", "cutlery14-3140d919d08c2a44", "cutlery organiser-d2f57b83ab5a048c", "spoon-187843e083891950", "glass-4a4dfe5f7f4d15b5" ] }, { "key": "P07-20240530-102543:85e1d619bfcefdabe29336f2916cdc5938661fc2", "video_id": "P07-20240530-102543", "question": "After the scale reads over \"20\", what is the first thing I touch with only my left hand, that my right hand doesn't also touch?", "answer_choice_0": "A knife", "answer_choice_1": "The sugar bag", "answer_choice_2": "Kitchen faucet", "answer_choice_3": "The cabinet handle", "answer_choice_4": "Measuring cup", "answer_id": 3, "answer": "The cabinet handle", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Event Occurence", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "piece of paper-e3c4dd5595274b83", "pen-1d1bedb0692b222b", "small cheese grater-3880df074397175b", "glass-4a4dfe5f7f4d15b5", "glass2-dcf718dfba2d6c1e", "glass3-dc8815d66dc23e97", "glass4-710519b3c22aa5a4", "mug-c2160817972a2375", "mug2-884a6d20d120dd9b", "bowl-1f2a19a60439d92e", "bowl2-0a125aca643e1b8e", "bowl3-f5d0cc006f25122e", "wooden spoon-288bfd3197a48a8d", "knife-faa9d7c1a68cb98c", "glass jug-0208d781dfddc260", "glass jug2-6be21e0d05a66b24", "bowl4-2671372265986d18", "dish-4eb2321eef3a9ea8", "dish2-8750ca603b078a3e", "dish3-72445a6842983782", "dish4-fae14162ec1502c9", "dish5-d0066438fa9d0764", "dish6-d0531b05732a4ed2", "plate-2f744d8182f02eac", "plastic tub-27783f8e52eeada1", "plastic box2-38f70b13f90caa87", "mug3-2e016a8533b887cd", "mug4-6ba48a851f8de918", "cutlery organiser-d2f57b83ab5a048c", "cutlery-69547d0e85c016a7", "cutlery2-e9039880cc5ae9a2", "cutlery3-1e46994a38e42af8", "cutlery4-bbaf0ef0f5e2d0f5", "cutlery5-79da77cef673ba14", "forks-2d9550fdb651c17b", "forks2-728016f98db2b29d", "cutlery6-25ac101e20948004", "cutlery7-f114212e23983fb4", "cutlery8-b1460df52355e712", "cutlery9-b722a9001f288c68", "cutlery10-b58fbfc63703903d", "cutlery11-9a6dffed8bf661bd", "knives-4e10802612bd2112", "cutlery12-61a6b90d19179483", "cutlery13-d9622eba701c3eb0", "cutlery14-3140d919d08c2a44", "spoon-187843e083891950", "wooden spoon2-3f392d8e7ca9baa9", "glass5-ad10e5acdb7bf5cc", "bowl5-a1811a30c0b4c95a", "glass6-d41100715e7bfa0e", "mug5-98164392d067525b", "bowl6-22948cff3c9a735a", "digital weighing scale-d18eaa1e2759e89e", "butter pack-79f71b2222e02f6f", "knife2-65eb37cb61e079d4", "phone-99b7048ce196acdf", "bottle of dishwashing liquid-e46c8a5a6242974e", "fork3-f61478996797cd4a", "large silver bowl-1e7813e6e613a8c2", "packet of chocolate chips-0dfd92cec40be341", "packet of sugar-3b15c481102a833f", "another packet of sugar-f3cfce63317849c4", "frozen banana-e1dcd9734165a98e", "small bowl-edc59345ebbbf00d", "stool-746961e147f9c80b", "eggshell halves-1f61d7324317ade6", "egg-0ea47b75fd5e54e0", "dishcloth-55b4dad939e268fe", "food colouring-25f4b62e069fb91a", "bottle of vanilla extract-c74628b4f0c44823", "different bottle of vanilla extract-6faaa4e7ab87aa43", "cap of vanilla extract-9b0a2df70b4af5c9", "measuring spoon-79aa4b829b720cae", "self-raising flour-b22f670b3d29fdf5", "jar of cinnamon-2c78c72e1436808e", "lid-87bef665f302d4dd", "bundle of measuring spoons-e7547c0cd4e140c1", "wooden spoon3-25be19ed35e79e70", "flour2-ce8eebab899cb76a", "teaspoon-21a63251a249bdb1", "tea towel-1d6b81273de3c112", "baking tray-2c38c6880ed32299", "another tray-08d98dbb7a2290e7", "baking tray3-f267fe8668a0e669", "baking tray2-c8dacad37c5d99f6", "large frying pan-8f165e52c0a1907b", "saucepan-d77c6538c765b529", "colander-b6dc83a8108d1092", "saucepan2-5bcb83c6b05845fe", "stack of pans-dc779649831e4d7d", "lid2-be6c135443054d64", "spoon2-70ca298babbde1e0", "orange sponge-91f0584e90d28efb", "cloth-42233e44c039ef1e", "scrubbing brush-e6b13dee9d26a23e", "pair of tongs-67a14eff4502fbf6", "wooden spoon4-64492f46be762100", "small bowl2-f7e1155f875d754a", "knife3-50ca003f99bdc912", "knife4-d668923b0458ec0a", "torn off piece of baking paper-51e59bd8cf995835", "baking paper-575d2b02b73b0734", "tea towel2-d839e5738570e458", "teaspoon2-387d4b2deae96246", "teaspoon3-1d4e3d773bbfb682", "teaspoon4-920a37a312e4b45d", "coffee basket-13a2d19317e7799b", "drawer of coffee grinder-6dfb938c5e8e96c5", "coffee grinder-0aefaa1990908ac5", "coffee beans package-9e75b0d00284cea1", "lid of coffee grinder-a446ab50984202f1", "coffee spoon-15cd40a172d1b8f3", "lid of drawer of coffee grinder-20efd78f1cb1701d", "mug6-63788c45b999cee4", "drip tray-a7a99e98d6075642", "spoon3-4cc6fcca892d5ec4", "spoon4-26d9a1e27ce73a8d", "baking paper2-7dd10eaaedeb9b03" ], "reasoning": "1. As I watched the video, I noted that from 04:02 to 04:08 the user gets out a digital scale (digital weighing scale-d18eaa1e2759e89e) and sets it on the counter .\n2. I then watched as the user weighed butter (butter pack-79f71b2222e02f6f) from 04:53-05:07 though the scale (digital weighing scale-d18eaa1e2759e89e) never read over \"20\", so that criteria has not been met yet.\n3. Later in the video, from 07:02-07:30, the user weighs sugar (packet of sugar-3b15c481102a833f) using the scale (digital weighing scale-d18eaa1e2759e89e).\n4. In this time span, the user weighs in multiple stages, stopping eventually at 07:30 when the scale (digital weighing scale-d18eaa1e2759e89e) reads \"20.2\".\n5. Next, I saw the user grab the sugar bag (packet of sugar-3b15c481102a833f) with both hands from 07:33-07:35, which is not exclusively the left hand.\n6. Finally, at 07:35, the user grabs the cabinet door with just the left hand, indicating this as the answer to the question.", "referenced_object_masks": [ "packet of sugar-3b15c481102a833f", "butter pack-79f71b2222e02f6f", "digital weighing scale-d18eaa1e2759e89e" ] }, { "key": "P07-20240530-102543:8bf5073c9345db7086e286dbefdd866f9a30a089", "video_id": "P07-20240530-102543", "question": "What do the first 2 unique ingredients placed into the mixing bowl share in common with the ingredient prepared at 11:06?", "answer_choice_0": "They share the same consistency", "answer_choice_1": "They share the same flavors", "answer_choice_2": "They share nothing in common", "answer_choice_3": "They share the same colors", "answer_choice_4": "They are tasty eaten raw", "answer_id": 3, "answer": "They share the same colors", "question_type": [ "Temporal Reasoning", "Object Recognition", "Situational Awareness", "Action Recognition" ], "objects_of_interest": [ "piece of paper-e3c4dd5595274b83", "pen-1d1bedb0692b222b", "small cheese grater-3880df074397175b", "glass-4a4dfe5f7f4d15b5", "glass2-dcf718dfba2d6c1e", "glass3-dc8815d66dc23e97", "glass4-710519b3c22aa5a4", "mug-c2160817972a2375", "mug2-884a6d20d120dd9b", "bowl-1f2a19a60439d92e", "bowl2-0a125aca643e1b8e", "bowl3-f5d0cc006f25122e", "wooden spoon-288bfd3197a48a8d", "knife-faa9d7c1a68cb98c", "glass jug-0208d781dfddc260", "glass jug2-6be21e0d05a66b24", "bowl4-2671372265986d18", "dish-4eb2321eef3a9ea8", "dish2-8750ca603b078a3e", "dish3-72445a6842983782", "dish4-fae14162ec1502c9", "dish5-d0066438fa9d0764", "dish6-d0531b05732a4ed2", "plate-2f744d8182f02eac", "plastic tub-27783f8e52eeada1", "plastic box2-38f70b13f90caa87", "mug3-2e016a8533b887cd", "mug4-6ba48a851f8de918", "cutlery organiser-d2f57b83ab5a048c", "cutlery-69547d0e85c016a7", "cutlery2-e9039880cc5ae9a2", "cutlery3-1e46994a38e42af8", "cutlery4-bbaf0ef0f5e2d0f5", "cutlery5-79da77cef673ba14", "forks-2d9550fdb651c17b", "forks2-728016f98db2b29d", "cutlery6-25ac101e20948004", "cutlery7-f114212e23983fb4", "cutlery8-b1460df52355e712", "cutlery9-b722a9001f288c68", "cutlery10-b58fbfc63703903d", "cutlery11-9a6dffed8bf661bd", "knives-4e10802612bd2112", "cutlery12-61a6b90d19179483", "cutlery13-d9622eba701c3eb0", "cutlery14-3140d919d08c2a44", "spoon-187843e083891950", "wooden spoon2-3f392d8e7ca9baa9", "glass5-ad10e5acdb7bf5cc", "bowl5-a1811a30c0b4c95a", "glass6-d41100715e7bfa0e", "mug5-98164392d067525b", "bowl6-22948cff3c9a735a", "digital weighing scale-d18eaa1e2759e89e", "butter pack-79f71b2222e02f6f", "knife2-65eb37cb61e079d4", "phone-99b7048ce196acdf", "bottle of dishwashing liquid-e46c8a5a6242974e", "fork3-f61478996797cd4a", "large silver bowl-1e7813e6e613a8c2", "packet of chocolate chips-0dfd92cec40be341", "packet of sugar-3b15c481102a833f", "another packet of sugar-f3cfce63317849c4", "frozen banana-e1dcd9734165a98e", "small bowl-edc59345ebbbf00d", "stool-746961e147f9c80b", "eggshell halves-1f61d7324317ade6", "egg-0ea47b75fd5e54e0", "dishcloth-55b4dad939e268fe", "food colouring-25f4b62e069fb91a", "bottle of vanilla extract-c74628b4f0c44823", "different bottle of vanilla extract-6faaa4e7ab87aa43", "cap of vanilla extract-9b0a2df70b4af5c9", "measuring spoon-79aa4b829b720cae", "self-raising flour-b22f670b3d29fdf5", "jar of cinnamon-2c78c72e1436808e", "lid-87bef665f302d4dd", "bundle of measuring spoons-e7547c0cd4e140c1", "wooden spoon3-25be19ed35e79e70", "flour2-ce8eebab899cb76a", "teaspoon-21a63251a249bdb1", "tea towel-1d6b81273de3c112", "baking tray-2c38c6880ed32299", "another tray-08d98dbb7a2290e7", "baking tray3-f267fe8668a0e669", "baking tray2-c8dacad37c5d99f6", "large frying pan-8f165e52c0a1907b", "saucepan-d77c6538c765b529", "colander-b6dc83a8108d1092", "saucepan2-5bcb83c6b05845fe", "stack of pans-dc779649831e4d7d", "lid2-be6c135443054d64", "spoon2-70ca298babbde1e0", "orange sponge-91f0584e90d28efb", "cloth-42233e44c039ef1e", "scrubbing brush-e6b13dee9d26a23e", "pair of tongs-67a14eff4502fbf6", "wooden spoon4-64492f46be762100", "small bowl2-f7e1155f875d754a", "knife3-50ca003f99bdc912", "knife4-d668923b0458ec0a", "torn off piece of baking paper-51e59bd8cf995835", "baking paper-575d2b02b73b0734", "tea towel2-d839e5738570e458", "teaspoon2-387d4b2deae96246", "teaspoon3-1d4e3d773bbfb682", "teaspoon4-920a37a312e4b45d", "coffee basket-13a2d19317e7799b", "drawer of coffee grinder-6dfb938c5e8e96c5", "coffee grinder-0aefaa1990908ac5", "coffee beans package-9e75b0d00284cea1", "lid of coffee grinder-a446ab50984202f1", "coffee spoon-15cd40a172d1b8f3", "lid of drawer of coffee grinder-20efd78f1cb1701d", "mug6-63788c45b999cee4", "drip tray-a7a99e98d6075642", "spoon3-4cc6fcca892d5ec4", "spoon4-26d9a1e27ce73a8d", "baking paper2-7dd10eaaedeb9b03" ], "reasoning": "1. I look for the first 2 unique ingredients added to the mixing bowl (large silver bowl-1e7813e6e613a8c2).\n2. The first is sugar (another packet of sugar-f3cfce63317849c4), added to the bowl at 07:04 and at 07:20 from a different packet ('packet of sugar-3b15c481102a833f).\n3. The second ingredient added is melted butter from a packet of butter (butter pack-79f71b2222e02f6f), added at 07:47.\n4. I look to see the ingredient prepared at 11:06, and I notice that it is an egg (egg-0ea47b75fd5e54e0). The user is in the process of separating its white from its yolk.\n5. Examining these ingredients, I realize that the egg (egg-0ea47b75fd5e54e0) is white from its shell and yellow from its yolk, and, simultaneously, the sugar (packet of sugar-3b15c481102a833f, another packet of sugar-f3cfce63317849c4) is white and the melted butter from the pack (butter pack-79f71b2222e02f6f) is yellow.\n6. Therefore, these ingredients share the same color profile.", "referenced_object_masks": [ "packet of sugar-3b15c481102a833f", "another packet of sugar-f3cfce63317849c4", "butter pack-79f71b2222e02f6f", "egg-0ea47b75fd5e54e0", "large silver bowl-1e7813e6e613a8c2" ] }, { "key": "P07-20240530-102543:b0338239c89a828694170a8433d1b17b8dc51b7a", "video_id": "P07-20240530-102543", "question": "After I spend 23 seconds washing the second baking sheet I handle in the sink (counting from the time I turn on the faucet), what 2 patterns of scrubbing do I make on the sheet?", "answer_choice_0": "Up and down, side to side", "answer_choice_1": "Outlining the sides, up and down", "answer_choice_2": "Zig zag motion, up and down", "answer_choice_3": "Side to side, outlining the sides", "answer_choice_4": "Circular, Side to side", "answer_id": 3, "answer": "Side to side, outlining the sides", "question_type": [ "Action Recognition", "Temporal Reasoning", "Object Recognition", "Counting", "Egocentric & Spatial Perception", "State Changes" ], "objects_of_interest": [ "piece of paper-e3c4dd5595274b83", "pen-1d1bedb0692b222b", "small cheese grater-3880df074397175b", "glass-4a4dfe5f7f4d15b5", "glass2-dcf718dfba2d6c1e", "glass3-dc8815d66dc23e97", "glass4-710519b3c22aa5a4", "mug-c2160817972a2375", "mug2-884a6d20d120dd9b", "bowl-1f2a19a60439d92e", "bowl2-0a125aca643e1b8e", "bowl3-f5d0cc006f25122e", "wooden spoon-288bfd3197a48a8d", "knife-faa9d7c1a68cb98c", "glass jug-0208d781dfddc260", "glass jug2-6be21e0d05a66b24", "bowl4-2671372265986d18", "dish-4eb2321eef3a9ea8", "dish2-8750ca603b078a3e", "dish3-72445a6842983782", "dish4-fae14162ec1502c9", "dish5-d0066438fa9d0764", "dish6-d0531b05732a4ed2", "plate-2f744d8182f02eac", "plastic tub-27783f8e52eeada1", "plastic box2-38f70b13f90caa87", "mug3-2e016a8533b887cd", "mug4-6ba48a851f8de918", "cutlery organiser-d2f57b83ab5a048c", "cutlery-69547d0e85c016a7", "cutlery2-e9039880cc5ae9a2", "cutlery3-1e46994a38e42af8", "cutlery4-bbaf0ef0f5e2d0f5", "cutlery5-79da77cef673ba14", "forks-2d9550fdb651c17b", "forks2-728016f98db2b29d", "cutlery6-25ac101e20948004", "cutlery7-f114212e23983fb4", "cutlery8-b1460df52355e712", "cutlery9-b722a9001f288c68", "cutlery10-b58fbfc63703903d", "cutlery11-9a6dffed8bf661bd", "knives-4e10802612bd2112", "cutlery12-61a6b90d19179483", "cutlery13-d9622eba701c3eb0", "cutlery14-3140d919d08c2a44", "spoon-187843e083891950", "wooden spoon2-3f392d8e7ca9baa9", "glass5-ad10e5acdb7bf5cc", "bowl5-a1811a30c0b4c95a", "glass6-d41100715e7bfa0e", "mug5-98164392d067525b", "bowl6-22948cff3c9a735a", "digital weighing scale-d18eaa1e2759e89e", "butter pack-79f71b2222e02f6f", "knife2-65eb37cb61e079d4", "phone-99b7048ce196acdf", "bottle of dishwashing liquid-e46c8a5a6242974e", "fork3-f61478996797cd4a", "large silver bowl-1e7813e6e613a8c2", "packet of chocolate chips-0dfd92cec40be341", "packet of sugar-3b15c481102a833f", "another packet of sugar-f3cfce63317849c4", "frozen banana-e1dcd9734165a98e", "small bowl-edc59345ebbbf00d", "stool-746961e147f9c80b", "eggshell halves-1f61d7324317ade6", "egg-0ea47b75fd5e54e0", "dishcloth-55b4dad939e268fe", "food colouring-25f4b62e069fb91a", "bottle of vanilla extract-c74628b4f0c44823", "different bottle of vanilla extract-6faaa4e7ab87aa43", "cap of vanilla extract-9b0a2df70b4af5c9", "measuring spoon-79aa4b829b720cae", "self-raising flour-b22f670b3d29fdf5", "jar of cinnamon-2c78c72e1436808e", "lid-87bef665f302d4dd", "bundle of measuring spoons-e7547c0cd4e140c1", "wooden spoon3-25be19ed35e79e70", "flour2-ce8eebab899cb76a", "teaspoon-21a63251a249bdb1", "tea towel-1d6b81273de3c112", "baking tray-2c38c6880ed32299", "another tray-08d98dbb7a2290e7", "baking tray3-f267fe8668a0e669", "baking tray2-c8dacad37c5d99f6", "large frying pan-8f165e52c0a1907b", "saucepan-d77c6538c765b529", "colander-b6dc83a8108d1092", "saucepan2-5bcb83c6b05845fe", "stack of pans-dc779649831e4d7d", "lid2-be6c135443054d64", "spoon2-70ca298babbde1e0", "orange sponge-91f0584e90d28efb", "cloth-42233e44c039ef1e", "scrubbing brush-e6b13dee9d26a23e", "pair of tongs-67a14eff4502fbf6", "wooden spoon4-64492f46be762100", "small bowl2-f7e1155f875d754a", "knife3-50ca003f99bdc912", "knife4-d668923b0458ec0a", "torn off piece of baking paper-51e59bd8cf995835", "baking paper-575d2b02b73b0734", "tea towel2-d839e5738570e458", "teaspoon2-387d4b2deae96246", "teaspoon3-1d4e3d773bbfb682", "teaspoon4-920a37a312e4b45d", "coffee basket-13a2d19317e7799b", "drawer of coffee grinder-6dfb938c5e8e96c5", "coffee grinder-0aefaa1990908ac5", "coffee beans package-9e75b0d00284cea1", "lid of coffee grinder-a446ab50984202f1", "coffee spoon-15cd40a172d1b8f3", "lid of drawer of coffee grinder-20efd78f1cb1701d", "mug6-63788c45b999cee4", "drip tray-a7a99e98d6075642", "spoon3-4cc6fcca892d5ec4", "spoon4-26d9a1e27ce73a8d", "baking paper2-7dd10eaaedeb9b03" ], "reasoning": "1. I look for the time the user handles the second baking sheet.\n2. The user handles the first baking sheet at 20:14 (baking tray-2c38c6880ed32299).\n3. At 20:22, the user brings out a second baking sheet (baking tray2-c8dacad37c5d99f6). At 20:30, the user puts the second baking sheet (baking tray2-c8dacad37c5d99f6) into the sink, and, at 21:28, the user turns on the faucet to prepare to wash it.\n4. 23 seconds after 21:28 results in the timecode 21:51, so I observe the scrubbing patterns made on the sheet after this time.\n5. The first scrubbing pattern I notice, from 21:51-21:54, is of the user scrubbing side to side, from the top down.\n6. The second scrubbing pattern I notice, from 21:54-21:56, is of the user outlining the sides of the baking sheet (baking tray2-c8dacad37c5d99f6) in a square-like fashion.\n7. Therefore, these are the 2 scrubbing patterns made on the baking sheet (baking tray2-c8dacad37c5d99f6).", "referenced_object_masks": [ "baking tray2-c8dacad37c5d99f6", "baking tray-2c38c6880ed32299" ] }, { "key": "P07-20240530-102543:b410cf745e72dd08ce467c5fde8e9b6aa5829a79", "video_id": "P07-20240530-102543", "question": "While putting the dishes away, how many times do I return to the dishwasher after reaching up into a cupboard before I put the mug with the duck away?", "answer_choice_0": "2 times", "answer_choice_1": "5 times", "answer_choice_2": "6 times", "answer_choice_3": "3 times", "answer_choice_4": "4 times", "answer_id": 4, "answer": "4 times", "question_type": [ "Egocentric & Spatial Perception", "Action Recognition", "Object Recognition", "Situational Awareness", "Counting", "Temporal Reasoning" ], "objects_of_interest": [ "piece of paper-e3c4dd5595274b83", "pen-1d1bedb0692b222b", "small cheese grater-3880df074397175b", "glass-4a4dfe5f7f4d15b5", "glass2-dcf718dfba2d6c1e", "glass3-dc8815d66dc23e97", "glass4-710519b3c22aa5a4", "mug-c2160817972a2375", "mug2-884a6d20d120dd9b", "bowl-1f2a19a60439d92e", "bowl2-0a125aca643e1b8e", "bowl3-f5d0cc006f25122e", "wooden spoon-288bfd3197a48a8d", "knife-faa9d7c1a68cb98c", "glass jug-0208d781dfddc260", "glass jug2-6be21e0d05a66b24", "bowl4-2671372265986d18", "dish-4eb2321eef3a9ea8", "dish2-8750ca603b078a3e", "dish3-72445a6842983782", "dish4-fae14162ec1502c9", "dish5-d0066438fa9d0764", "dish6-d0531b05732a4ed2", "plate-2f744d8182f02eac", "plastic tub-27783f8e52eeada1", "plastic box2-38f70b13f90caa87", "mug3-2e016a8533b887cd", "mug4-6ba48a851f8de918", "cutlery organiser-d2f57b83ab5a048c", "cutlery-69547d0e85c016a7", "cutlery2-e9039880cc5ae9a2", "cutlery3-1e46994a38e42af8", "cutlery4-bbaf0ef0f5e2d0f5", "cutlery5-79da77cef673ba14", "forks-2d9550fdb651c17b", "forks2-728016f98db2b29d", "cutlery6-25ac101e20948004", "cutlery7-f114212e23983fb4", "cutlery8-b1460df52355e712", "cutlery9-b722a9001f288c68", "cutlery10-b58fbfc63703903d", "cutlery11-9a6dffed8bf661bd", "knives-4e10802612bd2112", "cutlery12-61a6b90d19179483", "cutlery13-d9622eba701c3eb0", "cutlery14-3140d919d08c2a44", "spoon-187843e083891950", "wooden spoon2-3f392d8e7ca9baa9", "glass5-ad10e5acdb7bf5cc", "bowl5-a1811a30c0b4c95a", "glass6-d41100715e7bfa0e", "mug5-98164392d067525b", "bowl6-22948cff3c9a735a", "digital weighing scale-d18eaa1e2759e89e", "butter pack-79f71b2222e02f6f", "knife2-65eb37cb61e079d4", "phone-99b7048ce196acdf", "bottle of dishwashing liquid-e46c8a5a6242974e", "fork3-f61478996797cd4a", "large silver bowl-1e7813e6e613a8c2", "packet of chocolate chips-0dfd92cec40be341", "packet of sugar-3b15c481102a833f", "another packet of sugar-f3cfce63317849c4", "frozen banana-e1dcd9734165a98e", "small bowl-edc59345ebbbf00d", "stool-746961e147f9c80b", "eggshell halves-1f61d7324317ade6", "egg-0ea47b75fd5e54e0", "dishcloth-55b4dad939e268fe", "food colouring-25f4b62e069fb91a", "bottle of vanilla extract-c74628b4f0c44823", "different bottle of vanilla extract-6faaa4e7ab87aa43", "cap of vanilla extract-9b0a2df70b4af5c9", "measuring spoon-79aa4b829b720cae", "self-raising flour-b22f670b3d29fdf5", "jar of cinnamon-2c78c72e1436808e", "lid-87bef665f302d4dd", "bundle of measuring spoons-e7547c0cd4e140c1", "wooden spoon3-25be19ed35e79e70", "flour2-ce8eebab899cb76a", "teaspoon-21a63251a249bdb1", "tea towel-1d6b81273de3c112", "baking tray-2c38c6880ed32299", "another tray-08d98dbb7a2290e7", "baking tray3-f267fe8668a0e669", "baking tray2-c8dacad37c5d99f6", "large frying pan-8f165e52c0a1907b", "saucepan-d77c6538c765b529", "colander-b6dc83a8108d1092", "saucepan2-5bcb83c6b05845fe", "stack of pans-dc779649831e4d7d", "lid2-be6c135443054d64", "spoon2-70ca298babbde1e0", "orange sponge-91f0584e90d28efb", "cloth-42233e44c039ef1e", "scrubbing brush-e6b13dee9d26a23e", "pair of tongs-67a14eff4502fbf6", "wooden spoon4-64492f46be762100", "small bowl2-f7e1155f875d754a", "knife3-50ca003f99bdc912", "knife4-d668923b0458ec0a", "torn off piece of baking paper-51e59bd8cf995835", "baking paper-575d2b02b73b0734", "tea towel2-d839e5738570e458", "teaspoon2-387d4b2deae96246", "teaspoon3-1d4e3d773bbfb682", "teaspoon4-920a37a312e4b45d", "coffee basket-13a2d19317e7799b", "drawer of coffee grinder-6dfb938c5e8e96c5", "coffee grinder-0aefaa1990908ac5", "coffee beans package-9e75b0d00284cea1", "lid of coffee grinder-a446ab50984202f1", "coffee spoon-15cd40a172d1b8f3", "lid of drawer of coffee grinder-20efd78f1cb1701d", "mug6-63788c45b999cee4", "drip tray-a7a99e98d6075642", "spoon3-4cc6fcca892d5ec4", "spoon4-26d9a1e27ce73a8d", "baking paper2-7dd10eaaedeb9b03" ], "reasoning": "1. I look for the time the user takes objects from the dishwasher, and I find the time from 00:10-02:51. After 2:51, the user begins to load the dishwasher with dirty dishes.\n2. The user returns to the dishwasher from 00:28-00:29 after reaching up into a cupboard.\n3. The user returns to the dishwasher a second time from 00:39-00:40 after reaching up into the cupboard from earlier.\n4. The user returns to the dishwasher a third time from 00:58-00:59 after reaching up into a different cupboard to put bowls (bowl-1f2a19a60439d92e, bowl2-0a125aca643e1b8e, bowl3-f5d0cc006f25122e) away.\n5. The user returns to the dishwasher a fourth time from 01:20-01:21, after once reaching up to put a white bowl (bowl4-2671372265986d18) into a cupboard at 01:12.\n6. The user returns to the dishwasher after this from 01:55-02:06, but at this time the user has not reached up into a cupboard, so I do not count this time.\n7. After this, the user puts a white mug with a duck on it (mug4-6ba48a851f8de918) away into a cupboard from 02:09-02:10, so I stop counting the times the user returns to the dishwasher.\n6. Counting all these instances up, I come to a total of 4 times.", "referenced_object_masks": [ "mug4-6ba48a851f8de918", "bowl-1f2a19a60439d92e", "bowl2-0a125aca643e1b8e", "bowl3-f5d0cc006f25122e", "bowl4-2671372265986d18" ] }, { "key": "P07-20240530-102543:b8e7faa7f019cb322f1fdb536ef2febeab21df00", "video_id": "P07-20240530-102543", "question": "Where did the thirteenth object taken from the dishwasher go?", "answer_choice_0": "Into a cupboard", "answer_choice_1": "Into the sink", "answer_choice_2": "Onto the counter", "answer_choice_3": "Into a knife block", "answer_choice_4": "Onto a cutting board", "answer_id": 3, "answer": "Into a knife block", "question_type": [ "Counting", "Action Recognition", "Situational Awareness", "Object Recognition" ], "objects_of_interest": [ "piece of paper-e3c4dd5595274b83", "pen-1d1bedb0692b222b", "small cheese grater-3880df074397175b", "glass-4a4dfe5f7f4d15b5", "glass2-dcf718dfba2d6c1e", "glass3-dc8815d66dc23e97", "glass4-710519b3c22aa5a4", "mug-c2160817972a2375", "mug2-884a6d20d120dd9b", "bowl-1f2a19a60439d92e", "bowl2-0a125aca643e1b8e", "bowl3-f5d0cc006f25122e", "wooden spoon-288bfd3197a48a8d", "knife-faa9d7c1a68cb98c", "glass jug-0208d781dfddc260", "glass jug2-6be21e0d05a66b24", "bowl4-2671372265986d18", "dish-4eb2321eef3a9ea8", "dish2-8750ca603b078a3e", "dish3-72445a6842983782", "dish4-fae14162ec1502c9", "dish5-d0066438fa9d0764", "dish6-d0531b05732a4ed2", "plate-2f744d8182f02eac", "plastic tub-27783f8e52eeada1", "plastic box2-38f70b13f90caa87", "mug3-2e016a8533b887cd", "mug4-6ba48a851f8de918", "cutlery organiser-d2f57b83ab5a048c", "cutlery-69547d0e85c016a7", "cutlery2-e9039880cc5ae9a2", "cutlery3-1e46994a38e42af8", "cutlery4-bbaf0ef0f5e2d0f5", "cutlery5-79da77cef673ba14", "forks-2d9550fdb651c17b", "forks2-728016f98db2b29d", "cutlery6-25ac101e20948004", "cutlery7-f114212e23983fb4", "cutlery8-b1460df52355e712", "cutlery9-b722a9001f288c68", "cutlery10-b58fbfc63703903d", "cutlery11-9a6dffed8bf661bd", "knives-4e10802612bd2112", "cutlery12-61a6b90d19179483", "cutlery13-d9622eba701c3eb0", "cutlery14-3140d919d08c2a44", "spoon-187843e083891950", "wooden spoon2-3f392d8e7ca9baa9", "glass5-ad10e5acdb7bf5cc", "bowl5-a1811a30c0b4c95a", "glass6-d41100715e7bfa0e", "mug5-98164392d067525b", "bowl6-22948cff3c9a735a", "digital weighing scale-d18eaa1e2759e89e", "butter pack-79f71b2222e02f6f", "knife2-65eb37cb61e079d4", "phone-99b7048ce196acdf", "bottle of dishwashing liquid-e46c8a5a6242974e", "fork3-f61478996797cd4a", "large silver bowl-1e7813e6e613a8c2", "packet of chocolate chips-0dfd92cec40be341", "packet of sugar-3b15c481102a833f", "another packet of sugar-f3cfce63317849c4", "frozen banana-e1dcd9734165a98e", "small bowl-edc59345ebbbf00d", "stool-746961e147f9c80b", "eggshell halves-1f61d7324317ade6", "egg-0ea47b75fd5e54e0", "dishcloth-55b4dad939e268fe", "food colouring-25f4b62e069fb91a", "bottle of vanilla extract-c74628b4f0c44823", "different bottle of vanilla extract-6faaa4e7ab87aa43", "cap of vanilla extract-9b0a2df70b4af5c9", "measuring spoon-79aa4b829b720cae", "self-raising flour-b22f670b3d29fdf5", "jar of cinnamon-2c78c72e1436808e", "lid-87bef665f302d4dd", "bundle of measuring spoons-e7547c0cd4e140c1", "wooden spoon3-25be19ed35e79e70", "flour2-ce8eebab899cb76a", "teaspoon-21a63251a249bdb1", "tea towel-1d6b81273de3c112", "baking tray-2c38c6880ed32299", "another tray-08d98dbb7a2290e7", "baking tray3-f267fe8668a0e669", "baking tray2-c8dacad37c5d99f6", "large frying pan-8f165e52c0a1907b", "saucepan-d77c6538c765b529", "colander-b6dc83a8108d1092", "saucepan2-5bcb83c6b05845fe", "stack of pans-dc779649831e4d7d", "lid2-be6c135443054d64", "spoon2-70ca298babbde1e0", "orange sponge-91f0584e90d28efb", "cloth-42233e44c039ef1e", "scrubbing brush-e6b13dee9d26a23e", "pair of tongs-67a14eff4502fbf6", "wooden spoon4-64492f46be762100", "small bowl2-f7e1155f875d754a", "knife3-50ca003f99bdc912", "knife4-d668923b0458ec0a", "torn off piece of baking paper-51e59bd8cf995835", "baking paper-575d2b02b73b0734", "tea towel2-d839e5738570e458", "teaspoon2-387d4b2deae96246", "teaspoon3-1d4e3d773bbfb682", "teaspoon4-920a37a312e4b45d", "coffee basket-13a2d19317e7799b", "drawer of coffee grinder-6dfb938c5e8e96c5", "coffee grinder-0aefaa1990908ac5", "coffee beans package-9e75b0d00284cea1", "lid of coffee grinder-a446ab50984202f1", "coffee spoon-15cd40a172d1b8f3", "lid of drawer of coffee grinder-20efd78f1cb1701d", "mug6-63788c45b999cee4", "drip tray-a7a99e98d6075642", "spoon3-4cc6fcca892d5ec4", "spoon4-26d9a1e27ce73a8d", "baking paper2-7dd10eaaedeb9b03" ], "reasoning": "1. I look for the time the user takes objects from the dishwasher. From 00:10-02:51, the user opens the dishwasher and puts away dishes and utensils. After 2:51, the user begins to load the dishwasher with dirty dishes.\n2. I find that the first object taken from the dishwasher is a handheld cheese grater (small cheese grater-3880df074397175b) at 00:13.\n3. The next 3 objects taken from the dishwasher are drinking glasses: 1 (glass-4a4dfe5f7f4d15b5) from 00:16-00:17, 1 (glass2-dcf718dfba2d6c1e) from 00:17-00:18, and 1 (glass3-dc8815d66dc23e97) from 00:19-00:20.\n4. The next object, another drinking glass (glass4-710519b3c22aa5a4), is taken at 00:29.\n5. At 00:32, the user hooks a white mug with their left hand , and grabs a tan mug (mug-c2160817972a2375) and a blue mug . At 00:34, the 2 mugs (mug-c2160817972a2375) are revealed more clearly.\n6. The next 3 objects are 3 bowls, one (bowl-1f2a19a60439d92e) taken from 00:41-00:42, the other (bowl2-0a125aca643e1b8e) from 00:42-00:43, and the last (bowl3-f5d0cc006f25122e) from 00:46-00:48.\n7. From 01:00-01:01, the user takes a wooden spoon (wooden spoon-288bfd3197a48a8d) and a knife (knife-faa9d7c1a68cb98c) from the dishwasher.\n8. Counting all these items up, I realize that the thirteenth item is the knife (knife-faa9d7c1a68cb98c).\n9. From 01:02-01:03, I see the user put the knife (knife-faa9d7c1a68cb98c) in a knife block .\n10. Therefore, the answer is knife block .", "referenced_object_masks": [ "mug-c2160817972a2375", "bowl2-0a125aca643e1b8e", "glass3-dc8815d66dc23e97", "bowl3-f5d0cc006f25122e", "bowl-1f2a19a60439d92e", "knife-faa9d7c1a68cb98c", "glass2-dcf718dfba2d6c1e", "wooden spoon-288bfd3197a48a8d", "small cheese grater-3880df074397175b", "glass4-710519b3c22aa5a4", "glass-4a4dfe5f7f4d15b5" ] }, { "key": "P07-20240530-102543:ce58ceee7a132d9d28174d619498ec2bda1c1413", "video_id": "P07-20240530-102543", "question": "For how much longer do I stir the butter I melt in the measuring cup than I wash my hands?", "answer_choice_0": "5 seconds", "answer_choice_1": "10 seconds", "answer_choice_2": "3 seconds", "answer_choice_3": "9 seconds", "answer_choice_4": "12 seconds", "answer_id": 0, "answer": "5 seconds", "question_type": [ "Action Recognition", "Object Recognition", "Numerical Reasoning", "Temporal Reasoning", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "piece of paper-e3c4dd5595274b83", "pen-1d1bedb0692b222b", "small cheese grater-3880df074397175b", "glass-4a4dfe5f7f4d15b5", "glass2-dcf718dfba2d6c1e", "glass3-dc8815d66dc23e97", "glass4-710519b3c22aa5a4", "mug-c2160817972a2375", "mug2-884a6d20d120dd9b", "bowl-1f2a19a60439d92e", "bowl2-0a125aca643e1b8e", "bowl3-f5d0cc006f25122e", "wooden spoon-288bfd3197a48a8d", "knife-faa9d7c1a68cb98c", "glass jug-0208d781dfddc260", "glass jug2-6be21e0d05a66b24", "bowl4-2671372265986d18", "dish-4eb2321eef3a9ea8", "dish2-8750ca603b078a3e", "dish3-72445a6842983782", "dish4-fae14162ec1502c9", "dish5-d0066438fa9d0764", "dish6-d0531b05732a4ed2", "plate-2f744d8182f02eac", "plastic tub-27783f8e52eeada1", "plastic box2-38f70b13f90caa87", "mug3-2e016a8533b887cd", "mug4-6ba48a851f8de918", "cutlery organiser-d2f57b83ab5a048c", "cutlery-69547d0e85c016a7", "cutlery2-e9039880cc5ae9a2", "cutlery3-1e46994a38e42af8", "cutlery4-bbaf0ef0f5e2d0f5", "cutlery5-79da77cef673ba14", "forks-2d9550fdb651c17b", "forks2-728016f98db2b29d", "cutlery6-25ac101e20948004", "cutlery7-f114212e23983fb4", "cutlery8-b1460df52355e712", "cutlery9-b722a9001f288c68", "cutlery10-b58fbfc63703903d", "cutlery11-9a6dffed8bf661bd", "knives-4e10802612bd2112", "cutlery12-61a6b90d19179483", "cutlery13-d9622eba701c3eb0", "cutlery14-3140d919d08c2a44", "spoon-187843e083891950", "wooden spoon2-3f392d8e7ca9baa9", "glass5-ad10e5acdb7bf5cc", "bowl5-a1811a30c0b4c95a", "glass6-d41100715e7bfa0e", "mug5-98164392d067525b", "bowl6-22948cff3c9a735a", "digital weighing scale-d18eaa1e2759e89e", "butter pack-79f71b2222e02f6f", "knife2-65eb37cb61e079d4", "phone-99b7048ce196acdf", "bottle of dishwashing liquid-e46c8a5a6242974e", "fork3-f61478996797cd4a", "large silver bowl-1e7813e6e613a8c2", "packet of chocolate chips-0dfd92cec40be341", "packet of sugar-3b15c481102a833f", "another packet of sugar-f3cfce63317849c4", "frozen banana-e1dcd9734165a98e", "small bowl-edc59345ebbbf00d", "stool-746961e147f9c80b", "eggshell halves-1f61d7324317ade6", "egg-0ea47b75fd5e54e0", "dishcloth-55b4dad939e268fe", "food colouring-25f4b62e069fb91a", "bottle of vanilla extract-c74628b4f0c44823", "different bottle of vanilla extract-6faaa4e7ab87aa43", "cap of vanilla extract-9b0a2df70b4af5c9", "measuring spoon-79aa4b829b720cae", "self-raising flour-b22f670b3d29fdf5", "jar of cinnamon-2c78c72e1436808e", "lid-87bef665f302d4dd", "bundle of measuring spoons-e7547c0cd4e140c1", "wooden spoon3-25be19ed35e79e70", "flour2-ce8eebab899cb76a", "teaspoon-21a63251a249bdb1", "tea towel-1d6b81273de3c112", "baking tray-2c38c6880ed32299", "another tray-08d98dbb7a2290e7", "baking tray3-f267fe8668a0e669", "baking tray2-c8dacad37c5d99f6", "large frying pan-8f165e52c0a1907b", "saucepan-d77c6538c765b529", "colander-b6dc83a8108d1092", "saucepan2-5bcb83c6b05845fe", "stack of pans-dc779649831e4d7d", "lid2-be6c135443054d64", "spoon2-70ca298babbde1e0", "orange sponge-91f0584e90d28efb", "cloth-42233e44c039ef1e", "scrubbing brush-e6b13dee9d26a23e", "pair of tongs-67a14eff4502fbf6", "wooden spoon4-64492f46be762100", "small bowl2-f7e1155f875d754a", "knife3-50ca003f99bdc912", "knife4-d668923b0458ec0a", "torn off piece of baking paper-51e59bd8cf995835", "baking paper-575d2b02b73b0734", "tea towel2-d839e5738570e458", "teaspoon2-387d4b2deae96246", "teaspoon3-1d4e3d773bbfb682", "teaspoon4-920a37a312e4b45d", "coffee basket-13a2d19317e7799b", "drawer of coffee grinder-6dfb938c5e8e96c5", "coffee grinder-0aefaa1990908ac5", "coffee beans package-9e75b0d00284cea1", "lid of coffee grinder-a446ab50984202f1", "coffee spoon-15cd40a172d1b8f3", "lid of drawer of coffee grinder-20efd78f1cb1701d", "mug6-63788c45b999cee4", "drip tray-a7a99e98d6075642", "spoon3-4cc6fcca892d5ec4", "spoon4-26d9a1e27ce73a8d", "baking paper2-7dd10eaaedeb9b03" ], "reasoning": "1. I watched the video and noted that from 04:30-05:22, the user weighs butter (butter pack-79f71b2222e02f6f) on the scale (digital weighing scale-d18eaa1e2759e89e).\n2. After this, I saw the user go to the sink to wash their hands, which is performed from 05:33-05:49.\n3. I did the math and calculated that the user washed their hands for 16 seconds.\n4. After washing their hands, I watched as the user got a fork (fork3-f61478996797cd4a) and stirred the melted butter (butter pack-79f71b2222e02f6f) from 06:02-06:23.\n5. This was calculated to have occurred for 21 seconds.\n6. I then did the math and determined that the butter stirring lasted 5 seconds more, because 21 - 16 = 5.", "referenced_object_masks": [ "butter pack-79f71b2222e02f6f", "fork3-f61478996797cd4a", "digital weighing scale-d18eaa1e2759e89e" ] }, { "key": "P07-20240530-111344:3ea426c4c22ade6364cf8ff8e6b5a9b102fadd33", "video_id": "P07-20240530-111344", "question": "Where did I put the cookies in relation to the oven?", "answer_choice_0": "On top of the oven", "answer_choice_1": "To the left of the oven", "answer_choice_2": "Below the oven", "answer_choice_3": "To the right of the oven", "answer_choice_4": "On the table in front of the oven", "answer_id": 3, "answer": "To the right of the oven", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "orange tea towel-a72fcce28360e11f", "blue tea towel-6d52da71783cd927", "baking tray-59a17d88756108e6", "baking tray2-1a1ddbeb806d3148", "mug-14aa3ea51f4a4d24", "kettle-8180514a6b1c5b0f", "notepad-e7e91999a2194805", "pen-9dc9c721047937c3", "mixing bowl-e23c8d0e0e8e12f7", "spoon-6feecfe449e8fab1", "spoon2-70e665ce6b66852e", "fork-cc710e820cdb8033", "scrubbing brush-da7ef6d11ee8bd32" ], "reasoning": "1. I watched the video to observe the positioning of the oven and saw this position at 00:05.\n2. I then saw the user take out 2 trays (baking tray-59a17d88756108e6, baking tray2-1a1ddbeb806d3148) of cookies from the oven between 00:09-00:28.\n3. I saw the user place both of these trays (baking tray-59a17d88756108e6, baking tray2-1a1ddbeb806d3148) to the right of the oven .", "referenced_object_masks": [ "baking tray-59a17d88756108e6", "baking tray2-1a1ddbeb806d3148" ] }, { "key": "P07-20240530-111344:746d0eeae0170c1baea28027d502b57181984a3d", "video_id": "P07-20240530-111344", "question": "When I'm washing the dishes, in what relative direction is my mug to me?", "answer_choice_0": "6 o'clock", "answer_choice_1": "9 o'clock", "answer_choice_2": "12 o'clock", "answer_choice_3": "7 o'clock", "answer_choice_4": "2 o'clock", "answer_id": 3, "answer": "7 o'clock", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Counterfactual", "Action Recognition" ], "objects_of_interest": [ "orange tea towel-a72fcce28360e11f", "blue tea towel-6d52da71783cd927", "baking tray-59a17d88756108e6", "baking tray2-1a1ddbeb806d3148", "mug-14aa3ea51f4a4d24", "kettle-8180514a6b1c5b0f", "notepad-e7e91999a2194805", "pen-9dc9c721047937c3", "mixing bowl-e23c8d0e0e8e12f7", "spoon-6feecfe449e8fab1", "spoon2-70e665ce6b66852e", "fork-cc710e820cdb8033", "scrubbing brush-da7ef6d11ee8bd32" ], "reasoning": "1. I watched the video to observe the position of the user when they washed the dishes, which consists of the bowl (mixing bowl-e23c8d0e0e8e12f7) and the spoon (spoon-6feecfe449e8fab1), and saw the user facing the window at 01:14.\n2. I then watched the video to find the position of the mug (mug-14aa3ea51f4a4d24) when the user was washing the dishes and found the mug (mug-14aa3ea51f4a4d24) for the first time at 00:45.\n3. Between 00:45-00:58, the user moves the mug (mug-14aa3ea51f4a4d24) to the left and pours some water into it using the kettle (kettle-8180514a6b1c5b0f).\n4. Between 01:04-01:11, the user moves the mug (mug-14aa3ea51f4a4d24) to the kitchen island.\n5. The user then turned around completely between 01:12-01:14 and took a small step to the right to arrive at the position at the sink where they started to wash the dishes at 01:14.\n6. The mug (mug-14aa3ea51f4a4d24) is directly behind the user and off to the left a little bit because of the small step to the right the user took, so the mug (mug-14aa3ea51f4a4d24) in the 7 o'clock position relative to the user while they're washing the dishes.", "referenced_object_masks": [ "spoon-6feecfe449e8fab1", "mug-14aa3ea51f4a4d24", "kettle-8180514a6b1c5b0f", "mixing bowl-e23c8d0e0e8e12f7" ] }, { "key": "P07-20240530-111344:7f9f0ad264112be33f828e02fd8634fa195692ce", "video_id": "P07-20240530-111344", "question": "What percentage of the cutlery I put in the dishwasher are spoons?", "answer_choice_0": "25%", "answer_choice_1": "75%", "answer_choice_2": "50%", "answer_choice_3": "33.33%", "answer_choice_4": "66.66%", "answer_id": 4, "answer": "66.66%", "question_type": [ "Counterfactual", "Numerical Reasoning", "Object Recognition" ], "objects_of_interest": [ "orange tea towel-a72fcce28360e11f", "blue tea towel-6d52da71783cd927", "baking tray-59a17d88756108e6", "baking tray2-1a1ddbeb806d3148", "mug-14aa3ea51f4a4d24", "kettle-8180514a6b1c5b0f", "notepad-e7e91999a2194805", "pen-9dc9c721047937c3", "mixing bowl-e23c8d0e0e8e12f7", "spoon-6feecfe449e8fab1", "spoon2-70e665ce6b66852e", "fork-cc710e820cdb8033", "scrubbing brush-da7ef6d11ee8bd32" ], "reasoning": "1. I watched the video to find when the user put the cutlery in the dishwasher and saw this happen between 01:25-01:34.\n2. I observed the user putting away a spoon (spoon-6feecfe449e8fab1) at 01:30.\n3. I then observed the user putting away another spoon (spoon2-70e665ce6b66852e) at 01:32.\n4. Lastly, I observed the user putting away 1 fork (fork-cc710e820cdb8033) at 01:33.\n5. Therefore, the percentage of the cutlery that are spoons would be calculated out of 2/3, which would be 66.66%.", "referenced_object_masks": [ "spoon-6feecfe449e8fab1", "fork-cc710e820cdb8033", "spoon2-70e665ce6b66852e" ] }, { "key": "P07-20240530-111344:ba7244bd1e9275ca99a89bd6c55cdedc309f79c7", "video_id": "P07-20240530-111344", "question": "If I multiply the number of leaves on my plant by the number of cookies I made, what number would I get?", "answer_choice_0": "54", "answer_choice_1": "92", "answer_choice_2": "48", "answer_choice_3": "209", "answer_choice_4": "102", "answer_id": 2, "answer": "48", "question_type": [ "Counterfactual", "Object Recognition", "Numerical Reasoning" ], "objects_of_interest": [ "orange tea towel-a72fcce28360e11f", "blue tea towel-6d52da71783cd927", "baking tray-59a17d88756108e6", "baking tray2-1a1ddbeb806d3148", "mug-14aa3ea51f4a4d24", "kettle-8180514a6b1c5b0f", "notepad-e7e91999a2194805", "pen-9dc9c721047937c3", "mixing bowl-e23c8d0e0e8e12f7", "spoon-6feecfe449e8fab1", "spoon2-70e665ce6b66852e", "fork-cc710e820cdb8033", "scrubbing brush-da7ef6d11ee8bd32" ], "reasoning": "1. I watched the video to find the user's plant and saw it on their windowsill at 01:16, and it appears to have 3 leaves.\n2. I then watched the video to see how many cookies the user made and found this at 00:28 where there appears to be 16 cookies.\n3. 3x16=48", "referenced_object_masks": [] }, { "key": "P07-20240530-111344:bac106c87308441daca927add6f9af84d6cde708", "video_id": "P07-20240530-111344", "question": "If I am facing the oven, in what approximate cardinal direction is the location of the washing machine?", "answer_choice_0": "South", "answer_choice_1": "East", "answer_choice_2": "Southeast", "answer_choice_3": "Northeast", "answer_choice_4": "Northwest", "answer_id": 2, "answer": "Southeast", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Counterfactual" ], "objects_of_interest": [ "orange tea towel-a72fcce28360e11f", "blue tea towel-6d52da71783cd927", "baking tray-59a17d88756108e6", "baking tray2-1a1ddbeb806d3148", "mug-14aa3ea51f4a4d24", "kettle-8180514a6b1c5b0f", "notepad-e7e91999a2194805", "pen-9dc9c721047937c3", "mixing bowl-e23c8d0e0e8e12f7", "spoon-6feecfe449e8fab1", "spoon2-70e665ce6b66852e", "fork-cc710e820cdb8033", "scrubbing brush-da7ef6d11ee8bd32" ], "reasoning": "1. I watched the video to see when the user was facing the oven and saw this position at 00:06, and this position would be considered north.\n2. I then watched the video to see the location of the washing machine and saw it at 00:01, where it is to the right of the user's position at the oven .\n3. Additionally, between 00:39-00:44, I observed the user walking from the oven to the washing machine in a diagonal line.\n4. Therefore, if the washing machine is to the right and on a diagonal from the oven , its position can be described as being southeast of the oven .", "referenced_object_masks": [] }, { "key": "P07-20240531-101357:25828045929bd6aeab1876fd13afdfac1f714bdd", "video_id": "P07-20240531-101357", "question": "How many times did I shake out the orange sponge?", "answer_choice_0": "15", "answer_choice_1": "25", "answer_choice_2": "5", "answer_choice_3": "9", "answer_choice_4": "13", "answer_id": 0, "answer": "15", "question_type": [ "Action Recognition", "Object Recognition", "Counting" ], "objects_of_interest": [ "knives-da32887663bf88e7", "sharp serrated knife-a9efa5745af4efcd", "grey bowl-2ea1ead89cf0ca39", "air fryer drawer-548ddf4a997e1060", "sink plug-e07c03e6d44177c3", "butter-91c4ac8e91d9bea9", "small bowl-835acf8e538a6821", "tea towel-6dd93b6ce1627c92", "tea towel2-883b034880fa3497", "plate-34bf570345db0358", "plate2-c933bee1cee741bd", "mug-1e19f7704ab6cba4", "plate3-14884df98ab3f8ba", "mug2-9a911343951c3d13", "mug3-4453aa17a53cc1ec", "mug4-83a9ad880675e89a", "mug5-35ba4b4363e00179", "mug6-39b5b15cbfef0cf0", "mug7-70b35744d90f4507", "mug8-97007affcfefd95d", "bowl-2cb7dba2f96fa45e", "mug9-220305c9b4e89361", "bowl2-33971baa490afe80", "mug10-fb7a3d3418694721", "spoon-9a16cf6df819fd5a", "bowl3-013bcc41399cb30a", "mug11-34df5ee3533346e4", "spoon2-4f28ce4d0f1e47b7", "two spoons-9c0dc3b6c77e973e", "kettle-41a47f1de3fa994e", "coffee_jar-cd88dcffe25626ec", "bottom of blender-d1822f73fd6fe81e", "measuring spoons-320d57fbaaa3167f", "knife-986ce189d3e132fe", "large silver bowl-ed31ef1298167d78", "tupperware container-45d8005821836fba", "digital scale-399321f2e0524e4e", "tupperware container2-789949bf1d328344", "container-1a712faa183d4367", "container2-4ac6ada8964d72bf", "container_lids-6a5b7284b856d29c", "tupperware-e4d21c732b3ebc34", "dishwashing liquid-eadc8c930ba10104", "metal grille-ea85c096a633d062", "dish scrubber-16d0a94a43cf74ff", "drying mat-4de1553966fef787", "food waste bin-442b6f49ebf956b4", "sponge-ba81249bb2e96c35", "coffee machine-3cdf10e488440629", "coffee grinder-68a956d09ac830d9", "empty bags-0d90462e744d77ba", "one empty bag-97dea6983e634121", "basket-1d890532d2ada6c4", "lid of basket-763a8eda1e489494", "basket of coffee equipment-21d228d9e5010d66", "coffee spoon-a87fa1fb694c4d6d", "two tubes of green food dye-74dbd8d387703026", "drip coffee apparatus-3f1c1a0daf74e5f4", "drip coffee apparatus cover-9c8f93197f72bffa", "drip coffee apparatus cover2-480277bb89250b0d", "drying rack-ca1bb49389495bc7", "cnatiner-0f7de696eae28134", "wooden scrubbing brush-0d5aacab9105afa3", "white draining tray-ae677913a46135fe", "plant-978f4417d8e69d18", "tub of bicarbonate of soda-f157afa17522305d", "tea towel3-50a0f989f9277861", "scourer-c47ccf9d69e94653", "bottle brush-52540c8757109d6a", "dishcloth-d71834f183f86a63", "spray bottle-dd88497f62e48279", "tupperware2-bc3f303b94b6b0a6", "container3-a3ad20298cb4d997", "roll of kitchen roll-a058bf80e952cdb7", "smart home meter-205f0e7ad81c70c6", "spoon rest-98cd8fe465118fc0", "bottle of sesame oil-16d6f81bfaea0e93", "cup-c8ac729624c5072c", "stool-aaed0831c8b1f6f3", "yellow spray bottle-c3c4dba045884c98", "bottle of disinfectant-3293cc69a91aeacb", "plastic bag-fb9b5cbfd0806236", "coffee apparatus-43c510bec90c4af8", "espresso basket-52521f3b52d162c5", "lid of container-f01b54e9c53e129e", "tea towel4-ca725f448fa50817", "cup2-9b6993f5df813dff", "spoon3-24a837de3742d8de", "cap-e5d6cc584cdbfe4f" ], "reasoning": "1. I watched the video to observe when the user shook the orange sponge (sponge-ba81249bb2e96c35) out and saw this happen twice at 06:03 and 06:06.\n2. I then observed the user shaking out the sponge (sponge-ba81249bb2e96c35) twice at 06:31.\n3. The user then shakes out the sponge (sponge-ba81249bb2e96c35) 2 times at 10:22.\n4. I then observed the user shaking the sponge (sponge-ba81249bb2e96c35) 3 times between 13:25-13:28.\n5. I then the observed the user shake the sponge (sponge-ba81249bb2e96c35) while they were cleaning it at 18:16, 18:28, 18:33, twice at 18:38, and once more at 18:41 (a total of 6 times) before putting it away to dry.\n6. Therefore, the user shook the orange sponge (sponge-ba81249bb2e96c35) a total of 15 times.", "referenced_object_masks": [ "sponge-ba81249bb2e96c35" ] }, { "key": "P07-20240531-101357:4a7523682382ab823a2510cb6b62711b4739eeb7", "video_id": "P07-20240531-101357", "question": "What is the mathematical difference between the number of items I put in the dishwasher's top shelf and its bottom shelf?", "answer_choice_0": "4", "answer_choice_1": "8", "answer_choice_2": "2", "answer_choice_3": "6", "answer_choice_4": "10", "answer_id": 3, "answer": "6", "question_type": [ "Situational Awareness", "Counting", "Action Recognition", "Egocentric & Spatial Perception", "Numerical Reasoning", "State Changes" ], "objects_of_interest": [ "knives-da32887663bf88e7", "sharp serrated knife-a9efa5745af4efcd", "grey bowl-2ea1ead89cf0ca39", "air fryer drawer-548ddf4a997e1060", "sink plug-e07c03e6d44177c3", "butter-91c4ac8e91d9bea9", "small bowl-835acf8e538a6821", "tea towel-6dd93b6ce1627c92", "tea towel2-883b034880fa3497", "plate-34bf570345db0358", "plate2-c933bee1cee741bd", "mug-1e19f7704ab6cba4", "plate3-14884df98ab3f8ba", "mug2-9a911343951c3d13", "mug3-4453aa17a53cc1ec", "mug4-83a9ad880675e89a", "mug5-35ba4b4363e00179", "mug6-39b5b15cbfef0cf0", "mug7-70b35744d90f4507", "mug8-97007affcfefd95d", "bowl-2cb7dba2f96fa45e", "mug9-220305c9b4e89361", "bowl2-33971baa490afe80", "mug10-fb7a3d3418694721", "spoon-9a16cf6df819fd5a", "bowl3-013bcc41399cb30a", "mug11-34df5ee3533346e4", "spoon2-4f28ce4d0f1e47b7", "two spoons-9c0dc3b6c77e973e", "kettle-41a47f1de3fa994e", "coffee_jar-cd88dcffe25626ec", "bottom of blender-d1822f73fd6fe81e", "measuring spoons-320d57fbaaa3167f", "knife-986ce189d3e132fe", "large silver bowl-ed31ef1298167d78", "tupperware container-45d8005821836fba", "digital scale-399321f2e0524e4e", "tupperware container2-789949bf1d328344", "container-1a712faa183d4367", "container2-4ac6ada8964d72bf", "container_lids-6a5b7284b856d29c", "tupperware-e4d21c732b3ebc34", "dishwashing liquid-eadc8c930ba10104", "metal grille-ea85c096a633d062", "dish scrubber-16d0a94a43cf74ff", "drying mat-4de1553966fef787", "food waste bin-442b6f49ebf956b4", "sponge-ba81249bb2e96c35", "coffee machine-3cdf10e488440629", "coffee grinder-68a956d09ac830d9", "empty bags-0d90462e744d77ba", "one empty bag-97dea6983e634121", "basket-1d890532d2ada6c4", "lid of basket-763a8eda1e489494", "basket of coffee equipment-21d228d9e5010d66", "coffee spoon-a87fa1fb694c4d6d", "two tubes of green food dye-74dbd8d387703026", "drip coffee apparatus-3f1c1a0daf74e5f4", "drip coffee apparatus cover-9c8f93197f72bffa", "drip coffee apparatus cover2-480277bb89250b0d", "drying rack-ca1bb49389495bc7", "cnatiner-0f7de696eae28134", "wooden scrubbing brush-0d5aacab9105afa3", "white draining tray-ae677913a46135fe", "plant-978f4417d8e69d18", "tub of bicarbonate of soda-f157afa17522305d", "tea towel3-50a0f989f9277861", "scourer-c47ccf9d69e94653", "bottle brush-52540c8757109d6a", "dishcloth-d71834f183f86a63", "spray bottle-dd88497f62e48279", "tupperware2-bc3f303b94b6b0a6", "container3-a3ad20298cb4d997", "roll of kitchen roll-a058bf80e952cdb7", "smart home meter-205f0e7ad81c70c6", "spoon rest-98cd8fe465118fc0", "bottle of sesame oil-16d6f81bfaea0e93", "cup-c8ac729624c5072c", "stool-aaed0831c8b1f6f3", "yellow spray bottle-c3c4dba045884c98", "bottle of disinfectant-3293cc69a91aeacb", "plastic bag-fb9b5cbfd0806236", "coffee apparatus-43c510bec90c4af8", "espresso basket-52521f3b52d162c5", "lid of container-f01b54e9c53e129e", "tea towel4-ca725f448fa50817", "cup2-9b6993f5df813dff", "spoon3-24a837de3742d8de", "cap-e5d6cc584cdbfe4f" ], "reasoning": "1. I watched the video and found the dishwasher at 01:12.\n2. I counted each item the user put in the dishwasher's bottom shelf, calculating a total of 10 from 01:22-01:28 and 01:51-03:11, which were a plate (plate3-14884df98ab3f8ba), 2 mugs (mug7-70b35744d90f4507, mug8-97007affcfefd95d), a bowl (bowl2-33971baa490afe80), another 2 mugs (mug9-220305c9b4e89361, mug10-fb7a3d3418694721), 2 spoons (spoon-9a16cf6df819fd5a, spoon2-4f28ce4d0f1e47b7), another mug (mug11-34df5ee3533346e4), and another bowl (bowl3-013bcc41399cb30a).\n3. I counted each item the user put in the dishwasher's top shelf, calculating a total of 4 from 01:33-01:49 and 03:13-03:15, which were 3 mugs (mug-1e19f7704ab6cba4, mug4-83a9ad880675e89a, mug6-39b5b15cbfef0cf0) and a knife (sharp serrated knife-a9efa5745af4efcd).\n4. I calculated 10-4=6.\n5. Therefore, the mathematical difference between the number of items the user put in the dishwasher's top shelf and its bottom shelf is 6.", "referenced_object_masks": [ "spoon-9a16cf6df819fd5a", "spoon2-4f28ce4d0f1e47b7", "plate3-14884df98ab3f8ba", "bowl2-33971baa490afe80", "mug9-220305c9b4e89361", "mug10-fb7a3d3418694721", "bowl3-013bcc41399cb30a", "sharp serrated knife-a9efa5745af4efcd", "mug7-70b35744d90f4507", "mug8-97007affcfefd95d", "mug11-34df5ee3533346e4", "mug-1e19f7704ab6cba4", "mug4-83a9ad880675e89a", "mug6-39b5b15cbfef0cf0" ] }, { "key": "P07-20240531-101357:50d231e009f63d4ebbc6830d5f6e6a66a7e69553", "video_id": "P07-20240531-101357", "question": "If I halved the number of items I put in the refrigerator, how many would I have put in it?", "answer_choice_0": "2", "answer_choice_1": "4", "answer_choice_2": "6", "answer_choice_3": "8", "answer_choice_4": "1", "answer_id": 4, "answer": "1", "question_type": [ "Situational Awareness", "Counting", "Action Recognition", "Counterfactual", "Egocentric & Spatial Perception", "Numerical Reasoning", "State Changes" ], "objects_of_interest": [ "knives-da32887663bf88e7", "sharp serrated knife-a9efa5745af4efcd", "grey bowl-2ea1ead89cf0ca39", "air fryer drawer-548ddf4a997e1060", "sink plug-e07c03e6d44177c3", "butter-91c4ac8e91d9bea9", "small bowl-835acf8e538a6821", "tea towel-6dd93b6ce1627c92", "tea towel2-883b034880fa3497", "plate-34bf570345db0358", "plate2-c933bee1cee741bd", "mug-1e19f7704ab6cba4", "plate3-14884df98ab3f8ba", "mug2-9a911343951c3d13", "mug3-4453aa17a53cc1ec", "mug4-83a9ad880675e89a", "mug5-35ba4b4363e00179", "mug6-39b5b15cbfef0cf0", "mug7-70b35744d90f4507", "mug8-97007affcfefd95d", "bowl-2cb7dba2f96fa45e", "mug9-220305c9b4e89361", "bowl2-33971baa490afe80", "mug10-fb7a3d3418694721", "spoon-9a16cf6df819fd5a", "bowl3-013bcc41399cb30a", "mug11-34df5ee3533346e4", "spoon2-4f28ce4d0f1e47b7", "two spoons-9c0dc3b6c77e973e", "kettle-41a47f1de3fa994e", "coffee_jar-cd88dcffe25626ec", "bottom of blender-d1822f73fd6fe81e", "measuring spoons-320d57fbaaa3167f", "knife-986ce189d3e132fe", "large silver bowl-ed31ef1298167d78", "tupperware container-45d8005821836fba", "digital scale-399321f2e0524e4e", "tupperware container2-789949bf1d328344", "container-1a712faa183d4367", "container2-4ac6ada8964d72bf", "container_lids-6a5b7284b856d29c", "tupperware-e4d21c732b3ebc34", "dishwashing liquid-eadc8c930ba10104", "metal grille-ea85c096a633d062", "dish scrubber-16d0a94a43cf74ff", "drying mat-4de1553966fef787", "food waste bin-442b6f49ebf956b4", "sponge-ba81249bb2e96c35", "coffee machine-3cdf10e488440629", "coffee grinder-68a956d09ac830d9", "empty bags-0d90462e744d77ba", "one empty bag-97dea6983e634121", "basket-1d890532d2ada6c4", "lid of basket-763a8eda1e489494", "basket of coffee equipment-21d228d9e5010d66", "coffee spoon-a87fa1fb694c4d6d", "two tubes of green food dye-74dbd8d387703026", "drip coffee apparatus-3f1c1a0daf74e5f4", "drip coffee apparatus cover-9c8f93197f72bffa", "drip coffee apparatus cover2-480277bb89250b0d", "drying rack-ca1bb49389495bc7", "cnatiner-0f7de696eae28134", "wooden scrubbing brush-0d5aacab9105afa3", "white draining tray-ae677913a46135fe", "plant-978f4417d8e69d18", "tub of bicarbonate of soda-f157afa17522305d", "tea towel3-50a0f989f9277861", "scourer-c47ccf9d69e94653", "bottle brush-52540c8757109d6a", "dishcloth-d71834f183f86a63", "spray bottle-dd88497f62e48279", "tupperware2-bc3f303b94b6b0a6", "container3-a3ad20298cb4d997", "roll of kitchen roll-a058bf80e952cdb7", "smart home meter-205f0e7ad81c70c6", "spoon rest-98cd8fe465118fc0", "bottle of sesame oil-16d6f81bfaea0e93", "cup-c8ac729624c5072c", "stool-aaed0831c8b1f6f3", "yellow spray bottle-c3c4dba045884c98", "bottle of disinfectant-3293cc69a91aeacb", "plastic bag-fb9b5cbfd0806236", "coffee apparatus-43c510bec90c4af8", "espresso basket-52521f3b52d162c5", "lid of container-f01b54e9c53e129e", "tea towel4-ca725f448fa50817", "cup2-9b6993f5df813dff", "spoon3-24a837de3742d8de", "cap-e5d6cc584cdbfe4f" ], "reasoning": "1. I watched the video and found the refrigerator at 00:44.\n2. I counted each item the user put in the refrigerator, calculating a total of 2 at 00:47-00:49, which were a pack of butter (butter-91c4ac8e91d9bea9) and a bowl (small bowl-835acf8e538a6821).\n3. I calculated 2/2=1.\n4. Therefore, if the user halved the number of items they put in the refrigerator, they would have put 1 item in it.", "referenced_object_masks": [ "small bowl-835acf8e538a6821", "butter-91c4ac8e91d9bea9" ] }, { "key": "P07-20240531-101357:78fff7d68e9284c98b74f6c02adf2d67b68ac079", "video_id": "P07-20240531-101357", "question": "What directional pattern did I make when I was pouring the remaining boiling water in the two sink basins?", "answer_choice_0": "right, left, right, left", "answer_choice_1": "left, right, left, right", "answer_choice_2": "Left to right", "answer_choice_3": "Up and down", "answer_choice_4": "Up, down, side to side", "answer_id": 1, "answer": "left, right, left, right", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "knives-da32887663bf88e7", "sharp serrated knife-a9efa5745af4efcd", "grey bowl-2ea1ead89cf0ca39", "air fryer drawer-548ddf4a997e1060", "sink plug-e07c03e6d44177c3", "butter-91c4ac8e91d9bea9", "small bowl-835acf8e538a6821", "tea towel-6dd93b6ce1627c92", "tea towel2-883b034880fa3497", "plate-34bf570345db0358", "plate2-c933bee1cee741bd", "mug-1e19f7704ab6cba4", "plate3-14884df98ab3f8ba", "mug2-9a911343951c3d13", "mug3-4453aa17a53cc1ec", "mug4-83a9ad880675e89a", "mug5-35ba4b4363e00179", "mug6-39b5b15cbfef0cf0", "mug7-70b35744d90f4507", "mug8-97007affcfefd95d", "bowl-2cb7dba2f96fa45e", "mug9-220305c9b4e89361", "bowl2-33971baa490afe80", "mug10-fb7a3d3418694721", "spoon-9a16cf6df819fd5a", "bowl3-013bcc41399cb30a", "mug11-34df5ee3533346e4", "spoon2-4f28ce4d0f1e47b7", "two spoons-9c0dc3b6c77e973e", "kettle-41a47f1de3fa994e", "coffee_jar-cd88dcffe25626ec", "bottom of blender-d1822f73fd6fe81e", "measuring spoons-320d57fbaaa3167f", "knife-986ce189d3e132fe", "large silver bowl-ed31ef1298167d78", "tupperware container-45d8005821836fba", "digital scale-399321f2e0524e4e", "tupperware container2-789949bf1d328344", "container-1a712faa183d4367", "container2-4ac6ada8964d72bf", "container_lids-6a5b7284b856d29c", "tupperware-e4d21c732b3ebc34", "dishwashing liquid-eadc8c930ba10104", "metal grille-ea85c096a633d062", "dish scrubber-16d0a94a43cf74ff", "drying mat-4de1553966fef787", "food waste bin-442b6f49ebf956b4", "sponge-ba81249bb2e96c35", "coffee machine-3cdf10e488440629", "coffee grinder-68a956d09ac830d9", "empty bags-0d90462e744d77ba", "one empty bag-97dea6983e634121", "basket-1d890532d2ada6c4", "lid of basket-763a8eda1e489494", "basket of coffee equipment-21d228d9e5010d66", "coffee spoon-a87fa1fb694c4d6d", "two tubes of green food dye-74dbd8d387703026", "drip coffee apparatus-3f1c1a0daf74e5f4", "drip coffee apparatus cover-9c8f93197f72bffa", "drip coffee apparatus cover2-480277bb89250b0d", "drying rack-ca1bb49389495bc7", "cnatiner-0f7de696eae28134", "wooden scrubbing brush-0d5aacab9105afa3", "white draining tray-ae677913a46135fe", "plant-978f4417d8e69d18", "tub of bicarbonate of soda-f157afa17522305d", "tea towel3-50a0f989f9277861", "scourer-c47ccf9d69e94653", "bottle brush-52540c8757109d6a", "dishcloth-d71834f183f86a63", "spray bottle-dd88497f62e48279", "tupperware2-bc3f303b94b6b0a6", "container3-a3ad20298cb4d997", "roll of kitchen roll-a058bf80e952cdb7", "smart home meter-205f0e7ad81c70c6", "spoon rest-98cd8fe465118fc0", "bottle of sesame oil-16d6f81bfaea0e93", "cup-c8ac729624c5072c", "stool-aaed0831c8b1f6f3", "yellow spray bottle-c3c4dba045884c98", "bottle of disinfectant-3293cc69a91aeacb", "plastic bag-fb9b5cbfd0806236", "coffee apparatus-43c510bec90c4af8", "espresso basket-52521f3b52d162c5", "lid of container-f01b54e9c53e129e", "tea towel4-ca725f448fa50817", "cup2-9b6993f5df813dff", "spoon3-24a837de3742d8de", "cap-e5d6cc584cdbfe4f" ], "reasoning": "1. I watched the video to find when the user pours boiling water into the two sink basins and saw this happen between 42:22-42:41.\n2. The user starts out by pouring the boiling water into the left sink basin at 42:22.\n3. I then observed the user pour the boiling water into the right sink basin at 42:27.\n4. I then observed the user once again pouring the boiling water into the left sink basin at 42:36.\n5. The user then pours the remaining water into the right sink basin at 42:40.\n6. Therefore, the directional pattern the user made when they poured out the boiling water was left, right, left, right.", "referenced_object_masks": [] }, { "key": "P07-20240531-101357:8fe62af513322e82b66f1382739ce8a8af07d475", "video_id": "P07-20240531-101357", "question": "How many items did I move to a new location in order to make space for the tupperware bowl and lid to fit in the drawer?", "answer_choice_0": "3", "answer_choice_1": "1", "answer_choice_2": "7", "answer_choice_3": "5", "answer_choice_4": "10", "answer_id": 0, "answer": "3", "question_type": [ "Action Recognition", "Object Recognition", "Counting", "State Changes" ], "objects_of_interest": [ "knives-da32887663bf88e7", "sharp serrated knife-a9efa5745af4efcd", "grey bowl-2ea1ead89cf0ca39", "air fryer drawer-548ddf4a997e1060", "sink plug-e07c03e6d44177c3", "butter-91c4ac8e91d9bea9", "small bowl-835acf8e538a6821", "tea towel-6dd93b6ce1627c92", "tea towel2-883b034880fa3497", "plate-34bf570345db0358", "plate2-c933bee1cee741bd", "mug-1e19f7704ab6cba4", "plate3-14884df98ab3f8ba", "mug2-9a911343951c3d13", "mug3-4453aa17a53cc1ec", "mug4-83a9ad880675e89a", "mug5-35ba4b4363e00179", "mug6-39b5b15cbfef0cf0", "mug7-70b35744d90f4507", "mug8-97007affcfefd95d", "bowl-2cb7dba2f96fa45e", "mug9-220305c9b4e89361", "bowl2-33971baa490afe80", "mug10-fb7a3d3418694721", "spoon-9a16cf6df819fd5a", "bowl3-013bcc41399cb30a", "mug11-34df5ee3533346e4", "spoon2-4f28ce4d0f1e47b7", "two spoons-9c0dc3b6c77e973e", "kettle-41a47f1de3fa994e", "coffee_jar-cd88dcffe25626ec", "bottom of blender-d1822f73fd6fe81e", "measuring spoons-320d57fbaaa3167f", "knife-986ce189d3e132fe", "large silver bowl-ed31ef1298167d78", "tupperware container-45d8005821836fba", "digital scale-399321f2e0524e4e", "tupperware container2-789949bf1d328344", "container-1a712faa183d4367", "container2-4ac6ada8964d72bf", "container_lids-6a5b7284b856d29c", "tupperware-e4d21c732b3ebc34", "dishwashing liquid-eadc8c930ba10104", "metal grille-ea85c096a633d062", "dish scrubber-16d0a94a43cf74ff", "drying mat-4de1553966fef787", "food waste bin-442b6f49ebf956b4", "sponge-ba81249bb2e96c35", "coffee machine-3cdf10e488440629", "coffee grinder-68a956d09ac830d9", "empty bags-0d90462e744d77ba", "one empty bag-97dea6983e634121", "basket-1d890532d2ada6c4", "lid of basket-763a8eda1e489494", "basket of coffee equipment-21d228d9e5010d66", "coffee spoon-a87fa1fb694c4d6d", "two tubes of green food dye-74dbd8d387703026", "drip coffee apparatus-3f1c1a0daf74e5f4", "drip coffee apparatus cover-9c8f93197f72bffa", "drip coffee apparatus cover2-480277bb89250b0d", "drying rack-ca1bb49389495bc7", "cnatiner-0f7de696eae28134", "wooden scrubbing brush-0d5aacab9105afa3", "white draining tray-ae677913a46135fe", "plant-978f4417d8e69d18", "tub of bicarbonate of soda-f157afa17522305d", "tea towel3-50a0f989f9277861", "scourer-c47ccf9d69e94653", "bottle brush-52540c8757109d6a", "dishcloth-d71834f183f86a63", "spray bottle-dd88497f62e48279", "tupperware2-bc3f303b94b6b0a6", "container3-a3ad20298cb4d997", "roll of kitchen roll-a058bf80e952cdb7", "smart home meter-205f0e7ad81c70c6", "spoon rest-98cd8fe465118fc0", "bottle of sesame oil-16d6f81bfaea0e93", "cup-c8ac729624c5072c", "stool-aaed0831c8b1f6f3", "yellow spray bottle-c3c4dba045884c98", "bottle of disinfectant-3293cc69a91aeacb", "plastic bag-fb9b5cbfd0806236", "coffee apparatus-43c510bec90c4af8", "espresso basket-52521f3b52d162c5", "lid of container-f01b54e9c53e129e", "tea towel4-ca725f448fa50817", "cup2-9b6993f5df813dff", "spoon3-24a837de3742d8de", "cap-e5d6cc584cdbfe4f" ], "reasoning": "1. I watched the video to find when the user was trying to make a tupperware bowl (tupperware container-45d8005821836fba) and lid fit into a drawer and saw this happen between 04:22-04:42.\n2. To make room for the tupperware bowl (tupperware container-45d8005821836fba) and lid , the user touches many objects but only moves a few to a new location, starting with a tupperware container (tupperware container2-789949bf1d328344) at 04:24. I noticed that the user placed this container (tupperware container2-789949bf1d328344) inside another container at 04:26. This shift created additional space in the drawer, and I counted this container (tupperware container2-789949bf1d328344) as the first object the user moved to a new location. \n3. At 04:33-04:36, I saw the user move a plastic lid (container_lids-6a5b7284b856d29c) to a new location which also created a shift in space in the drawer. I counted this as the second item moved to a new location. \n4. At 04:38, I saw the user move the plastic cup (tupperware-e4d21c732b3ebc34) from the back of the drawer to left of the first container (tupperware container2-789949bf1d328344), which also created additional space in the drawer. I counted this as the third object moved to a new location. \n5. Finally, I saw the user find a free spot to put the tupperware bowl (tupperware container-45d8005821836fba) and lid at 04:40, meaning that they moved a total of 3 objects (tupperware container2-789949bf1d328344, container_lids-6a5b7284b856d29c, tupperware-e4d21c732b3ebc34) to a new lcoation before finding a spot for the tupperware (tupperware container-45d8005821836fba) and lid in the drawer.", "referenced_object_masks": [ "tupperware-e4d21c732b3ebc34", "tupperware container2-789949bf1d328344", "container_lids-6a5b7284b856d29c", "tupperware container-45d8005821836fba" ] }, { "key": "P07-20240531-101357:9398af4130a5304ca31a9a6ef056485ddfdf5356", "video_id": "P07-20240531-101357", "question": "What item do I pick up directly before and after I the second time I pour liquid from an object into the sink?", "answer_choice_0": "A black air fryer drawer", "answer_choice_1": "A sink plug", "answer_choice_2": "A gray bowl", "answer_choice_3": "An orange coffee mug", "answer_choice_4": "A white coffee mug", "answer_id": 1, "answer": "A sink plug", "question_type": [ "Situational Awareness", "Counting", "Action Recognition", "Egocentric & Spatial Perception", "Object Recognition", "State Changes" ], "objects_of_interest": [ "knives-da32887663bf88e7", "sharp serrated knife-a9efa5745af4efcd", "grey bowl-2ea1ead89cf0ca39", "air fryer drawer-548ddf4a997e1060", "sink plug-e07c03e6d44177c3", "butter-91c4ac8e91d9bea9", "small bowl-835acf8e538a6821", "tea towel-6dd93b6ce1627c92", "tea towel2-883b034880fa3497", "plate-34bf570345db0358", "plate2-c933bee1cee741bd", "mug-1e19f7704ab6cba4", "plate3-14884df98ab3f8ba", "mug2-9a911343951c3d13", "mug3-4453aa17a53cc1ec", "mug4-83a9ad880675e89a", "mug5-35ba4b4363e00179", "mug6-39b5b15cbfef0cf0", "mug7-70b35744d90f4507", "mug8-97007affcfefd95d", "bowl-2cb7dba2f96fa45e", "mug9-220305c9b4e89361", "bowl2-33971baa490afe80", "mug10-fb7a3d3418694721", "spoon-9a16cf6df819fd5a", "bowl3-013bcc41399cb30a", "mug11-34df5ee3533346e4", "spoon2-4f28ce4d0f1e47b7", "two spoons-9c0dc3b6c77e973e", "kettle-41a47f1de3fa994e", "coffee_jar-cd88dcffe25626ec", "bottom of blender-d1822f73fd6fe81e", "measuring spoons-320d57fbaaa3167f", "knife-986ce189d3e132fe", "large silver bowl-ed31ef1298167d78", "tupperware container-45d8005821836fba", "digital scale-399321f2e0524e4e", "tupperware container2-789949bf1d328344", "container-1a712faa183d4367", "container2-4ac6ada8964d72bf", "container_lids-6a5b7284b856d29c", "tupperware-e4d21c732b3ebc34", "dishwashing liquid-eadc8c930ba10104", "metal grille-ea85c096a633d062", "dish scrubber-16d0a94a43cf74ff", "drying mat-4de1553966fef787", "food waste bin-442b6f49ebf956b4", "sponge-ba81249bb2e96c35", "coffee machine-3cdf10e488440629", "coffee grinder-68a956d09ac830d9", "empty bags-0d90462e744d77ba", "one empty bag-97dea6983e634121", "basket-1d890532d2ada6c4", "lid of basket-763a8eda1e489494", "basket of coffee equipment-21d228d9e5010d66", "coffee spoon-a87fa1fb694c4d6d", "two tubes of green food dye-74dbd8d387703026", "drip coffee apparatus-3f1c1a0daf74e5f4", "drip coffee apparatus cover-9c8f93197f72bffa", "drip coffee apparatus cover2-480277bb89250b0d", "drying rack-ca1bb49389495bc7", "cnatiner-0f7de696eae28134", "wooden scrubbing brush-0d5aacab9105afa3", "white draining tray-ae677913a46135fe", "plant-978f4417d8e69d18", "tub of bicarbonate of soda-f157afa17522305d", "tea towel3-50a0f989f9277861", "scourer-c47ccf9d69e94653", "bottle brush-52540c8757109d6a", "dishcloth-d71834f183f86a63", "spray bottle-dd88497f62e48279", "tupperware2-bc3f303b94b6b0a6", "container3-a3ad20298cb4d997", "roll of kitchen roll-a058bf80e952cdb7", "smart home meter-205f0e7ad81c70c6", "spoon rest-98cd8fe465118fc0", "bottle of sesame oil-16d6f81bfaea0e93", "cup-c8ac729624c5072c", "stool-aaed0831c8b1f6f3", "yellow spray bottle-c3c4dba045884c98", "bottle of disinfectant-3293cc69a91aeacb", "plastic bag-fb9b5cbfd0806236", "coffee apparatus-43c510bec90c4af8", "espresso basket-52521f3b52d162c5", "lid of container-f01b54e9c53e129e", "tea towel4-ca725f448fa50817", "cup2-9b6993f5df813dff", "spoon3-24a837de3742d8de", "cap-e5d6cc584cdbfe4f" ], "reasoning": "1. I watched the video and found the sink at 00:05.\n2. I counted each item the user poured in the sink.\n3. At 00:10, I saw the user pour the first item in the sink, which was an air fryer drawer-548ddf4a997e1060 (air fryer drawer-548ddf4a997e1060) full of water.\n4. At 00:17, I saw the user pour the second item in the sink, which was a gray bowl (grey bowl-2ea1ead89cf0ca39) full of yellow liquid.\n5. I looked to see what item the user picks up just before and after the gray bowl (grey bowl-2ea1ead89cf0ca39).\n6. At 00:16, the user picks up a sink plug (sink plug-e07c03e6d44177c3), which they put down at 0:18.\n7. The user picks up the sink plug (sink plug-e07c03e6d44177c3) again at 00:25, making the item picked up before and after the second item that liquid is poured from into the sink the sink plug (sink plug-e07c03e6d44177c3).", "referenced_object_masks": [ "sink plug-e07c03e6d44177c3", "grey bowl-2ea1ead89cf0ca39", "air fryer drawer-548ddf4a997e1060" ] }, { "key": "P07-20240531-101357:a3636a29078903d3b3fc4ae91a90a96a3f335523", "video_id": "P07-20240531-101357", "question": "In how many different locations did I put dishes away that were on the drying rack or the drying mat?", "answer_choice_0": "3", "answer_choice_1": "5", "answer_choice_2": "2", "answer_choice_3": "7", "answer_choice_4": "4", "answer_id": 1, "answer": "5", "question_type": [ "Action Recognition", "Object Recognition", "Counting", "State Changes" ], "objects_of_interest": [ "knives-da32887663bf88e7", "sharp serrated knife-a9efa5745af4efcd", "grey bowl-2ea1ead89cf0ca39", "air fryer drawer-548ddf4a997e1060", "sink plug-e07c03e6d44177c3", "butter-91c4ac8e91d9bea9", "small bowl-835acf8e538a6821", "tea towel-6dd93b6ce1627c92", "tea towel2-883b034880fa3497", "plate-34bf570345db0358", "plate2-c933bee1cee741bd", "mug-1e19f7704ab6cba4", "plate3-14884df98ab3f8ba", "mug2-9a911343951c3d13", "mug3-4453aa17a53cc1ec", "mug4-83a9ad880675e89a", "mug5-35ba4b4363e00179", "mug6-39b5b15cbfef0cf0", "mug7-70b35744d90f4507", "mug8-97007affcfefd95d", "bowl-2cb7dba2f96fa45e", "mug9-220305c9b4e89361", "bowl2-33971baa490afe80", "mug10-fb7a3d3418694721", "spoon-9a16cf6df819fd5a", "bowl3-013bcc41399cb30a", "mug11-34df5ee3533346e4", "spoon2-4f28ce4d0f1e47b7", "two spoons-9c0dc3b6c77e973e", "kettle-41a47f1de3fa994e", "coffee_jar-cd88dcffe25626ec", "bottom of blender-d1822f73fd6fe81e", "measuring spoons-320d57fbaaa3167f", "knife-986ce189d3e132fe", "large silver bowl-ed31ef1298167d78", "tupperware container-45d8005821836fba", "digital scale-399321f2e0524e4e", "tupperware container2-789949bf1d328344", "container-1a712faa183d4367", "container2-4ac6ada8964d72bf", "container_lids-6a5b7284b856d29c", "tupperware-e4d21c732b3ebc34", "dishwashing liquid-eadc8c930ba10104", "metal grille-ea85c096a633d062", "dish scrubber-16d0a94a43cf74ff", "drying mat-4de1553966fef787", "food waste bin-442b6f49ebf956b4", "sponge-ba81249bb2e96c35", "coffee machine-3cdf10e488440629", "coffee grinder-68a956d09ac830d9", "empty bags-0d90462e744d77ba", "one empty bag-97dea6983e634121", "basket-1d890532d2ada6c4", "lid of basket-763a8eda1e489494", "basket of coffee equipment-21d228d9e5010d66", "coffee spoon-a87fa1fb694c4d6d", "two tubes of green food dye-74dbd8d387703026", "drip coffee apparatus-3f1c1a0daf74e5f4", "drip coffee apparatus cover-9c8f93197f72bffa", "drip coffee apparatus cover2-480277bb89250b0d", "drying rack-ca1bb49389495bc7", "cnatiner-0f7de696eae28134", "wooden scrubbing brush-0d5aacab9105afa3", "white draining tray-ae677913a46135fe", "plant-978f4417d8e69d18", "tub of bicarbonate of soda-f157afa17522305d", "tea towel3-50a0f989f9277861", "scourer-c47ccf9d69e94653", "bottle brush-52540c8757109d6a", "dishcloth-d71834f183f86a63", "spray bottle-dd88497f62e48279", "tupperware2-bc3f303b94b6b0a6", "container3-a3ad20298cb4d997", "roll of kitchen roll-a058bf80e952cdb7", "smart home meter-205f0e7ad81c70c6", "spoon rest-98cd8fe465118fc0", "bottle of sesame oil-16d6f81bfaea0e93", "cup-c8ac729624c5072c", "stool-aaed0831c8b1f6f3", "yellow spray bottle-c3c4dba045884c98", "bottle of disinfectant-3293cc69a91aeacb", "plastic bag-fb9b5cbfd0806236", "coffee apparatus-43c510bec90c4af8", "espresso basket-52521f3b52d162c5", "lid of container-f01b54e9c53e129e", "tea towel4-ca725f448fa50817", "cup2-9b6993f5df813dff", "spoon3-24a837de3742d8de", "cap-e5d6cc584cdbfe4f" ], "reasoning": "1. I watched the video to see when the user put dishes away that had been on the drying rack or drying mat and saw this happen from 03:39-04:42.\n2. I first observed the user putting a knife (knife-986ce189d3e132fe) into a knife block at 03:46. \n3. I then observed the user putting 2 measuring spoons (measuring spoons-320d57fbaaa3167f) into a drawer to the left of the sink at 03:49.\n4. I then observed the user putting away the bottom of a blender (bottom of blender-d1822f73fd6fe81e) into a drawer to the left of the stove at 03:52.\n5. I then observed the user putting away a large silver bowl (large silver bowl-ed31ef1298167d78) in the top drawer on the right side of the island at 04:21.\n6. Finally, I observed the user putting away a tupperware bowl (tupperware container-45d8005821836fba) and lid in the bottom drawer on the right side of the island at 04:40.\n7. Therefore, the user put away their dishes in 5 different locations.", "referenced_object_masks": [ "knife-986ce189d3e132fe", "measuring spoons-320d57fbaaa3167f", "large silver bowl-ed31ef1298167d78", "tupperware container-45d8005821836fba", "bottom of blender-d1822f73fd6fe81e" ] }, { "key": "P07-20240531-101357:a9a6bc0c7164be8effe8ca18660795c267c66ab9", "video_id": "P07-20240531-101357", "question": "Where did I move the yellow and white tea towel to?", "answer_choice_0": "Into the dishwasher", "answer_choice_1": "Onto the hanger", "answer_choice_2": "Around the sink", "answer_choice_3": "Into a white bag", "answer_choice_4": "Onto a table", "answer_id": 3, "answer": "Into a white bag", "question_type": [ "Object Recognition", "Action Recognition", "State Changes", "Situational Awareness", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "knives-da32887663bf88e7", "sharp serrated knife-a9efa5745af4efcd", "grey bowl-2ea1ead89cf0ca39", "air fryer drawer-548ddf4a997e1060", "sink plug-e07c03e6d44177c3", "butter-91c4ac8e91d9bea9", "small bowl-835acf8e538a6821", "tea towel-6dd93b6ce1627c92", "tea towel2-883b034880fa3497", "plate-34bf570345db0358", "plate2-c933bee1cee741bd", "mug-1e19f7704ab6cba4", "plate3-14884df98ab3f8ba", "mug2-9a911343951c3d13", "mug3-4453aa17a53cc1ec", "mug4-83a9ad880675e89a", "mug5-35ba4b4363e00179", "mug6-39b5b15cbfef0cf0", "mug7-70b35744d90f4507", "mug8-97007affcfefd95d", "bowl-2cb7dba2f96fa45e", "mug9-220305c9b4e89361", "bowl2-33971baa490afe80", "mug10-fb7a3d3418694721", "spoon-9a16cf6df819fd5a", "bowl3-013bcc41399cb30a", "mug11-34df5ee3533346e4", "spoon2-4f28ce4d0f1e47b7", "two spoons-9c0dc3b6c77e973e", "kettle-41a47f1de3fa994e", "coffee_jar-cd88dcffe25626ec", "bottom of blender-d1822f73fd6fe81e", "measuring spoons-320d57fbaaa3167f", "knife-986ce189d3e132fe", "large silver bowl-ed31ef1298167d78", "tupperware container-45d8005821836fba", "digital scale-399321f2e0524e4e", "tupperware container2-789949bf1d328344", "container-1a712faa183d4367", "container2-4ac6ada8964d72bf", "container_lids-6a5b7284b856d29c", "tupperware-e4d21c732b3ebc34", "dishwashing liquid-eadc8c930ba10104", "metal grille-ea85c096a633d062", "dish scrubber-16d0a94a43cf74ff", "drying mat-4de1553966fef787", "food waste bin-442b6f49ebf956b4", "sponge-ba81249bb2e96c35", "coffee machine-3cdf10e488440629", "coffee grinder-68a956d09ac830d9", "empty bags-0d90462e744d77ba", "one empty bag-97dea6983e634121", "basket-1d890532d2ada6c4", "lid of basket-763a8eda1e489494", "basket of coffee equipment-21d228d9e5010d66", "coffee spoon-a87fa1fb694c4d6d", "two tubes of green food dye-74dbd8d387703026", "drip coffee apparatus-3f1c1a0daf74e5f4", "drip coffee apparatus cover-9c8f93197f72bffa", "drip coffee apparatus cover2-480277bb89250b0d", "drying rack-ca1bb49389495bc7", "cnatiner-0f7de696eae28134", "wooden scrubbing brush-0d5aacab9105afa3", "white draining tray-ae677913a46135fe", "plant-978f4417d8e69d18", "tub of bicarbonate of soda-f157afa17522305d", "tea towel3-50a0f989f9277861", "scourer-c47ccf9d69e94653", "bottle brush-52540c8757109d6a", "dishcloth-d71834f183f86a63", "spray bottle-dd88497f62e48279", "tupperware2-bc3f303b94b6b0a6", "container3-a3ad20298cb4d997", "roll of kitchen roll-a058bf80e952cdb7", "smart home meter-205f0e7ad81c70c6", "spoon rest-98cd8fe465118fc0", "bottle of sesame oil-16d6f81bfaea0e93", "cup-c8ac729624c5072c", "stool-aaed0831c8b1f6f3", "yellow spray bottle-c3c4dba045884c98", "bottle of disinfectant-3293cc69a91aeacb", "plastic bag-fb9b5cbfd0806236", "coffee apparatus-43c510bec90c4af8", "espresso basket-52521f3b52d162c5", "lid of container-f01b54e9c53e129e", "tea towel4-ca725f448fa50817", "cup2-9b6993f5df813dff", "spoon3-24a837de3742d8de", "cap-e5d6cc584cdbfe4f" ], "reasoning": "1. I watched the video and found the yellow and white tea towel (tea towel-6dd93b6ce1627c92) at 00:54.\n2. At 00:55, I saw the user move the yellow and white tea towel (tea towel-6dd93b6ce1627c92) from a hanger.\n3. At 01:00, I saw the user toss the yellow and white tea towel (tea towel-6dd93b6ce1627c92) into a white bag.\n4. Therefore, the user moved the yellow and white tea towel (tea towel-6dd93b6ce1627c92) into a white bag.", "referenced_object_masks": [ "tea towel-6dd93b6ce1627c92" ] }, { "key": "P07-20240531-101357:d5e3e9ffc6fa4853da9ecd75ce74f980218d1162", "video_id": "P07-20240531-101357", "question": "How many different layouts did I try before I made the black cups and lids fit in the small basket?", "answer_choice_0": "3", "answer_choice_1": "2", "answer_choice_2": "1", "answer_choice_3": "5", "answer_choice_4": "4", "answer_id": 1, "answer": "2", "question_type": [ "Action Recognition", "Object Recognition", "Counting", "State Changes" ], "objects_of_interest": [ "knives-da32887663bf88e7", "sharp serrated knife-a9efa5745af4efcd", "grey bowl-2ea1ead89cf0ca39", "air fryer drawer-548ddf4a997e1060", "sink plug-e07c03e6d44177c3", "butter-91c4ac8e91d9bea9", "small bowl-835acf8e538a6821", "tea towel-6dd93b6ce1627c92", "tea towel2-883b034880fa3497", "plate-34bf570345db0358", "plate2-c933bee1cee741bd", "mug-1e19f7704ab6cba4", "plate3-14884df98ab3f8ba", "mug2-9a911343951c3d13", "mug3-4453aa17a53cc1ec", "mug4-83a9ad880675e89a", "mug5-35ba4b4363e00179", "mug6-39b5b15cbfef0cf0", "mug7-70b35744d90f4507", "mug8-97007affcfefd95d", "bowl-2cb7dba2f96fa45e", "mug9-220305c9b4e89361", "bowl2-33971baa490afe80", "mug10-fb7a3d3418694721", "spoon-9a16cf6df819fd5a", "bowl3-013bcc41399cb30a", "mug11-34df5ee3533346e4", "spoon2-4f28ce4d0f1e47b7", "two spoons-9c0dc3b6c77e973e", "kettle-41a47f1de3fa994e", "coffee_jar-cd88dcffe25626ec", "bottom of blender-d1822f73fd6fe81e", "measuring spoons-320d57fbaaa3167f", "knife-986ce189d3e132fe", "large silver bowl-ed31ef1298167d78", "tupperware container-45d8005821836fba", "digital scale-399321f2e0524e4e", "tupperware container2-789949bf1d328344", "container-1a712faa183d4367", "container2-4ac6ada8964d72bf", "container_lids-6a5b7284b856d29c", "tupperware-e4d21c732b3ebc34", "dishwashing liquid-eadc8c930ba10104", "metal grille-ea85c096a633d062", "dish scrubber-16d0a94a43cf74ff", "drying mat-4de1553966fef787", "food waste bin-442b6f49ebf956b4", "sponge-ba81249bb2e96c35", "coffee machine-3cdf10e488440629", "coffee grinder-68a956d09ac830d9", "empty bags-0d90462e744d77ba", "one empty bag-97dea6983e634121", "basket-1d890532d2ada6c4", "lid of basket-763a8eda1e489494", "basket of coffee equipment-21d228d9e5010d66", "coffee spoon-a87fa1fb694c4d6d", "two tubes of green food dye-74dbd8d387703026", "drip coffee apparatus-3f1c1a0daf74e5f4", "drip coffee apparatus cover-9c8f93197f72bffa", "drip coffee apparatus cover2-480277bb89250b0d", "drying rack-ca1bb49389495bc7", "cnatiner-0f7de696eae28134", "wooden scrubbing brush-0d5aacab9105afa3", "white draining tray-ae677913a46135fe", "plant-978f4417d8e69d18", "tub of bicarbonate of soda-f157afa17522305d", "tea towel3-50a0f989f9277861", "scourer-c47ccf9d69e94653", "bottle brush-52540c8757109d6a", "dishcloth-d71834f183f86a63", "spray bottle-dd88497f62e48279", "tupperware2-bc3f303b94b6b0a6", "container3-a3ad20298cb4d997", "roll of kitchen roll-a058bf80e952cdb7", "smart home meter-205f0e7ad81c70c6", "spoon rest-98cd8fe465118fc0", "bottle of sesame oil-16d6f81bfaea0e93", "cup-c8ac729624c5072c", "stool-aaed0831c8b1f6f3", "yellow spray bottle-c3c4dba045884c98", "bottle of disinfectant-3293cc69a91aeacb", "plastic bag-fb9b5cbfd0806236", "coffee apparatus-43c510bec90c4af8", "espresso basket-52521f3b52d162c5", "lid of container-f01b54e9c53e129e", "tea towel4-ca725f448fa50817", "cup2-9b6993f5df813dff", "spoon3-24a837de3742d8de", "cap-e5d6cc584cdbfe4f" ], "reasoning": "1. I watched the video to observe when the user was putting away the black cups and lids (drip coffee apparatus-3f1c1a0daf74e5f4) into a small basket (basket of coffee equipment-21d228d9e5010d66) and saw this happen between 25:58-26:30.\n2. Between 26:02-26:08, the first layout the user tried was putting the black cups with their lids still on right next to each other in the basket (basket of coffee equipment-21d228d9e5010d66), but they did not fit this way.\n3. Between 26:09-26:29, the second layout the user tried was putting the black cups in first, then putting the lids on top of them, and this layout successfully fit into the small basket (basket of coffee equipment-21d228d9e5010d66).", "referenced_object_masks": [ "basket of coffee equipment-21d228d9e5010d66", "drip coffee apparatus-3f1c1a0daf74e5f4" ] }, { "key": "P07-20240531-101357:fdcee41111bf026e01a156c11820005a8cd6f10e", "video_id": "P07-20240531-101357", "question": "What direction would I have turned to look at the sink if it were in the opposite direction while I was standing at the refrigerator?", "answer_choice_0": "180 degrees around", "answer_choice_1": "90 degrees to the right", "answer_choice_2": "90 degrees to the left", "answer_choice_3": "90 degrees upward", "answer_choice_4": "90 degrees downward", "answer_id": 2, "answer": "90 degrees to the left", "question_type": [ "Situational Awareness", "Action Recognition", "Cause and Effect", "Egocentric & Spatial Perception", "Temporal Reasoning", "Counterfactual" ], "objects_of_interest": [ "knives-da32887663bf88e7", "sharp serrated knife-a9efa5745af4efcd", "grey bowl-2ea1ead89cf0ca39", "air fryer drawer-548ddf4a997e1060", "sink plug-e07c03e6d44177c3", "butter-91c4ac8e91d9bea9", "small bowl-835acf8e538a6821", "tea towel-6dd93b6ce1627c92", "tea towel2-883b034880fa3497", "plate-34bf570345db0358", "plate2-c933bee1cee741bd", "mug-1e19f7704ab6cba4", "plate3-14884df98ab3f8ba", "mug2-9a911343951c3d13", "mug3-4453aa17a53cc1ec", "mug4-83a9ad880675e89a", "mug5-35ba4b4363e00179", "mug6-39b5b15cbfef0cf0", "mug7-70b35744d90f4507", "mug8-97007affcfefd95d", "bowl-2cb7dba2f96fa45e", "mug9-220305c9b4e89361", "bowl2-33971baa490afe80", "mug10-fb7a3d3418694721", "spoon-9a16cf6df819fd5a", "bowl3-013bcc41399cb30a", "mug11-34df5ee3533346e4", "spoon2-4f28ce4d0f1e47b7", "two spoons-9c0dc3b6c77e973e", "kettle-41a47f1de3fa994e", "coffee_jar-cd88dcffe25626ec", "bottom of blender-d1822f73fd6fe81e", "measuring spoons-320d57fbaaa3167f", "knife-986ce189d3e132fe", "large silver bowl-ed31ef1298167d78", "tupperware container-45d8005821836fba", "digital scale-399321f2e0524e4e", "tupperware container2-789949bf1d328344", "container-1a712faa183d4367", "container2-4ac6ada8964d72bf", "container_lids-6a5b7284b856d29c", "tupperware-e4d21c732b3ebc34", "dishwashing liquid-eadc8c930ba10104", "metal grille-ea85c096a633d062", "dish scrubber-16d0a94a43cf74ff", "drying mat-4de1553966fef787", "food waste bin-442b6f49ebf956b4", "sponge-ba81249bb2e96c35", "coffee machine-3cdf10e488440629", "coffee grinder-68a956d09ac830d9", "empty bags-0d90462e744d77ba", "one empty bag-97dea6983e634121", "basket-1d890532d2ada6c4", "lid of basket-763a8eda1e489494", "basket of coffee equipment-21d228d9e5010d66", "coffee spoon-a87fa1fb694c4d6d", "two tubes of green food dye-74dbd8d387703026", "drip coffee apparatus-3f1c1a0daf74e5f4", "drip coffee apparatus cover-9c8f93197f72bffa", "drip coffee apparatus cover2-480277bb89250b0d", "drying rack-ca1bb49389495bc7", "cnatiner-0f7de696eae28134", "wooden scrubbing brush-0d5aacab9105afa3", "white draining tray-ae677913a46135fe", "plant-978f4417d8e69d18", "tub of bicarbonate of soda-f157afa17522305d", "tea towel3-50a0f989f9277861", "scourer-c47ccf9d69e94653", "bottle brush-52540c8757109d6a", "dishcloth-d71834f183f86a63", "spray bottle-dd88497f62e48279", "tupperware2-bc3f303b94b6b0a6", "container3-a3ad20298cb4d997", "roll of kitchen roll-a058bf80e952cdb7", "smart home meter-205f0e7ad81c70c6", "spoon rest-98cd8fe465118fc0", "bottle of sesame oil-16d6f81bfaea0e93", "cup-c8ac729624c5072c", "stool-aaed0831c8b1f6f3", "yellow spray bottle-c3c4dba045884c98", "bottle of disinfectant-3293cc69a91aeacb", "plastic bag-fb9b5cbfd0806236", "coffee apparatus-43c510bec90c4af8", "espresso basket-52521f3b52d162c5", "lid of container-f01b54e9c53e129e", "tea towel4-ca725f448fa50817", "cup2-9b6993f5df813dff", "spoon3-24a837de3742d8de", "cap-e5d6cc584cdbfe4f" ], "reasoning": "1. I watched the video and found the refrigerator at 00:44.\n2. I saw the user standing at the refrigerator at 00:44-00:51.\n3. I saw the sink at 00:51 when the user turned to the right to look at it.\n4. Since the user turned 90 degrees to the right to look from the refrigerator to the sink at 00:51, the sink was consequently to the right of the user while standing at the refrigerator.\n5. The opposite of \"to the right\" is \"to the left\".\n6. Therefore, the user would have turned 90 degrees to the left to look at the sink if it were in the opposite direction while they were standing at the refrigerator.", "referenced_object_masks": [] }, { "key": "P07-20240531-201703:0593f5af1eb2860633d1547cab27d9357f926fe0", "video_id": "P07-20240531-201703", "question": "How much more time elapses between when I start and finish peeling the first half of the sweet potato than when I start and finish peeling the second half?", "answer_choice_0": "6 seconds", "answer_choice_1": "16 seconds", "answer_choice_2": "1 minute and 11 seconds", "answer_choice_3": "1 minute and 27 seconds", "answer_choice_4": "2 minutes and 11 seconds", "answer_id": 1, "answer": "16 seconds", "question_type": [ "Action Recognition", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "phone-42a98d3013724a70", "can of food-d510fa3ec3938c04", "can of food2-7f2eadea9d252077", "bag of lettuce-b8666865c4280bb4", "bag of avocados-95ed02584e8ca125", "sweet potato-a233e39636ff2ee4", "food waste bin-8c45834ce35fb0ab", "scissors-fb29108f71d55dac", "grater-8570e92acf6c0efd", "metal spatula-265c8cf0c1c23fba", "wooden spoons-8d01074246c0dbf5", "chopstick-c084266fc5eae37e", "rubber spatula-629919e513255512", "knife1-bb3986bb7fa0a490", "knife2-6882b6d700d8a2b1", "knife3-9b03c290e4634797", "knife4-0ddcec2f3ea97cc5", "half sweet potato-28b396ecc66ed008", "weighing scales-671472a9d2952e34", "bowl1-345f5d6cc6ae17fa", "pen-35382b78a26652a2", "notebook-3e472d415be7e731", "baking tray-a0b987863741e114", "bottle1-ae10002246bcee88", "bottle of vegetable oil-73ff541c9a2469f9", "packet of cashew nuts-8897418cc85ea1e3", "measuring spoon-967737fd3f1eee47", "green airpods case-c4bb4ecac5cd82d0", "spice jar1-bcc82d5e486c95a2", "spice jar2-a2767262282bb364", "spice jar3-fe04f98f709e2cb3", "spice jar4-e4ebb4df530e87b0", "spice jar5-e60b0620e660437c", "lid of spice jar5-9cdcbbdddcb08952", "lid of spice jar4-d17741237173ab49", "spoon1-51ee55d5babda8f7", "phone2-6477559e162fcc4d", "blender jar-26d3262312b3562a", "blender-3fbfb5f4011dba62", "blender blade-c2ef7d34302ea708", "tray of avocados-ecc906f11957db35", "avocado1-3c750496ef8349bd", "avocado half2-2cc1097671ce2c7c", "pot of yogurt-9fc6a049d25e6d99", "spoon2-0e0c1e59465f09b1", "jar of olives-9498b5928bda152a", "bag of parsley-361f0386fa894b39", "bag of coriander-77c5dd8040ba123d", "bag of coriander2-119134c85e77d94b", "garlic bulb-6d3b451e9d38d517", "bowl2-dc0974501a95d9ea", "lettuce1-062f8055f7a0f620", "lettuce leaves-56b089f6e9afbeba", "lime-c397717c32f919cd", "lid of can of food-8dd8e9e1c80cfacd", "can opener-08c956e403814956", "avocado skin-b7e792ee060bf306", "avocado skin2-8dbb0c02231a65d2", "tea towel-0447401b38ea3638", "chopping board-56d8c949ac927956", "plastic lid1-93f3d52fb43d9853", "plastic lid2-b42d743f9579f171", "carton of milk-e8f65dd4dca2a463", "chopping board2-02c1fe3dc45f8069", "kitchen roll-183c3f0e7d6f384a", "sheet of kitchen roll-20147c2fd1a958fd", "new spoon1-4a5362653d255f9e", "glass1-79078e2b49db66ee", "plastic box1-c30a86ab7b1c215d", "white plastic lid-5efa8ba4b3bee66f", "bottle of washing up liquid-0b8d03b20f1c8166", "dish scrubbing brush-7fb3170352a0d7a8", "plastic tub-bd11bb3088d1260f", "spray bottle1-2ca06a0ae7aaaaeb", "spray bottle2-7f4728d46fd87a35", "food bin bag1-f4530464503504e7", "cloth-5da0b5b8cba1402a", "circular hob attachment-f92958a330d6664f", "yellow tray-3421baccb7e788c4", "fork1-70c218a848520e77", "bowl3-a89c549cbcbb4d73", "spoon3-df3685200281da39" ], "reasoning": "1. I watched for the user to begin peeling the sweet potato (sweet potato-a233e39636ff2ee4). I saw them trim its ends and prepare it for peeling on the cutting board from 01:23-01:30.\n2. The user began peeling the first half of the sweet potato (sweet potato-a233e39636ff2ee4) at 01:31.\n3. The user made one long knife stroke along the length of the sweet potato (sweet potato-a233e39636ff2ee4) at 02:09, marking the beginning of their peeling the second half of the sweet potato (sweet potato-a233e39636ff2ee4).\n4. The user peels this second half of the sweet potato (sweet potato-a233e39636ff2ee4) from 02:09-02:39, at which point they cut the potato (sweet potato-a233e39636ff2ee4) into two halves (OBJ, half sweet potato-28b396ecc66ed008).\n5. At this point the user goes back to peeling the first half of the sweet potato from 02:44-02:58.\n6. At 03:04 the user goes back to peeling the second half of the sweet potato (half sweet potato-28b396ecc66ed008) until 03:20.\n7. At 03:26 the user begins to slice the sweet potato half (half sweet potato-28b396ecc66ed008), marking the end of the peeling.\n8. From here, I calculated the total amount of time that elapsed from when the user began and finished peeling the first half of the sweet potato (sweet potato-a233e39636ff2ee4) by taking the start time (01:31) and substracting it from the end time (02:58): 02:58 - 01:31 = 01:27.\n9. I then calculated the total amount of time that elapsed from when the user began and finished peeling the second half of the sweet potato (sweet potato-a233e39636ff2ee4) by taking the start time (02:09) and substracting it from the end time (03:20): 03:20 - 02:09 = 01:11\n10. Finally I calculated the difference of the two times by substracting the total amount of time that it took to peel the second half of the sweet potato (sweet potato-a233e39636ff2ee4) from the total amount of time that it took to peel the first half: 01:27 - 01:11 = 00:16, or 16 seconds.", "referenced_object_masks": [ "sweet potato-a233e39636ff2ee4", "half sweet potato-28b396ecc66ed008" ] }, { "key": "P07-20240531-201703:0c1a93f8e7a7dc7b2e7793bd9f7849465e4fa4f0", "video_id": "P07-20240531-201703", "question": "If I doubled the amount of plant pots along the windowsill, how many more round containers in my spice drawer would there be than pots?", "answer_choice_0": "22", "answer_choice_1": "21", "answer_choice_2": "19", "answer_choice_3": "20", "answer_choice_4": "18", "answer_id": 0, "answer": "22", "question_type": [ "Counterfactual", "Counting", "Numerical Reasoning", "Object Recognition" ], "objects_of_interest": [ "phone-42a98d3013724a70", "can of food-d510fa3ec3938c04", "can of food2-7f2eadea9d252077", "bag of lettuce-b8666865c4280bb4", "bag of avocados-95ed02584e8ca125", "sweet potato-a233e39636ff2ee4", "food waste bin-8c45834ce35fb0ab", "scissors-fb29108f71d55dac", "grater-8570e92acf6c0efd", "metal spatula-265c8cf0c1c23fba", "wooden spoons-8d01074246c0dbf5", "chopstick-c084266fc5eae37e", "rubber spatula-629919e513255512", "knife1-bb3986bb7fa0a490", "knife2-6882b6d700d8a2b1", "knife3-9b03c290e4634797", "knife4-0ddcec2f3ea97cc5", "half sweet potato-28b396ecc66ed008", "weighing scales-671472a9d2952e34", "bowl1-345f5d6cc6ae17fa", "pen-35382b78a26652a2", "notebook-3e472d415be7e731", "baking tray-a0b987863741e114", "bottle1-ae10002246bcee88", "bottle of vegetable oil-73ff541c9a2469f9", "packet of cashew nuts-8897418cc85ea1e3", "measuring spoon-967737fd3f1eee47", "green airpods case-c4bb4ecac5cd82d0", "spice jar1-bcc82d5e486c95a2", "spice jar2-a2767262282bb364", "spice jar3-fe04f98f709e2cb3", "spice jar4-e4ebb4df530e87b0", "spice jar5-e60b0620e660437c", "lid of spice jar5-9cdcbbdddcb08952", "lid of spice jar4-d17741237173ab49", "spoon1-51ee55d5babda8f7", "phone2-6477559e162fcc4d", "blender jar-26d3262312b3562a", "blender-3fbfb5f4011dba62", "blender blade-c2ef7d34302ea708", "tray of avocados-ecc906f11957db35", "avocado1-3c750496ef8349bd", "avocado half2-2cc1097671ce2c7c", "pot of yogurt-9fc6a049d25e6d99", "spoon2-0e0c1e59465f09b1", "jar of olives-9498b5928bda152a", "bag of parsley-361f0386fa894b39", "bag of coriander-77c5dd8040ba123d", "bag of coriander2-119134c85e77d94b", "garlic bulb-6d3b451e9d38d517", "bowl2-dc0974501a95d9ea", "lettuce1-062f8055f7a0f620", "lettuce leaves-56b089f6e9afbeba", "lime-c397717c32f919cd", "lid of can of food-8dd8e9e1c80cfacd", "can opener-08c956e403814956", "avocado skin-b7e792ee060bf306", "avocado skin2-8dbb0c02231a65d2", "tea towel-0447401b38ea3638", "chopping board-56d8c949ac927956", "plastic lid1-93f3d52fb43d9853", "plastic lid2-b42d743f9579f171", "carton of milk-e8f65dd4dca2a463", "chopping board2-02c1fe3dc45f8069", "kitchen roll-183c3f0e7d6f384a", "sheet of kitchen roll-20147c2fd1a958fd", "new spoon1-4a5362653d255f9e", "glass1-79078e2b49db66ee", "plastic box1-c30a86ab7b1c215d", "white plastic lid-5efa8ba4b3bee66f", "bottle of washing up liquid-0b8d03b20f1c8166", "dish scrubbing brush-7fb3170352a0d7a8", "plastic tub-bd11bb3088d1260f", "spray bottle1-2ca06a0ae7aaaaeb", "spray bottle2-7f4728d46fd87a35", "food bin bag1-f4530464503504e7", "cloth-5da0b5b8cba1402a", "circular hob attachment-f92958a330d6664f", "yellow tray-3421baccb7e788c4", "fork1-70c218a848520e77", "bowl3-a89c549cbcbb4d73", "spoon3-df3685200281da39" ], "reasoning": "1. I looked for a clear view of plant pots by a window. At 00:18 I could count 7 pots along the window behind the sink.\n2. I then looked for round containers in a spice drawer. I saw the user open a drawer at 06:36 with a full view of its contents, including many spice jars, at 06:38.\n3. I counted only the round containers inside the drawer for a total of 36 round containers .\n4. I doubled the amount of plant pots for a total of 14.\n5. I subtracted 14 from 36 to reach 22.", "referenced_object_masks": [] }, { "key": "P07-20240531-201703:6ede78d50f78dc25cf17815ba7b0cc19a26ff423", "video_id": "P07-20240531-201703", "question": "What is the difference in the yogurt container's position between the beginning and the end of the video?", "answer_choice_0": "It is thrown away in the trash", "answer_choice_1": "It is put in the microwave", "answer_choice_2": "It is Placed on the counter", "answer_choice_3": "It is put in the sink", "answer_choice_4": "It is moved to the refrigerator", "answer_id": 4, "answer": "It is moved to the refrigerator", "question_type": [ "State Changes", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "phone-42a98d3013724a70", "can of food-d510fa3ec3938c04", "can of food2-7f2eadea9d252077", "bag of lettuce-b8666865c4280bb4", "bag of avocados-95ed02584e8ca125", "sweet potato-a233e39636ff2ee4", "food waste bin-8c45834ce35fb0ab", "scissors-fb29108f71d55dac", "grater-8570e92acf6c0efd", "metal spatula-265c8cf0c1c23fba", "wooden spoons-8d01074246c0dbf5", "chopstick-c084266fc5eae37e", "rubber spatula-629919e513255512", "knife1-bb3986bb7fa0a490", "knife2-6882b6d700d8a2b1", "knife3-9b03c290e4634797", "knife4-0ddcec2f3ea97cc5", "half sweet potato-28b396ecc66ed008", "weighing scales-671472a9d2952e34", "bowl1-345f5d6cc6ae17fa", "pen-35382b78a26652a2", "notebook-3e472d415be7e731", "baking tray-a0b987863741e114", "bottle1-ae10002246bcee88", "bottle of vegetable oil-73ff541c9a2469f9", "packet of cashew nuts-8897418cc85ea1e3", "measuring spoon-967737fd3f1eee47", "green airpods case-c4bb4ecac5cd82d0", "spice jar1-bcc82d5e486c95a2", "spice jar2-a2767262282bb364", "spice jar3-fe04f98f709e2cb3", "spice jar4-e4ebb4df530e87b0", "spice jar5-e60b0620e660437c", "lid of spice jar5-9cdcbbdddcb08952", "lid of spice jar4-d17741237173ab49", "spoon1-51ee55d5babda8f7", "phone2-6477559e162fcc4d", "blender jar-26d3262312b3562a", "blender-3fbfb5f4011dba62", "blender blade-c2ef7d34302ea708", "tray of avocados-ecc906f11957db35", "avocado1-3c750496ef8349bd", "avocado half2-2cc1097671ce2c7c", "pot of yogurt-9fc6a049d25e6d99", "spoon2-0e0c1e59465f09b1", "jar of olives-9498b5928bda152a", "bag of parsley-361f0386fa894b39", "bag of coriander-77c5dd8040ba123d", "bag of coriander2-119134c85e77d94b", "garlic bulb-6d3b451e9d38d517", "bowl2-dc0974501a95d9ea", "lettuce1-062f8055f7a0f620", "lettuce leaves-56b089f6e9afbeba", "lime-c397717c32f919cd", "lid of can of food-8dd8e9e1c80cfacd", "can opener-08c956e403814956", "avocado skin-b7e792ee060bf306", "avocado skin2-8dbb0c02231a65d2", "tea towel-0447401b38ea3638", "chopping board-56d8c949ac927956", "plastic lid1-93f3d52fb43d9853", "plastic lid2-b42d743f9579f171", "carton of milk-e8f65dd4dca2a463", "chopping board2-02c1fe3dc45f8069", "kitchen roll-183c3f0e7d6f384a", "sheet of kitchen roll-20147c2fd1a958fd", "new spoon1-4a5362653d255f9e", "glass1-79078e2b49db66ee", "plastic box1-c30a86ab7b1c215d", "white plastic lid-5efa8ba4b3bee66f", "bottle of washing up liquid-0b8d03b20f1c8166", "dish scrubbing brush-7fb3170352a0d7a8", "plastic tub-bd11bb3088d1260f", "spray bottle1-2ca06a0ae7aaaaeb", "spray bottle2-7f4728d46fd87a35", "food bin bag1-f4530464503504e7", "cloth-5da0b5b8cba1402a", "circular hob attachment-f92958a330d6664f", "yellow tray-3421baccb7e788c4", "fork1-70c218a848520e77", "bowl3-a89c549cbcbb4d73", "spoon3-df3685200281da39" ], "reasoning": "1. I watched the video to see when the yogurt container (pot of yogurt-9fc6a049d25e6d99) is first seen. This occurs from 11:34-11:44, when the yogurt container (pot of yogurt-9fc6a049d25e6d99) is grabbed from the counter.\n2. From 11:34-11:44, the user scoops some yogurt (pot of yogurt-9fc6a049d25e6d99) into the blender jar (blender jar-26d3262312b3562a).\n3. When done with this task, the user puts the yogurt (pot of yogurt-9fc6a049d25e6d99) on the cutting board , where it sits as the user works on other ingredients. \n4. I continued to watch to see what the user did with the yogurt (pot of yogurt-9fc6a049d25e6d99), observing from 23:37-23:42, when the user grabs the yogurt (pot of yogurt-9fc6a049d25e6d99) and puts it in the refrigerator .", "referenced_object_masks": [ "blender jar-26d3262312b3562a", "pot of yogurt-9fc6a049d25e6d99" ] }, { "key": "P07-20240531-201703:713deab077068eacb134e72cbb92ccd989d63d9d", "video_id": "P07-20240531-201703", "question": "If I would have used the third knife I touched in the knife block, how many knives from the left of the block would it have been?", "answer_choice_0": "4", "answer_choice_1": "3", "answer_choice_2": "1", "answer_choice_3": "5", "answer_choice_4": "2", "answer_id": 1, "answer": "3", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Counterfactual" ], "objects_of_interest": [ "phone-42a98d3013724a70", "can of food-d510fa3ec3938c04", "can of food2-7f2eadea9d252077", "bag of lettuce-b8666865c4280bb4", "bag of avocados-95ed02584e8ca125", "sweet potato-a233e39636ff2ee4", "food waste bin-8c45834ce35fb0ab", "scissors-fb29108f71d55dac", "grater-8570e92acf6c0efd", "metal spatula-265c8cf0c1c23fba", "wooden spoons-8d01074246c0dbf5", "chopstick-c084266fc5eae37e", "rubber spatula-629919e513255512", "knife1-bb3986bb7fa0a490", "knife2-6882b6d700d8a2b1", "knife3-9b03c290e4634797", "knife4-0ddcec2f3ea97cc5", "half sweet potato-28b396ecc66ed008", "weighing scales-671472a9d2952e34", "bowl1-345f5d6cc6ae17fa", "pen-35382b78a26652a2", "notebook-3e472d415be7e731", "baking tray-a0b987863741e114", "bottle1-ae10002246bcee88", "bottle of vegetable oil-73ff541c9a2469f9", "packet of cashew nuts-8897418cc85ea1e3", "measuring spoon-967737fd3f1eee47", "green airpods case-c4bb4ecac5cd82d0", "spice jar1-bcc82d5e486c95a2", "spice jar2-a2767262282bb364", "spice jar3-fe04f98f709e2cb3", "spice jar4-e4ebb4df530e87b0", "spice jar5-e60b0620e660437c", "lid of spice jar5-9cdcbbdddcb08952", "lid of spice jar4-d17741237173ab49", "spoon1-51ee55d5babda8f7", "phone2-6477559e162fcc4d", "blender jar-26d3262312b3562a", "blender-3fbfb5f4011dba62", "blender blade-c2ef7d34302ea708", "tray of avocados-ecc906f11957db35", "avocado1-3c750496ef8349bd", "avocado half2-2cc1097671ce2c7c", "pot of yogurt-9fc6a049d25e6d99", "spoon2-0e0c1e59465f09b1", "jar of olives-9498b5928bda152a", "bag of parsley-361f0386fa894b39", "bag of coriander-77c5dd8040ba123d", "bag of coriander2-119134c85e77d94b", "garlic bulb-6d3b451e9d38d517", "bowl2-dc0974501a95d9ea", "lettuce1-062f8055f7a0f620", "lettuce leaves-56b089f6e9afbeba", "lime-c397717c32f919cd", "lid of can of food-8dd8e9e1c80cfacd", "can opener-08c956e403814956", "avocado skin-b7e792ee060bf306", "avocado skin2-8dbb0c02231a65d2", "tea towel-0447401b38ea3638", "chopping board-56d8c949ac927956", "plastic lid1-93f3d52fb43d9853", "plastic lid2-b42d743f9579f171", "carton of milk-e8f65dd4dca2a463", "chopping board2-02c1fe3dc45f8069", "kitchen roll-183c3f0e7d6f384a", "sheet of kitchen roll-20147c2fd1a958fd", "new spoon1-4a5362653d255f9e", "glass1-79078e2b49db66ee", "plastic box1-c30a86ab7b1c215d", "white plastic lid-5efa8ba4b3bee66f", "bottle of washing up liquid-0b8d03b20f1c8166", "dish scrubbing brush-7fb3170352a0d7a8", "plastic tub-bd11bb3088d1260f", "spray bottle1-2ca06a0ae7aaaaeb", "spray bottle2-7f4728d46fd87a35", "food bin bag1-f4530464503504e7", "cloth-5da0b5b8cba1402a", "circular hob attachment-f92958a330d6664f", "yellow tray-3421baccb7e788c4", "fork1-70c218a848520e77", "bowl3-a89c549cbcbb4d73", "spoon3-df3685200281da39" ], "reasoning": "1. I watched the video to see the user stand in front of the knife block at 01:13.\n2. Next, I saw the user grab the knife (knife1-bb3986bb7fa0a490) that is four from the left in the knife block . The user holds this knife 9knife1-bb3986bb7fa0a490) for a second before returning it to its place in the knife block .\n3. Next, the user touches knife 2 (knife2-6882b6d700d8a2b1) at 01:18, which is the furthest to the left in the knife block .\n4. The user then touches and partially lifts knife 3 (knife3-9b03c290e4634797) from the knife block , which is three from the left in the block . This knife is also the third one touched by the user, making it the answer to the question.", "referenced_object_masks": [ "knife2-6882b6d700d8a2b1", "knife3-9b03c290e4634797", "knife1-bb3986bb7fa0a490" ] }, { "key": "P07-20240531-201703:7e711d02e6c79def551b769bb633bf7fff2042db", "video_id": "P07-20240531-201703", "question": "How many distinct items did I touch with my right hand inside the second drawer that I opened with my left hand?", "answer_choice_0": "12", "answer_choice_1": "10", "answer_choice_2": "13", "answer_choice_3": "9", "answer_choice_4": "11", "answer_id": 4, "answer": "11", "question_type": [ "Counting", "Egocentric & Spatial Perception", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "phone-42a98d3013724a70", "can of food-d510fa3ec3938c04", "can of food2-7f2eadea9d252077", "bag of lettuce-b8666865c4280bb4", "bag of avocados-95ed02584e8ca125", "sweet potato-a233e39636ff2ee4", "food waste bin-8c45834ce35fb0ab", "scissors-fb29108f71d55dac", "grater-8570e92acf6c0efd", "metal spatula-265c8cf0c1c23fba", "wooden spoons-8d01074246c0dbf5", "chopstick-c084266fc5eae37e", "rubber spatula-629919e513255512", "knife1-bb3986bb7fa0a490", "knife2-6882b6d700d8a2b1", "knife3-9b03c290e4634797", "knife4-0ddcec2f3ea97cc5", "half sweet potato-28b396ecc66ed008", "weighing scales-671472a9d2952e34", "bowl1-345f5d6cc6ae17fa", "pen-35382b78a26652a2", "notebook-3e472d415be7e731", "baking tray-a0b987863741e114", "bottle1-ae10002246bcee88", "bottle of vegetable oil-73ff541c9a2469f9", "packet of cashew nuts-8897418cc85ea1e3", "measuring spoon-967737fd3f1eee47", "green airpods case-c4bb4ecac5cd82d0", "spice jar1-bcc82d5e486c95a2", "spice jar2-a2767262282bb364", "spice jar3-fe04f98f709e2cb3", "spice jar4-e4ebb4df530e87b0", "spice jar5-e60b0620e660437c", "lid of spice jar5-9cdcbbdddcb08952", "lid of spice jar4-d17741237173ab49", "spoon1-51ee55d5babda8f7", "phone2-6477559e162fcc4d", "blender jar-26d3262312b3562a", "blender-3fbfb5f4011dba62", "blender blade-c2ef7d34302ea708", "tray of avocados-ecc906f11957db35", "avocado1-3c750496ef8349bd", "avocado half2-2cc1097671ce2c7c", "pot of yogurt-9fc6a049d25e6d99", "spoon2-0e0c1e59465f09b1", "jar of olives-9498b5928bda152a", "bag of parsley-361f0386fa894b39", "bag of coriander-77c5dd8040ba123d", "bag of coriander2-119134c85e77d94b", "garlic bulb-6d3b451e9d38d517", "bowl2-dc0974501a95d9ea", "lettuce1-062f8055f7a0f620", "lettuce leaves-56b089f6e9afbeba", "lime-c397717c32f919cd", "lid of can of food-8dd8e9e1c80cfacd", "can opener-08c956e403814956", "avocado skin-b7e792ee060bf306", "avocado skin2-8dbb0c02231a65d2", "tea towel-0447401b38ea3638", "chopping board-56d8c949ac927956", "plastic lid1-93f3d52fb43d9853", "plastic lid2-b42d743f9579f171", "carton of milk-e8f65dd4dca2a463", "chopping board2-02c1fe3dc45f8069", "kitchen roll-183c3f0e7d6f384a", "sheet of kitchen roll-20147c2fd1a958fd", "new spoon1-4a5362653d255f9e", "glass1-79078e2b49db66ee", "plastic box1-c30a86ab7b1c215d", "white plastic lid-5efa8ba4b3bee66f", "bottle of washing up liquid-0b8d03b20f1c8166", "dish scrubbing brush-7fb3170352a0d7a8", "plastic tub-bd11bb3088d1260f", "spray bottle1-2ca06a0ae7aaaaeb", "spray bottle2-7f4728d46fd87a35", "food bin bag1-f4530464503504e7", "cloth-5da0b5b8cba1402a", "circular hob attachment-f92958a330d6664f", "yellow tray-3421baccb7e788c4", "fork1-70c218a848520e77", "bowl3-a89c549cbcbb4d73", "spoon3-df3685200281da39" ], "reasoning": "1. I watched for the user to open drawers with their left hand. I saw them open the first at 00:44 and the second at 06:36.\n2. I counted the things the user touched inside the drawer with their left hand: a jar of spices at 06:41, a second jar of spices (spice jar1-bcc82d5e486c95a2) at 06:43, a third jar of spices (spice jar2-a2767262282bb364) at 06:45, a fourth jar of spices at 06:48, a fifth jar of spices (spice jar3-fe04f98f709e2cb3) at 06:49, a sixth jar of spices at 06:52, a seventh jar of spices (spice jar4-e4ebb4df530e87b0) at 06:55, an eighth jar of spices at 07:00, a ninth jar of spices at 07:03, a tenth jar of spices at 07:04, and an eleventh jar of spices (spice jar5-e60b0620e660437c) at 07:06.\n3. The user opens the drawer again at 09:11 to return the spice jars but does not touch anything else inside with their right hand.\n4. Therefore the user touched 11 things with their right hand.", "referenced_object_masks": [ "spice jar5-e60b0620e660437c", "spice jar1-bcc82d5e486c95a2", "spice jar3-fe04f98f709e2cb3", "spice jar2-a2767262282bb364", "spice jar4-e4ebb4df530e87b0" ] }, { "key": "P07-20240531-201703:a38d7ff18b8583b70f98679d5baa09fa51abb5f0", "video_id": "P07-20240531-201703", "question": "Into how many more pieces did I cut the second half of the sweet potato than the first?", "answer_choice_0": "14", "answer_choice_1": "12", "answer_choice_2": "17", "answer_choice_3": "15", "answer_choice_4": "10", "answer_id": 1, "answer": "12", "question_type": [ "Action Recognition", "Counting", "Object Recognition", "State Changes" ], "objects_of_interest": [ "phone-42a98d3013724a70", "can of food-d510fa3ec3938c04", "can of food2-7f2eadea9d252077", "bag of lettuce-b8666865c4280bb4", "bag of avocados-95ed02584e8ca125", "sweet potato-a233e39636ff2ee4", "food waste bin-8c45834ce35fb0ab", "scissors-fb29108f71d55dac", "grater-8570e92acf6c0efd", "metal spatula-265c8cf0c1c23fba", "wooden spoons-8d01074246c0dbf5", "chopstick-c084266fc5eae37e", "rubber spatula-629919e513255512", "knife1-bb3986bb7fa0a490", "knife2-6882b6d700d8a2b1", "knife3-9b03c290e4634797", "knife4-0ddcec2f3ea97cc5", "half sweet potato-28b396ecc66ed008", "weighing scales-671472a9d2952e34", "bowl1-345f5d6cc6ae17fa", "pen-35382b78a26652a2", "notebook-3e472d415be7e731", "baking tray-a0b987863741e114", "bottle1-ae10002246bcee88", "bottle of vegetable oil-73ff541c9a2469f9", "packet of cashew nuts-8897418cc85ea1e3", "measuring spoon-967737fd3f1eee47", "green airpods case-c4bb4ecac5cd82d0", "spice jar1-bcc82d5e486c95a2", "spice jar2-a2767262282bb364", "spice jar3-fe04f98f709e2cb3", "spice jar4-e4ebb4df530e87b0", "spice jar5-e60b0620e660437c", "lid of spice jar5-9cdcbbdddcb08952", "lid of spice jar4-d17741237173ab49", "spoon1-51ee55d5babda8f7", "phone2-6477559e162fcc4d", "blender jar-26d3262312b3562a", "blender-3fbfb5f4011dba62", "blender blade-c2ef7d34302ea708", "tray of avocados-ecc906f11957db35", "avocado1-3c750496ef8349bd", "avocado half2-2cc1097671ce2c7c", "pot of yogurt-9fc6a049d25e6d99", "spoon2-0e0c1e59465f09b1", "jar of olives-9498b5928bda152a", "bag of parsley-361f0386fa894b39", "bag of coriander-77c5dd8040ba123d", "bag of coriander2-119134c85e77d94b", "garlic bulb-6d3b451e9d38d517", "bowl2-dc0974501a95d9ea", "lettuce1-062f8055f7a0f620", "lettuce leaves-56b089f6e9afbeba", "lime-c397717c32f919cd", "lid of can of food-8dd8e9e1c80cfacd", "can opener-08c956e403814956", "avocado skin-b7e792ee060bf306", "avocado skin2-8dbb0c02231a65d2", "tea towel-0447401b38ea3638", "chopping board-56d8c949ac927956", "plastic lid1-93f3d52fb43d9853", "plastic lid2-b42d743f9579f171", "carton of milk-e8f65dd4dca2a463", "chopping board2-02c1fe3dc45f8069", "kitchen roll-183c3f0e7d6f384a", "sheet of kitchen roll-20147c2fd1a958fd", "new spoon1-4a5362653d255f9e", "glass1-79078e2b49db66ee", "plastic box1-c30a86ab7b1c215d", "white plastic lid-5efa8ba4b3bee66f", "bottle of washing up liquid-0b8d03b20f1c8166", "dish scrubbing brush-7fb3170352a0d7a8", "plastic tub-bd11bb3088d1260f", "spray bottle1-2ca06a0ae7aaaaeb", "spray bottle2-7f4728d46fd87a35", "food bin bag1-f4530464503504e7", "cloth-5da0b5b8cba1402a", "circular hob attachment-f92958a330d6664f", "yellow tray-3421baccb7e788c4", "fork1-70c218a848520e77", "bowl3-a89c549cbcbb4d73", "spoon3-df3685200281da39" ], "reasoning": "1. I watched for the user to cut half of a sweet potato into smaller pieces.\n2. From 03:25-03:42 I saw the user cut the first half of the sweet potato (half sweet potato-28b396ecc66ed008) into 16 pieces.\n3. From 03:44-04:18 I saw the user cut the second half of the sweet potato into 28 pieces.\n4. I subtracted 16 from 28 to reach 12 pieces.", "referenced_object_masks": [ "half sweet potato-28b396ecc66ed008" ] }, { "key": "P07-20240531-201703:af1aff544b8067422bcd7bdf73cef8c7ed78d60f", "video_id": "P07-20240531-201703", "question": "Next to what two items do I place the carton of milk in the refrigerator?", "answer_choice_0": "Avocado and onion", "answer_choice_1": "Can of corn and avocado", "answer_choice_2": "Can of beans and can of corn", "answer_choice_3": "Can of beans and pot of yogurt", "answer_choice_4": "Pot of yogurt and avocado", "answer_id": 2, "answer": "Can of beans and can of corn", "question_type": [ "Object Recognition", "Egocentric & Spatial Perception", "Situational Awareness" ], "objects_of_interest": [ "phone-42a98d3013724a70", "can of food-d510fa3ec3938c04", "can of food2-7f2eadea9d252077", "bag of lettuce-b8666865c4280bb4", "bag of avocados-95ed02584e8ca125", "sweet potato-a233e39636ff2ee4", "food waste bin-8c45834ce35fb0ab", "scissors-fb29108f71d55dac", "grater-8570e92acf6c0efd", "metal spatula-265c8cf0c1c23fba", "wooden spoons-8d01074246c0dbf5", "chopstick-c084266fc5eae37e", "rubber spatula-629919e513255512", "knife1-bb3986bb7fa0a490", "knife2-6882b6d700d8a2b1", "knife3-9b03c290e4634797", "knife4-0ddcec2f3ea97cc5", "half sweet potato-28b396ecc66ed008", "weighing scales-671472a9d2952e34", "bowl1-345f5d6cc6ae17fa", "pen-35382b78a26652a2", "notebook-3e472d415be7e731", "baking tray-a0b987863741e114", "bottle1-ae10002246bcee88", "bottle of vegetable oil-73ff541c9a2469f9", "packet of cashew nuts-8897418cc85ea1e3", "measuring spoon-967737fd3f1eee47", "green airpods case-c4bb4ecac5cd82d0", "spice jar1-bcc82d5e486c95a2", "spice jar2-a2767262282bb364", "spice jar3-fe04f98f709e2cb3", "spice jar4-e4ebb4df530e87b0", "spice jar5-e60b0620e660437c", "lid of spice jar5-9cdcbbdddcb08952", "lid of spice jar4-d17741237173ab49", "spoon1-51ee55d5babda8f7", "phone2-6477559e162fcc4d", "blender jar-26d3262312b3562a", "blender-3fbfb5f4011dba62", "blender blade-c2ef7d34302ea708", "tray of avocados-ecc906f11957db35", "avocado1-3c750496ef8349bd", "avocado half2-2cc1097671ce2c7c", "pot of yogurt-9fc6a049d25e6d99", "spoon2-0e0c1e59465f09b1", "jar of olives-9498b5928bda152a", "bag of parsley-361f0386fa894b39", "bag of coriander-77c5dd8040ba123d", "bag of coriander2-119134c85e77d94b", "garlic bulb-6d3b451e9d38d517", "bowl2-dc0974501a95d9ea", "lettuce1-062f8055f7a0f620", "lettuce leaves-56b089f6e9afbeba", "lime-c397717c32f919cd", "lid of can of food-8dd8e9e1c80cfacd", "can opener-08c956e403814956", "avocado skin-b7e792ee060bf306", "avocado skin2-8dbb0c02231a65d2", "tea towel-0447401b38ea3638", "chopping board-56d8c949ac927956", "plastic lid1-93f3d52fb43d9853", "plastic lid2-b42d743f9579f171", "carton of milk-e8f65dd4dca2a463", "chopping board2-02c1fe3dc45f8069", "kitchen roll-183c3f0e7d6f384a", "sheet of kitchen roll-20147c2fd1a958fd", "new spoon1-4a5362653d255f9e", "glass1-79078e2b49db66ee", "plastic box1-c30a86ab7b1c215d", "white plastic lid-5efa8ba4b3bee66f", "bottle of washing up liquid-0b8d03b20f1c8166", "dish scrubbing brush-7fb3170352a0d7a8", "plastic tub-bd11bb3088d1260f", "spray bottle1-2ca06a0ae7aaaaeb", "spray bottle2-7f4728d46fd87a35", "food bin bag1-f4530464503504e7", "cloth-5da0b5b8cba1402a", "circular hob attachment-f92958a330d6664f", "yellow tray-3421baccb7e788c4", "fork1-70c218a848520e77", "bowl3-a89c549cbcbb4d73", "spoon3-df3685200281da39" ], "reasoning": "1. I watched the video to identify when the carton of milk (carton of milk-e8f65dd4dca2a463) is placed in the refrigerator , noting that it takes place from 24:47-24:51.\n2. During this span, I noted that the carton of milk (carton of milk-e8f65dd4dca2a463) is laid next to two cans, one with a blue cover and the other with a red cover.\n3. I rewound the video and saw the user put the blue cover on the can of corn from 24:21-24:28.\nThen, from 24:28-24:33, the user puts the red cover on the can of black beans .\n4. I next observed the user carry these cans to the refrigerator placing them on the middle shelf from 24:37-24:43.\n5. I then watched the carton of milk (carton of milk-e8f65dd4dca2a463) get placed in the refrigerator, confirming that it is placed next to the beans and corn .", "referenced_object_masks": [ "carton of milk-e8f65dd4dca2a463" ] }, { "key": "P07-20240531-201703:bd857dcc6aab7b80a8c76ca0d26af33be6769636", "video_id": "P07-20240531-201703", "question": "Which ingredient do I use in the salad in two different ways?", "answer_choice_0": "Beans", "answer_choice_1": "Yogurt", "answer_choice_2": "Avocado", "answer_choice_3": "Corn", "answer_choice_4": "Lettuce", "answer_id": 2, "answer": "Avocado", "question_type": [ "State Changes", "Object Recognition", "Action Recognition", "Situational Awareness" ], "objects_of_interest": [ "phone-42a98d3013724a70", "can of food-d510fa3ec3938c04", "can of food2-7f2eadea9d252077", "bag of lettuce-b8666865c4280bb4", "bag of avocados-95ed02584e8ca125", "sweet potato-a233e39636ff2ee4", "food waste bin-8c45834ce35fb0ab", "scissors-fb29108f71d55dac", "grater-8570e92acf6c0efd", "metal spatula-265c8cf0c1c23fba", "wooden spoons-8d01074246c0dbf5", "chopstick-c084266fc5eae37e", "rubber spatula-629919e513255512", "knife1-bb3986bb7fa0a490", "knife2-6882b6d700d8a2b1", "knife3-9b03c290e4634797", "knife4-0ddcec2f3ea97cc5", "half sweet potato-28b396ecc66ed008", "weighing scales-671472a9d2952e34", "bowl1-345f5d6cc6ae17fa", "pen-35382b78a26652a2", "notebook-3e472d415be7e731", "baking tray-a0b987863741e114", "bottle1-ae10002246bcee88", "bottle of vegetable oil-73ff541c9a2469f9", "packet of cashew nuts-8897418cc85ea1e3", "measuring spoon-967737fd3f1eee47", "green airpods case-c4bb4ecac5cd82d0", "spice jar1-bcc82d5e486c95a2", "spice jar2-a2767262282bb364", "spice jar3-fe04f98f709e2cb3", "spice jar4-e4ebb4df530e87b0", "spice jar5-e60b0620e660437c", "lid of spice jar5-9cdcbbdddcb08952", "lid of spice jar4-d17741237173ab49", "spoon1-51ee55d5babda8f7", "phone2-6477559e162fcc4d", "blender jar-26d3262312b3562a", "blender-3fbfb5f4011dba62", "blender blade-c2ef7d34302ea708", "tray of avocados-ecc906f11957db35", "avocado1-3c750496ef8349bd", "avocado half2-2cc1097671ce2c7c", "pot of yogurt-9fc6a049d25e6d99", "spoon2-0e0c1e59465f09b1", "jar of olives-9498b5928bda152a", "bag of parsley-361f0386fa894b39", "bag of coriander-77c5dd8040ba123d", "bag of coriander2-119134c85e77d94b", "garlic bulb-6d3b451e9d38d517", "bowl2-dc0974501a95d9ea", "lettuce1-062f8055f7a0f620", "lettuce leaves-56b089f6e9afbeba", "lime-c397717c32f919cd", "lid of can of food-8dd8e9e1c80cfacd", "can opener-08c956e403814956", "avocado skin-b7e792ee060bf306", "avocado skin2-8dbb0c02231a65d2", "tea towel-0447401b38ea3638", "chopping board-56d8c949ac927956", "plastic lid1-93f3d52fb43d9853", "plastic lid2-b42d743f9579f171", "carton of milk-e8f65dd4dca2a463", "chopping board2-02c1fe3dc45f8069", "kitchen roll-183c3f0e7d6f384a", "sheet of kitchen roll-20147c2fd1a958fd", "new spoon1-4a5362653d255f9e", "glass1-79078e2b49db66ee", "plastic box1-c30a86ab7b1c215d", "white plastic lid-5efa8ba4b3bee66f", "bottle of washing up liquid-0b8d03b20f1c8166", "dish scrubbing brush-7fb3170352a0d7a8", "plastic tub-bd11bb3088d1260f", "spray bottle1-2ca06a0ae7aaaaeb", "spray bottle2-7f4728d46fd87a35", "food bin bag1-f4530464503504e7", "cloth-5da0b5b8cba1402a", "circular hob attachment-f92958a330d6664f", "yellow tray-3421baccb7e788c4", "fork1-70c218a848520e77", "bowl3-a89c549cbcbb4d73", "spoon3-df3685200281da39" ], "reasoning": "1. I watched the video to observe the user making the salad, which occurrs until around 22:00 when the user puts the dressing on the salad .\n2. First, I watched the user prep the sweet potato (sweet potato-a233e39636ff2ee4) from 01:21-08:25. They cube the potato (sweet potato-a233e39636ff2ee4) and season it before putting it in the oven . These are added to the salad from 21:25-21:35.\n3. Next, I watched the user grab the tray of avocadoes (bag of avocados-95ed02584e8ca125) at 09:40.\n4. Then, I watched the user cut open the first avocado (avocado1-3c750496ef8349bd) and put it into the blender cup (blender jar-26d3262312b3562a) from 10:00-10:47.\n5. After this, the user grabs the yogurt (pot of yogurt-9fc6a049d25e6d99) from the counter and scoops some into the blender jar (blender jar-26d3262312b3562a) from 11:34-11:44. The user also adds parsley (bag of parsley-361f0386fa894b39) to the blender jar (blender jar-26d3262312b3562a) from 12:48-12:58.\n6. Next, from 13:24-14:16, the user cut and peeled garlic (garlic bulb-6d3b451e9d38d517) before adding it to the blender jar (blender jar-26d3262312b3562a).\n7. From 15:27-17:10, the user washes and cuts lettuce (bag of lettuce-b8666865c4280bb4) for the salad.\n8. Then, from 17:21-17:51, the user scoops, weighs and then adds corn to the salad.\n9. After this, from 18:04-19:36, the user opens a can of black beans before draining them, weighing them, and putting them in the salad. \n10. Next, the user cut the second half of an avocado (avocado half2-2cc1097671ce2c7c) from 19:45-20:26.\n11. Lastly, I observed the user pour the contents of the blender jar (blender jar-26d3262312b3562a) into the salad from 21:37-21:44, meaning avocado is used both in this misture and as an additional ingrediient.", "referenced_object_masks": [ "pot of yogurt-9fc6a049d25e6d99", "sweet potato-a233e39636ff2ee4", "avocado half2-2cc1097671ce2c7c", "garlic bulb-6d3b451e9d38d517", "blender jar-26d3262312b3562a", "bag of avocados-95ed02584e8ca125", "bag of parsley-361f0386fa894b39", "avocado1-3c750496ef8349bd", "bag of lettuce-b8666865c4280bb4" ] }, { "key": "P07-20240531-201703:c143cada6b4cfca0bcc07a6ba1017b534bdca7d4", "video_id": "P07-20240531-201703", "question": "After scooping the yogurt into the blender cup, what is the average of all the numbers that show on the scale as I weigh the food?", "answer_choice_0": "70.7", "answer_choice_1": "62.6", "answer_choice_2": "55.3", "answer_choice_3": "67.2", "answer_choice_4": "60.5", "answer_id": 1, "answer": "62.6", "question_type": [ "Event Occurence", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "phone-42a98d3013724a70", "can of food-d510fa3ec3938c04", "can of food2-7f2eadea9d252077", "bag of lettuce-b8666865c4280bb4", "bag of avocados-95ed02584e8ca125", "sweet potato-a233e39636ff2ee4", "food waste bin-8c45834ce35fb0ab", "scissors-fb29108f71d55dac", "grater-8570e92acf6c0efd", "metal spatula-265c8cf0c1c23fba", "wooden spoons-8d01074246c0dbf5", "chopstick-c084266fc5eae37e", "rubber spatula-629919e513255512", "knife1-bb3986bb7fa0a490", "knife2-6882b6d700d8a2b1", "knife3-9b03c290e4634797", "knife4-0ddcec2f3ea97cc5", "half sweet potato-28b396ecc66ed008", "weighing scales-671472a9d2952e34", "bowl1-345f5d6cc6ae17fa", "pen-35382b78a26652a2", "notebook-3e472d415be7e731", "baking tray-a0b987863741e114", "bottle1-ae10002246bcee88", "bottle of vegetable oil-73ff541c9a2469f9", "packet of cashew nuts-8897418cc85ea1e3", "measuring spoon-967737fd3f1eee47", "green airpods case-c4bb4ecac5cd82d0", "spice jar1-bcc82d5e486c95a2", "spice jar2-a2767262282bb364", "spice jar3-fe04f98f709e2cb3", "spice jar4-e4ebb4df530e87b0", "spice jar5-e60b0620e660437c", "lid of spice jar5-9cdcbbdddcb08952", "lid of spice jar4-d17741237173ab49", "spoon1-51ee55d5babda8f7", "phone2-6477559e162fcc4d", "blender jar-26d3262312b3562a", "blender-3fbfb5f4011dba62", "blender blade-c2ef7d34302ea708", "tray of avocados-ecc906f11957db35", "avocado1-3c750496ef8349bd", "avocado half2-2cc1097671ce2c7c", "pot of yogurt-9fc6a049d25e6d99", "spoon2-0e0c1e59465f09b1", "jar of olives-9498b5928bda152a", "bag of parsley-361f0386fa894b39", "bag of coriander-77c5dd8040ba123d", "bag of coriander2-119134c85e77d94b", "garlic bulb-6d3b451e9d38d517", "bowl2-dc0974501a95d9ea", "lettuce1-062f8055f7a0f620", "lettuce leaves-56b089f6e9afbeba", "lime-c397717c32f919cd", "lid of can of food-8dd8e9e1c80cfacd", "can opener-08c956e403814956", "avocado skin-b7e792ee060bf306", "avocado skin2-8dbb0c02231a65d2", "tea towel-0447401b38ea3638", "chopping board-56d8c949ac927956", "plastic lid1-93f3d52fb43d9853", "plastic lid2-b42d743f9579f171", "carton of milk-e8f65dd4dca2a463", "chopping board2-02c1fe3dc45f8069", "kitchen roll-183c3f0e7d6f384a", "sheet of kitchen roll-20147c2fd1a958fd", "new spoon1-4a5362653d255f9e", "glass1-79078e2b49db66ee", "plastic box1-c30a86ab7b1c215d", "white plastic lid-5efa8ba4b3bee66f", "bottle of washing up liquid-0b8d03b20f1c8166", "dish scrubbing brush-7fb3170352a0d7a8", "plastic tub-bd11bb3088d1260f", "spray bottle1-2ca06a0ae7aaaaeb", "spray bottle2-7f4728d46fd87a35", "food bin bag1-f4530464503504e7", "cloth-5da0b5b8cba1402a", "circular hob attachment-f92958a330d6664f", "yellow tray-3421baccb7e788c4", "fork1-70c218a848520e77", "bowl3-a89c549cbcbb4d73", "spoon3-df3685200281da39" ], "reasoning": "1. I watched the video to observe the user scooping yogurt (pot of yogurt-9fc6a049d25e6d99) into the bowl (blender jar-26d3262312b3562a) sitting on the scale (weighing scales-671472a9d2952e34) from 11:35-11:45.\n2. When the user finishes this, I observed that the scale (blender jar-26d3262312b3562a) says \"62\", at 11:47.\n3. Then, at 11:56, I observed the user press a button on the scale (blender jar-26d3262312b3562a), and the scale again reads \"62\". \n4. The user presses the scale (blender jar-26d3262312b3562a) again at 11:58, and it still reads \"62\". \n5. Next, at 12:00, the user presses the scale (blender jar-26d3262312b3562a) again, and it now reads \"64\". \n6. The user then presses the scale (blender jar-26d3262312b3562a) one more time, and it reads \"63\".\n7. I finished by doing the math to calculate the average - 62 + 62 + 62 + 64 + 63 = 313 / 5 = 62.6.", "referenced_object_masks": [ "blender jar-26d3262312b3562a", "weighing scales-671472a9d2952e34", "pot of yogurt-9fc6a049d25e6d99" ] }, { "key": "P07-20240531-201703:ffd3feb4d493e6050aabc7b50bc735b18d131904", "video_id": "P07-20240531-201703", "question": "If I were cooking at the stove, how would I get to my notepad?", "answer_choice_0": "Turn around and walk around the island", "answer_choice_1": "Turn around and walk forward", "answer_choice_2": "Turn right and walk forward", "answer_choice_3": "Turn left and walk forward", "answer_choice_4": "Sidestep to the left", "answer_id": 2, "answer": "Turn right and walk forward", "question_type": [ "Counterfactual", "Egocentric & Spatial Perception", "Object Recognition" ], "objects_of_interest": [ "phone-42a98d3013724a70", "can of food-d510fa3ec3938c04", "can of food2-7f2eadea9d252077", "bag of lettuce-b8666865c4280bb4", "bag of avocados-95ed02584e8ca125", "sweet potato-a233e39636ff2ee4", "food waste bin-8c45834ce35fb0ab", "scissors-fb29108f71d55dac", "grater-8570e92acf6c0efd", "metal spatula-265c8cf0c1c23fba", "wooden spoons-8d01074246c0dbf5", "chopstick-c084266fc5eae37e", "rubber spatula-629919e513255512", "knife1-bb3986bb7fa0a490", "knife2-6882b6d700d8a2b1", "knife3-9b03c290e4634797", "knife4-0ddcec2f3ea97cc5", "half sweet potato-28b396ecc66ed008", "weighing scales-671472a9d2952e34", "bowl1-345f5d6cc6ae17fa", "pen-35382b78a26652a2", "notebook-3e472d415be7e731", "baking tray-a0b987863741e114", "bottle1-ae10002246bcee88", "bottle of vegetable oil-73ff541c9a2469f9", "packet of cashew nuts-8897418cc85ea1e3", "measuring spoon-967737fd3f1eee47", "green airpods case-c4bb4ecac5cd82d0", "spice jar1-bcc82d5e486c95a2", "spice jar2-a2767262282bb364", "spice jar3-fe04f98f709e2cb3", "spice jar4-e4ebb4df530e87b0", "spice jar5-e60b0620e660437c", "lid of spice jar5-9cdcbbdddcb08952", "lid of spice jar4-d17741237173ab49", "spoon1-51ee55d5babda8f7", "phone2-6477559e162fcc4d", "blender jar-26d3262312b3562a", "blender-3fbfb5f4011dba62", "blender blade-c2ef7d34302ea708", "tray of avocados-ecc906f11957db35", "avocado1-3c750496ef8349bd", "avocado half2-2cc1097671ce2c7c", "pot of yogurt-9fc6a049d25e6d99", "spoon2-0e0c1e59465f09b1", "jar of olives-9498b5928bda152a", "bag of parsley-361f0386fa894b39", "bag of coriander-77c5dd8040ba123d", "bag of coriander2-119134c85e77d94b", "garlic bulb-6d3b451e9d38d517", "bowl2-dc0974501a95d9ea", "lettuce1-062f8055f7a0f620", "lettuce leaves-56b089f6e9afbeba", "lime-c397717c32f919cd", "lid of can of food-8dd8e9e1c80cfacd", "can opener-08c956e403814956", "avocado skin-b7e792ee060bf306", "avocado skin2-8dbb0c02231a65d2", "tea towel-0447401b38ea3638", "chopping board-56d8c949ac927956", "plastic lid1-93f3d52fb43d9853", "plastic lid2-b42d743f9579f171", "carton of milk-e8f65dd4dca2a463", "chopping board2-02c1fe3dc45f8069", "kitchen roll-183c3f0e7d6f384a", "sheet of kitchen roll-20147c2fd1a958fd", "new spoon1-4a5362653d255f9e", "glass1-79078e2b49db66ee", "plastic box1-c30a86ab7b1c215d", "white plastic lid-5efa8ba4b3bee66f", "bottle of washing up liquid-0b8d03b20f1c8166", "dish scrubbing brush-7fb3170352a0d7a8", "plastic tub-bd11bb3088d1260f", "spray bottle1-2ca06a0ae7aaaaeb", "spray bottle2-7f4728d46fd87a35", "food bin bag1-f4530464503504e7", "cloth-5da0b5b8cba1402a", "circular hob attachment-f92958a330d6664f", "yellow tray-3421baccb7e788c4", "fork1-70c218a848520e77", "bowl3-a89c549cbcbb4d73", "spoon3-df3685200281da39" ], "reasoning": "1. I watched the video to locate the stove and the notepad (notebook-3e472d415be7e731).\n2. At 00:07 the user turns to their left and I can see the stove set into the counter up against the wall.\n3. At 00:15 I can see the corner of the stove on the user's left while they look straight ahead at the counter running perpendicular to the stove. I can see the notepad (notebook-3e472d415be7e731) resting on the surface ahead.\n4. From 00:15-00:17, the user walks forward towards the notepad (notebook-3e472d415be7e731) and past the stove.\n5. Therefore, the user would have to turn right 90 degrees and walk forward to reach their notepad (notebook-3e472d415be7e731) from the stove.", "referenced_object_masks": [ "notebook-3e472d415be7e731" ] }, { "key": "P08-20240613-122900:0de1a29852bf3ce595ce6e4c3f3295b0becfed8c", "video_id": "P08-20240613-122900", "question": "Of the items on the counter, which one did I not use in the last minute of the video?", "answer_choice_0": "Espresso machine", "answer_choice_1": "Cup saucer", "answer_choice_2": "Espresso handle", "answer_choice_3": "Coffee grinder cap", "answer_choice_4": "Phone", "answer_id": 3, "answer": "Coffee grinder cap", "question_type": [ "Action Recognition", "Object Recognition", "Event Occurence" ], "objects_of_interest": [ "scale-9a12c2a3d2b5d42f", "small espresso cup-0aa55acdac04a9d6", "can of coffee beans-9e0001e6ba3e6d7b", "lid-e95bec22b5c8708d", "phone-3541555e3fc36ff8", "coffee grinder-31bcf671cbf7a126", "cap of coffee grinder-597beb4b555642d6", "espresso handle-649f8fc3f26c963d", "pot lid-6ca6757d39185fb5", "espresso cup2-cf2b1eccb7050a98", "correct espresso cup2-861578bccfa95b1d", "espresso scoop-d15633d5f98ef8da", "small plates-335ac76791e3c914" ], "reasoning": "1. I watch the final minute of the video to identify objects that the user touches. Since the video is 03:19 minutes long, I watch the span from 02:19-03:19 to identify these objects.\n2. At 02:21, I observe the user grip the espresso handle (espresso handle-649f8fc3f26c963d) with their left hand, eliminating A as an answer choice.\n3. At 02:35, I watch the user brace their right hand on the side of the espresso machine as they use their left hand to turn the espresso handle (espresso handle-649f8fc3f26c963d) into place - this eliminates C as an answer choice.\n4. From 02:37-02:39, I watch the user pick up their phone (phone-3541555e3fc36ff8) from the counter and place it off to the right, out of view - this eliminates B as an answer choice.\n5. At 02:57, I observe the user retrieving a cup saucer (small plates-335ac76791e3c914) from the drawer, and placing it on the countertop, eliminating E as an answer choice.\n6. I watch the rest of the video to ensure that I see no instances of the coffee grinder cap (cap of coffee grinder-597beb4b555642d6) being touched by the user, and the rewatch the final minute of the video to make doubly sure. When I observe no instances of the coffee grinder cap (cap of coffee grinder-597beb4b555642d6) being touched by the user, I select D as my answer.", "referenced_object_masks": [ "cap of coffee grinder-597beb4b555642d6", "small plates-335ac76791e3c914", "espresso handle-649f8fc3f26c963d", "phone-3541555e3fc36ff8" ] }, { "key": "P08-20240613-122900:ad6d65785a0d9389f9424201d2abbfd8c185e289", "video_id": "P08-20240613-122900", "question": "During the sequence in which I turn on the espresso machine, touch a pot lid, grind the espresso beans, pull up my shirtsleeves, and lift the espresso handle, which of these actions would have been performed second if they were considered in reverse chronological order?", "answer_choice_0": "Lift the espresso handle", "answer_choice_1": "Pull up shirtsleeves", "answer_choice_2": "Touch a pot lid", "answer_choice_3": "Turn on the espresso machine", "answer_choice_4": "Grind the espresso beans", "answer_id": 0, "answer": "Lift the espresso handle", "question_type": [ "Action Recognition", "Object Recognition", "Temporal Reasoning", "Counterfactual" ], "objects_of_interest": [ "scale-9a12c2a3d2b5d42f", "small espresso cup-0aa55acdac04a9d6", "can of coffee beans-9e0001e6ba3e6d7b", "lid-e95bec22b5c8708d", "phone-3541555e3fc36ff8", "coffee grinder-31bcf671cbf7a126", "cap of coffee grinder-597beb4b555642d6", "espresso handle-649f8fc3f26c963d", "pot lid-6ca6757d39185fb5", "espresso cup2-cf2b1eccb7050a98", "correct espresso cup2-861578bccfa95b1d", "espresso scoop-d15633d5f98ef8da", "small plates-335ac76791e3c914" ], "reasoning": "1. To find this sequence of actions in the video, I watch it until I identify the first chronological event: this occurs when the user pulls up their shirtsleeves from 01:11-01:12.\n2. I continue watching the video for the second chronological event, which occurs when the user grinds the espresso beans in the coffee grinder (coffee grinder-31bcf671cbf7a126) from 01:19-01:36.\n3. I continue watching for the third chronological event, which occurs when the user turns on the espresso machine at 01:37.\n4. I continue watching for the fourth chronological event, which occurs when the user lifts the espresso handle (espresso handle-649f8fc3f26c963d) at 01:40. \n5. I continue watching for the fifth and final chronological event, which occurs when the user touches the pot lid (pot lid-6ca6757d39185fb5) at 01:43.\n6. Next, I counterfactually imagine that these events were in reverse chronological order, and order them as such. The new order would be: touch a pot lid (pot lid-6ca6757d39185fb5), lift the espresso handle (espresso handle-649f8fc3f26c963d), turn on the espresso machine , grind the espresso beans and pull up shirtsleeves .\n7. Finally, I select the second event - lift the espresso handle (espresso handle-649f8fc3f26c963d) from the new chronology to get my answer, C.", "referenced_object_masks": [ "espresso handle-649f8fc3f26c963d", "pot lid-6ca6757d39185fb5", "coffee grinder-31bcf671cbf7a126" ] }, { "key": "P08-20240613-122900:b546fc404af2404c33e2beefdeeb75a58969788b", "video_id": "P08-20240613-122900", "question": "Which of the following kitchen items - the food scale, the tea kettle, the espresso machine, the kitchen sink, the coffee grinder - do I turn on second in chronological order in the video?", "answer_choice_0": "The coffee grinder", "answer_choice_1": "The tea kettle", "answer_choice_2": "The espresso machine", "answer_choice_3": "The food scale", "answer_choice_4": "The kitchen sink", "answer_id": 0, "answer": "The coffee grinder", "question_type": [ "Action Recognition", "Object Recognition", "State Changes", "Temporal Reasoning" ], "objects_of_interest": [ "scale-9a12c2a3d2b5d42f", "small espresso cup-0aa55acdac04a9d6", "can of coffee beans-9e0001e6ba3e6d7b", "lid-e95bec22b5c8708d", "phone-3541555e3fc36ff8", "coffee grinder-31bcf671cbf7a126", "cap of coffee grinder-597beb4b555642d6", "espresso handle-649f8fc3f26c963d", "pot lid-6ca6757d39185fb5", "espresso cup2-cf2b1eccb7050a98", "correct espresso cup2-861578bccfa95b1d", "espresso scoop-d15633d5f98ef8da", "small plates-335ac76791e3c914" ], "reasoning": "1. I watch the video to identify which of the following items is turned on first chronologically in the video: this occurs when the user turns on the food scale (scale-9a12c2a3d2b5d42f) at 00:17.\n2. I continue watching the video for when the next of the listed items is turned on in the video: this occurs at 01:19, when the user turns on the coffee grinder (coffee grinder-31bcf671cbf7a126).\n3. I continue watching the video to discover if/when the remaining answer choices are turned on by the user in the video. I identify that the user turns on the espresso machine at 01:37, and do not identify any instances in the video where the user turns on the kitchen sink or the tea kettle . Therefore, E is the correct choice.", "referenced_object_masks": [ "scale-9a12c2a3d2b5d42f", "coffee grinder-31bcf671cbf7a126" ] }, { "key": "P08-20240613-122900:d391a83f82c86b63037b4944a47705df7caf856a", "video_id": "P08-20240613-122900", "question": "If the box to the right of the coffee bean container in its original position had two fewer letters in its brand title, how many letters would it have in its brand title?", "answer_choice_0": "6", "answer_choice_1": "8", "answer_choice_2": "4", "answer_choice_3": "5", "answer_choice_4": "7", "answer_id": 0, "answer": "6", "question_type": [ "Egocentric & Spatial Perception", "Counting", "Reading", "Situational Awareness" ], "objects_of_interest": [ "scale-9a12c2a3d2b5d42f", "small espresso cup-0aa55acdac04a9d6", "can of coffee beans-9e0001e6ba3e6d7b", "lid-e95bec22b5c8708d", "phone-3541555e3fc36ff8", "coffee grinder-31bcf671cbf7a126", "cap of coffee grinder-597beb4b555642d6", "espresso handle-649f8fc3f26c963d", "pot lid-6ca6757d39185fb5", "espresso cup2-cf2b1eccb7050a98", "correct espresso cup2-861578bccfa95b1d", "espresso scoop-d15633d5f98ef8da", "small plates-335ac76791e3c914" ], "reasoning": "1. I first identify the original position of the coffee bean container (can of coffee beans-9e0001e6ba3e6d7b) in the video. I notice that it is in the very back left corner of the counter space, on top of a white box with rounded edges .\n2. Next, I identify the box to the right of the can of coffee beans (can of coffee beans-9e0001e6ba3e6d7b), also on top of the white box with rounded edges . This top right box is blue and carton-shaped. I read the words \"TICK TOCK\" in large white lettering on its front, and take this to be the box's brand title.\n3. I count the letters in the box's brand title, \"TICK TOCK\" - there are 8 letters in this title.\n4. Per the question, I subtract 2 from these 8 letters with the equation 8 - 2 = 6, making B, 6, the correct answer.", "referenced_object_masks": [ "can of coffee beans-9e0001e6ba3e6d7b" ] }, { "key": "P08-20240613-122900:eb206010c0df850189971a6ccfd2489040781111", "video_id": "P08-20240613-122900", "question": "What is the product of the number of coffee scoops I placed in the espresso handle from the coffee grinder, and that coffee's weight as most prominently displayed on the food scale?", "answer_choice_0": "20", "answer_choice_1": "12", "answer_choice_2": ", 9", "answer_choice_3": "16", "answer_choice_4": "15", "answer_id": 4, "answer": "15", "question_type": [ "Action Recognition", "Counting", "Reading", "Numerical Reasoning" ], "objects_of_interest": [ "scale-9a12c2a3d2b5d42f", "small espresso cup-0aa55acdac04a9d6", "can of coffee beans-9e0001e6ba3e6d7b", "lid-e95bec22b5c8708d", "phone-3541555e3fc36ff8", "coffee grinder-31bcf671cbf7a126", "cap of coffee grinder-597beb4b555642d6", "espresso handle-649f8fc3f26c963d", "pot lid-6ca6757d39185fb5", "espresso cup2-cf2b1eccb7050a98", "correct espresso cup2-861578bccfa95b1d", "espresso scoop-d15633d5f98ef8da", "small plates-335ac76791e3c914" ], "reasoning": "1. To obtain this answer, I watch the video at the point where I see the coffee grounds weighed on the food scale (scale-9a12c2a3d2b5d42f) from 00:37-01:07 - the number vascillates between 4 and 5, before ultimately settling on 5 for most of the span. This makes 5 the most prominent weight number, and the number I use for the calculation.\n2. At 02:01, I observe the user pick up a small espresso scoop (espresso scoop-d15633d5f98ef8da) from the top of the espresso machine . From 02:08-02:20, I observe the user use the scoop (espresso scoop-d15633d5f98ef8da) to take 3 scoops of coffee grounds from the coffee grinder (coffee grinder-31bcf671cbf7a126) and place them one by one into the espresso handle (espresso handle-649f8fc3f26c963d) before putting the scoop (espresso scoop-d15633d5f98ef8da) back on top of the machine . Thus, I use the number of scoops, 3, as the second part of my calculation.\n3. To obtain the product of these numbers, I do the equation 5 x 3 = 15, to obtain A as the correct answer.", "referenced_object_masks": [ "espresso scoop-d15633d5f98ef8da", "scale-9a12c2a3d2b5d42f", "espresso handle-649f8fc3f26c963d", "coffee grinder-31bcf671cbf7a126" ] }, { "key": "P08-20240613-123547:05493d7058fef7c012d7c2eb4c5fd733641e5b84", "video_id": "P08-20240613-123547", "question": "Which non-leafy ingredients cut up and put into the salad start with the letter that follows the next alphabetical letter of the ingredient in the white paper bag on the counter?", "answer_choice_0": "Cucumber, celery, cauliflower", "answer_choice_1": "There are no ingredients added to the salad that start with a letter following the first letter on the white bag", "answer_choice_2": "Avocado, artichoke, asparagus", "answer_choice_3": "Bell pepper, broccoli, beans", "answer_choice_4": "Cucumber, carrot, cherry tomatoes", "answer_id": 4, "answer": "Cucumber, carrot, cherry tomatoes", "question_type": [ "Action Recognition", "Object Recognition", "Reading", "State Changes" ], "objects_of_interest": [ "phone-c71ae9a3bcec75c1", "tupperware-43b7a5fb6b68d568", "cucumber-6aeea0b4cda07699", "bag of salad seeds-7aee574e1c89b8c1", "carrot-9380ebba453069f7", "tupperware2-11d15d017de38b1e", "chopping board-1e555b10192e39fe", "scale-8b9cdc1b75f2b4d7", "bowl-47b88311dd3a369d", "bag of rocket salad-1c72e084f272b6df", "knife-149014d6d9fd1a8f", "peeler-e532d3a65977c3a3", "objects-a0b649f34e9a7d51", "spoon-6931b465c8c2747a", "bag of coriander-cb010b85ff935057", "red onion-a31f12d436a9b1c4", "lid of tupperware2-fa4b3cd03ce89e67", "tupperware3-dd817903fabed8bd", "olive oil bottle-62f05cee3154cb62", "cap of olive oil-9115efcad32f79f3", "spoon2-f2a2584ba5844c70", "bottle of glaze-6bc7fc629fed0ca8", "salt-2af9f7e3701a6229", "pepper shaker-04a9871cab7bae40", "second bag of baguette-91822670d67badbf", "baguette-be78c6e617ae5bba", "knife2-c757600e32597c3c", "slice of bread-7260a59e27402913", "plate-d82a45d215ad08ce", "cloth-bcd9de4970e0eb2b", "fork-c37aedea914bb33c" ], "reasoning": "1. I look for the first letter of the ingredient in the white paper bag (baguette-be78c6e617ae5bba) on the counter, and I get a good glimpse at 08:03, when the user reaches the ingredients in the bowl underneath the bag (second bag of baguette-91822670d67badbf) in front of it.\n2. On the white bag (baguette-be78c6e617ae5bba) behind this white bag (second bag of baguette-91822670d67badbf), I see an upside down \"B,\" which makes sense since, later, from 15:18-15:23, the user cuts a baguette slice from the white bag (baguette-be78c6e617ae5bba).\n3. The letter that follows \"B\" is \"C\".\n4. I look at the non-leafy ingredients the user cuts and puts into their salad.\n5. From 02:54-03:50, the user cuts up cucumber slices (cucumber-6aeea0b4cda07699) and puts them in the salad.\n6. From 04:08-04:30, the user cuts up cherry tomatoes and puts them in the salad.\n7. From 04:55-06:06, the user peels, cuts, and places a carrot (carrot-9380ebba453069f7) into the salad.\n8. From 08:20-09:18, the user chops, weighs, peels, dices and scrapes red onion (red onion-a31f12d436a9b1c4) into the salad.\n9. All the other ingredients are either leafy or not chopped up.\n10. I realize that the cucumber (cucumber-6aeea0b4cda07699), the carrot (carrot-9380ebba453069f7), and the cherry tomatoes all start with the letter \"C.\"\n11. Therefore, the answer is cucumber (cucumber-6aeea0b4cda07699), carrot (carrot-9380ebba453069f7), and cherry tomatoes .", "referenced_object_masks": [ "red onion-a31f12d436a9b1c4", "second bag of baguette-91822670d67badbf", "carrot-9380ebba453069f7", "baguette-be78c6e617ae5bba", "cucumber-6aeea0b4cda07699" ] }, { "key": "P08-20240613-123547:26595f93e4afd0b6529c7e77d646274509233a70", "video_id": "P08-20240613-123547", "question": "When I scan the bag of greens with the phone app for the first time, where is the bag of seeds in relation to me?", "answer_choice_0": "In the fridge", "answer_choice_1": "By the sink", "answer_choice_2": "To the user's left", "answer_choice_3": "On the cutting board", "answer_choice_4": "To the user's right", "answer_id": 4, "answer": "To the user's right", "question_type": [ "Object Recognition", "Action Recognition", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "phone-c71ae9a3bcec75c1", "tupperware-43b7a5fb6b68d568", "cucumber-6aeea0b4cda07699", "bag of salad seeds-7aee574e1c89b8c1", "carrot-9380ebba453069f7", "tupperware2-11d15d017de38b1e", "chopping board-1e555b10192e39fe", "scale-8b9cdc1b75f2b4d7", "bowl-47b88311dd3a369d", "bag of rocket salad-1c72e084f272b6df", "knife-149014d6d9fd1a8f", "peeler-e532d3a65977c3a3", "objects-a0b649f34e9a7d51", "spoon-6931b465c8c2747a", "bag of coriander-cb010b85ff935057", "red onion-a31f12d436a9b1c4", "lid of tupperware2-fa4b3cd03ce89e67", "tupperware3-dd817903fabed8bd", "olive oil bottle-62f05cee3154cb62", "cap of olive oil-9115efcad32f79f3", "spoon2-f2a2584ba5844c70", "bottle of glaze-6bc7fc629fed0ca8", "salt-2af9f7e3701a6229", "pepper shaker-04a9871cab7bae40", "second bag of baguette-91822670d67badbf", "baguette-be78c6e617ae5bba", "knife2-c757600e32597c3c", "slice of bread-7260a59e27402913", "plate-d82a45d215ad08ce", "cloth-bcd9de4970e0eb2b", "fork-c37aedea914bb33c" ], "reasoning": "1. I locate the timestamp when the user scans the bag of greens (bag of rocket salad-1c72e084f272b6df) with the phone (phone-c71ae9a3bcec75c1) app. This occurs from 01:01-01:04. \n2. The bag of greens (bag of rocket salad-1c72e084f272b6df) is partially laying on top of the cutting board (chopping board-1e555b10192e39fe) in front of the user during this time. \n3. Next, I search for the bag of seeds (bag of salad seeds-7aee574e1c89b8c1) during the same timeframe of 01:01-01:04. I found them to the right of the greens (bag of rocket salad-1c72e084f272b6df) and cutting board (chopping board-1e555b10192e39fe) at 01:01. \n4. If the user is standing, facing the bag of greens (bag of rocket salad-1c72e084f272b6df) partially laying on top of the cutting board (chopping board-1e555b10192e39fe) when they scan the app, the seeds (bag of salad seeds-7aee574e1c89b8c1) are to the user's right.", "referenced_object_masks": [ "bag of rocket salad-1c72e084f272b6df", "bag of salad seeds-7aee574e1c89b8c1", "chopping board-1e555b10192e39fe", "phone-c71ae9a3bcec75c1" ] }, { "key": "P08-20240613-123547:378492068393bbf2d98fb00b09b05d8b7f686b42", "video_id": "P08-20240613-123547", "question": "Before I weigh the greens for a final time, how many times do I reach into the plastic bag of greens?", "answer_choice_0": "5", "answer_choice_1": "3", "answer_choice_2": "2", "answer_choice_3": "6", "answer_choice_4": "4", "answer_id": 0, "answer": "5", "question_type": [ "Counting", "Action Recognition", "Object Recognition" ], "objects_of_interest": [ "phone-c71ae9a3bcec75c1", "tupperware-43b7a5fb6b68d568", "cucumber-6aeea0b4cda07699", "bag of salad seeds-7aee574e1c89b8c1", "carrot-9380ebba453069f7", "tupperware2-11d15d017de38b1e", "chopping board-1e555b10192e39fe", "scale-8b9cdc1b75f2b4d7", "bowl-47b88311dd3a369d", "bag of rocket salad-1c72e084f272b6df", "knife-149014d6d9fd1a8f", "peeler-e532d3a65977c3a3", "objects-a0b649f34e9a7d51", "spoon-6931b465c8c2747a", "bag of coriander-cb010b85ff935057", "red onion-a31f12d436a9b1c4", "lid of tupperware2-fa4b3cd03ce89e67", "tupperware3-dd817903fabed8bd", "olive oil bottle-62f05cee3154cb62", "cap of olive oil-9115efcad32f79f3", "spoon2-f2a2584ba5844c70", "bottle of glaze-6bc7fc629fed0ca8", "salt-2af9f7e3701a6229", "pepper shaker-04a9871cab7bae40", "second bag of baguette-91822670d67badbf", "baguette-be78c6e617ae5bba", "knife2-c757600e32597c3c", "slice of bread-7260a59e27402913", "plate-d82a45d215ad08ce", "cloth-bcd9de4970e0eb2b", "fork-c37aedea914bb33c" ], "reasoning": "1. I determine that the greens (bag of rocket salad-1c72e084f272b6df) are weighed for a final time at 01:50. \n2. Next, I rewind the video to determine the number of times the user reaches into the bag of greens (bag of rocket salad-1c72e084f272b6df). \n3. I saw the user grab a handful of greens (bag of rocket salad-1c72e084f272b6df) at 1:16.\n4. I saw the user place the handful back in the bag (bag of rocket salad-1c72e084f272b6df) at 1:17. This is counted as the second time the user reaches into the bag (bag of rocket salad-1c72e084f272b6df).\n5. I saw the user reach into the bag (bag of rocket salad-1c72e084f272b6df) a third time at 1:19.\n6. Finally, I saw the user grab another handful of greens (bag of rocket salad-1c72e084f272b6df) at 1:26 and 1:30 for a total of 5 times.", "referenced_object_masks": [ "bag of rocket salad-1c72e084f272b6df" ] }, { "key": "P08-20240613-123547:598d5c31874b01e83342c6ab040684162400a052", "video_id": "P08-20240613-123547", "question": "After I vertically slice the cucumber into 4 equal slices, how many total bits of cucumber do I dice from the second and fourth slice?", "answer_choice_0": "78", "answer_choice_1": "37", "answer_choice_2": "45", "answer_choice_3": "60", "answer_choice_4": "20", "answer_id": 3, "answer": "60", "question_type": [ "Action Recognition", "Numerical Reasoning", "Counting", "Object Recognition", "State Changes" ], "objects_of_interest": [ "phone-c71ae9a3bcec75c1", "tupperware-43b7a5fb6b68d568", "cucumber-6aeea0b4cda07699", "bag of salad seeds-7aee574e1c89b8c1", "carrot-9380ebba453069f7", "tupperware2-11d15d017de38b1e", "chopping board-1e555b10192e39fe", "scale-8b9cdc1b75f2b4d7", "bowl-47b88311dd3a369d", "bag of rocket salad-1c72e084f272b6df", "knife-149014d6d9fd1a8f", "peeler-e532d3a65977c3a3", "objects-a0b649f34e9a7d51", "spoon-6931b465c8c2747a", "bag of coriander-cb010b85ff935057", "red onion-a31f12d436a9b1c4", "lid of tupperware2-fa4b3cd03ce89e67", "tupperware3-dd817903fabed8bd", "olive oil bottle-62f05cee3154cb62", "cap of olive oil-9115efcad32f79f3", "spoon2-f2a2584ba5844c70", "bottle of glaze-6bc7fc629fed0ca8", "salt-2af9f7e3701a6229", "pepper shaker-04a9871cab7bae40", "second bag of baguette-91822670d67badbf", "baguette-be78c6e617ae5bba", "knife2-c757600e32597c3c", "slice of bread-7260a59e27402913", "plate-d82a45d215ad08ce", "cloth-bcd9de4970e0eb2b", "fork-c37aedea914bb33c" ], "reasoning": "1. I look for the time the user cuts a cucumber (cucumber-6aeea0b4cda07699) into 4 slices, and I find the time range from 02:54-03:45.\n2. Within this time range, I locate the time the user cuts the second and fourth slice. I find the time for the second from 03:11-03:19, and I find the time for the fourth from 03:38-03:45.\n3. From 03:11-03:19, I notice that the user makes 3 cuts along the length of the cucumber (cucumber-6aeea0b4cda07699), and from 03:15-03:19, I notice that the user makes 6 cuts along the width of the cucumber (cucumber-6aeea0b4cda07699).\n4. Since this would create a diced cucumber slice (cucumber-6aeea0b4cda07699) with 4 lengthwise rows and 7 widthwise columns, the user would have created 28 diced bits of cucumber (cucumber-6aeea0b4cda07699).\n5. From 03:38-03:40, I notice the user make 3 cut along the length of the cucumber slice (cucumber-6aeea0b4cda07699) and, from 03:40 -03:45, I notice the user make 7 cuts along the width of the cucumber (cucumber-6aeea0b4cda07699).\n6. This creates a cucumber slice (cucumber-6aeea0b4cda07699) with 4 lengthwise rows and 8 widthwise columns, which would make a total of 32 diced bits of cucumber (cucumber-6aeea0b4cda07699).\n7. I add 28+32+60.", "referenced_object_masks": [ "cucumber-6aeea0b4cda07699" ] }, { "key": "P08-20240613-123547:a72745c5336beb9b6b62fa551e8fbcf34169ac00", "video_id": "P08-20240613-123547", "question": "During which time in the operation of my phone do I do press on the screen with both my right and my left thumb?", "answer_choice_0": "Fifth time, when I am inputting information about the tomatoes", "answer_choice_1": "Third time, when I am inputting salad mix information", "answer_choice_2": "Fourth time, when I am scanning and inputting information for the cucumber", "answer_choice_3": "Second time, when I am scanning the salad mix", "answer_choice_4": "First time, when I am setting up the app", "answer_id": 2, "answer": "Fourth time, when I am scanning and inputting information for the cucumber", "question_type": [ "Egocentric & Spatial Perception", "Temporal Reasoning", "Action Recognition", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "phone-c71ae9a3bcec75c1", "tupperware-43b7a5fb6b68d568", "cucumber-6aeea0b4cda07699", "bag of salad seeds-7aee574e1c89b8c1", "carrot-9380ebba453069f7", "tupperware2-11d15d017de38b1e", "chopping board-1e555b10192e39fe", "scale-8b9cdc1b75f2b4d7", "bowl-47b88311dd3a369d", "bag of rocket salad-1c72e084f272b6df", "knife-149014d6d9fd1a8f", "peeler-e532d3a65977c3a3", "objects-a0b649f34e9a7d51", "spoon-6931b465c8c2747a", "bag of coriander-cb010b85ff935057", "red onion-a31f12d436a9b1c4", "lid of tupperware2-fa4b3cd03ce89e67", "tupperware3-dd817903fabed8bd", "olive oil bottle-62f05cee3154cb62", "cap of olive oil-9115efcad32f79f3", "spoon2-f2a2584ba5844c70", "bottle of glaze-6bc7fc629fed0ca8", "salt-2af9f7e3701a6229", "pepper shaker-04a9871cab7bae40", "second bag of baguette-91822670d67badbf", "baguette-be78c6e617ae5bba", "knife2-c757600e32597c3c", "slice of bread-7260a59e27402913", "plate-d82a45d215ad08ce", "cloth-bcd9de4970e0eb2b", "fork-c37aedea914bb33c" ], "reasoning": "1. I look for the time the user operates their phone (phone-c71ae9a3bcec75c1), pressing on their screen with both their right and their left thumbs.\n2. I find many moments of the user operating their phone (phone-c71ae9a3bcec75c1): 00:01-00:08, 00:55-01:09, 01:38-01:51, 01:54-02:26, 02:46-02:53, 03:54-04:06, 04:35-04:46, 04:48-04:51, 06:10-06:32, 06:45-06:57, 07:02-07:30, 07:36-07:45, 08:05-08:20, 08:30-08:33, 09:23-09:37, 10:09-10:28, 11:12-11:45, 11:48-12:11, 12:22-12:38, 12:44-12:54, 13:01-13:09, 13:14-13:22, 13:55-14:00, 14:51-15:14, and 15:32-15:46.\n3. I realize that from 01:54-02:26 is the only time the user operates their phone (phone-c71ae9a3bcec75c1) and presses on the screen with their right and left thumb.\n4. I examine the context surrounding this time range and realize that it is in the middle of scanning and inputting information for the cucumber (cucumber-6aeea0b4cda07699).\n5. Therefore, the fourth time is my answer.", "referenced_object_masks": [ "cucumber-6aeea0b4cda07699", "phone-c71ae9a3bcec75c1" ] }, { "key": "P08-20240613-123547:ab97717846b2630bf74a2f5438990c0c0e0b54fc", "video_id": "P08-20240613-123547", "question": "What is the difference between how I put the cutting board in the sink the first time, and how I put the cutting board in the sink the second time?", "answer_choice_0": "First time, its top face slanted right; second time, its top face slanted left", "answer_choice_1": "First time, its top face slanting toward the wall; second time, its top face slanting toward me", "answer_choice_2": "I only put the cutting board in the sink once", "answer_choice_3": "First time, upside down; second time, right side up", "answer_choice_4": "First time, lengthwise; second time, widthwise", "answer_id": 4, "answer": "First time, lengthwise; second time, widthwise", "question_type": [ "Object Recognition", "Situational Awareness", "State Changes", "Action Recognition", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "phone-c71ae9a3bcec75c1", "tupperware-43b7a5fb6b68d568", "cucumber-6aeea0b4cda07699", "bag of salad seeds-7aee574e1c89b8c1", "carrot-9380ebba453069f7", "tupperware2-11d15d017de38b1e", "chopping board-1e555b10192e39fe", "scale-8b9cdc1b75f2b4d7", "bowl-47b88311dd3a369d", "bag of rocket salad-1c72e084f272b6df", "knife-149014d6d9fd1a8f", "peeler-e532d3a65977c3a3", "objects-a0b649f34e9a7d51", "spoon-6931b465c8c2747a", "bag of coriander-cb010b85ff935057", "red onion-a31f12d436a9b1c4", "lid of tupperware2-fa4b3cd03ce89e67", "tupperware3-dd817903fabed8bd", "olive oil bottle-62f05cee3154cb62", "cap of olive oil-9115efcad32f79f3", "spoon2-f2a2584ba5844c70", "bottle of glaze-6bc7fc629fed0ca8", "salt-2af9f7e3701a6229", "pepper shaker-04a9871cab7bae40", "second bag of baguette-91822670d67badbf", "baguette-be78c6e617ae5bba", "knife2-c757600e32597c3c", "slice of bread-7260a59e27402913", "plate-d82a45d215ad08ce", "cloth-bcd9de4970e0eb2b", "fork-c37aedea914bb33c" ], "reasoning": "1. I look for the first time I put the cutting board (cucumber-6aeea0b4cda07699) into the sink, and I find it at 14:43.\n2. I notice that the user puts the cutting board (cucumber-6aeea0b4cda07699) in lengthwise (with the narrower side facing the user).\n3. I notice that the user puts the cutting board (cucumber-6aeea0b4cda07699) in the sink the second time at 16:15.\n4. At this timestamp, I notice that the user puts the cutting board (cucumber-6aeea0b4cda07699) in widthwise (with the longer side facing the user).\n5. Therefore, the difference is that the first time the user put the cutting board (cucumber-6aeea0b4cda07699) in lengthwise, and the second time the user put the cutting board (cucumber-6aeea0b4cda07699) in widthwise.", "referenced_object_masks": [ "cucumber-6aeea0b4cda07699" ] }, { "key": "P08-20240613-123547:d9d137afcd286733f840d5edf2148ffc20880c19", "video_id": "P08-20240613-123547", "question": "Between the time the first half of the tomatoes are put on the cutting board and the time I add a new ingredient into the salad, how long are there exactly half of the original amount of tomatoes in the plastic container of tomatoes?", "answer_choice_0": "31 seconds", "answer_choice_1": "56 seconds", "answer_choice_2": "65 seconds", "answer_choice_3": "20 seconds", "answer_choice_4": "43 seconds", "answer_id": 4, "answer": "43 seconds", "question_type": [ "Action Recognition", "Numerical Reasoning", "Counting", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "phone-c71ae9a3bcec75c1", "tupperware-43b7a5fb6b68d568", "cucumber-6aeea0b4cda07699", "bag of salad seeds-7aee574e1c89b8c1", "carrot-9380ebba453069f7", "tupperware2-11d15d017de38b1e", "chopping board-1e555b10192e39fe", "scale-8b9cdc1b75f2b4d7", "bowl-47b88311dd3a369d", "bag of rocket salad-1c72e084f272b6df", "knife-149014d6d9fd1a8f", "peeler-e532d3a65977c3a3", "objects-a0b649f34e9a7d51", "spoon-6931b465c8c2747a", "bag of coriander-cb010b85ff935057", "red onion-a31f12d436a9b1c4", "lid of tupperware2-fa4b3cd03ce89e67", "tupperware3-dd817903fabed8bd", "olive oil bottle-62f05cee3154cb62", "cap of olive oil-9115efcad32f79f3", "spoon2-f2a2584ba5844c70", "bottle of glaze-6bc7fc629fed0ca8", "salt-2af9f7e3701a6229", "pepper shaker-04a9871cab7bae40", "second bag of baguette-91822670d67badbf", "baguette-be78c6e617ae5bba", "knife2-c757600e32597c3c", "slice of bread-7260a59e27402913", "plate-d82a45d215ad08ce", "cloth-bcd9de4970e0eb2b", "fork-c37aedea914bb33c" ], "reasoning": "1. First, I look for the time when the first half of the tomatoes are put on the cutting board (chopping board-1e555b10192e39fe), and I find this moment from 03:54.\n2. At this moment, I notice 3 tomatoes in the plastic container (tupperware-43b7a5fb6b68d568) of tomatoes, and 3 tomatoes on the cutting board (chopping board-1e555b10192e39fe). I start counting the time from 03:54 onward.\n3. At 04:05, the user pulls another tomato out of the container (tupperware-43b7a5fb6b68d568), making the number of tomatoes less than half.\n4. At 4:32 the user puts a stray tomato back into the plastic container (tupperware-43b7a5fb6b68d568), bringing the number of tomatoes back up to half again. I count the time from 04:32 onward.\n5. At 06:04, the user adds diced carrots (carrot-9380ebba453069f7) to the salad, a new ingredient, so I stop counting.\n6. I realize that there were exactly half of the original amount of tomatoes in the plastic container (tupperware-43b7a5fb6b68d568) from 03:54-04:05, and 04:32-06:04. This makes a total of 43 seconds.\n7. Therefore, the answer is 43 seconds.", "referenced_object_masks": [ "carrot-9380ebba453069f7", "chopping board-1e555b10192e39fe", "tupperware-43b7a5fb6b68d568" ] }, { "key": "P08-20240613-123547:f17380e2acac8ee068a7427e4dee3f79b54a6037", "video_id": "P08-20240613-123547", "question": "What is the difference between the number shown on the scale when it is finished measuring the bowl of lettuce and the number of times I slice through a cherry tomato?", "answer_choice_0": "9", "answer_choice_1": "15", "answer_choice_2": "24", "answer_choice_3": "27", "answer_choice_4": "33", "answer_id": 1, "answer": "15", "question_type": [ "Object Recognition", "Numerical Reasoning", "State Changes" ], "objects_of_interest": [ "phone-c71ae9a3bcec75c1", "tupperware-43b7a5fb6b68d568", "cucumber-6aeea0b4cda07699", "bag of salad seeds-7aee574e1c89b8c1", "carrot-9380ebba453069f7", "tupperware2-11d15d017de38b1e", "chopping board-1e555b10192e39fe", "scale-8b9cdc1b75f2b4d7", "bowl-47b88311dd3a369d", "bag of rocket salad-1c72e084f272b6df", "knife-149014d6d9fd1a8f", "peeler-e532d3a65977c3a3", "objects-a0b649f34e9a7d51", "spoon-6931b465c8c2747a", "bag of coriander-cb010b85ff935057", "red onion-a31f12d436a9b1c4", "lid of tupperware2-fa4b3cd03ce89e67", "tupperware3-dd817903fabed8bd", "olive oil bottle-62f05cee3154cb62", "cap of olive oil-9115efcad32f79f3", "spoon2-f2a2584ba5844c70", "bottle of glaze-6bc7fc629fed0ca8", "salt-2af9f7e3701a6229", "pepper shaker-04a9871cab7bae40", "second bag of baguette-91822670d67badbf", "baguette-be78c6e617ae5bba", "knife2-c757600e32597c3c", "slice of bread-7260a59e27402913", "plate-d82a45d215ad08ce", "cloth-bcd9de4970e0eb2b", "fork-c37aedea914bb33c" ], "reasoning": "1. I observe the user add lettuce from the bag (bag of rocket salad-1c72e084f272b6df) to the bowl (bowl-47b88311dd3a369d) on top of the scale (scale-8b9cdc1b75f2b4d7) from 01:26 to 01:37.\n2. I observe the scale (scale-8b9cdc1b75f2b4d7) finish measuring the lettuce in the bowl (bowl-47b88311dd3a369d) at 01:40, showing the number \"24\" up until the user removes the bowl (bowl-47b88311dd3a369d) at 01:50.\n3. I observe the user cutting the cherry tomatoes on the chopping board (chopping board-1e555b10192e39fe) from 04:08 to 04:25.\n4. I count the user slice through the cherry tomatoes 9 times from 04:08 to 04:25.\n5. I subtract 9 from 24 to get 15.", "referenced_object_masks": [ "scale-8b9cdc1b75f2b4d7", "bowl-47b88311dd3a369d", "chopping board-1e555b10192e39fe", "bag of rocket salad-1c72e084f272b6df" ] }, { "key": "P08-20240613-123547:f7583697c46102b618ffea2163a62d393fc89534", "video_id": "P08-20240613-123547", "question": "How many times did I cut the cucumber while standing it up vertically?", "answer_choice_0": "5", "answer_choice_1": "2", "answer_choice_2": "4", "answer_choice_3": "3", "answer_choice_4": "6", "answer_id": 3, "answer": "3", "question_type": [ "Counting", "Action Recognition", "Object Recognition" ], "objects_of_interest": [ "phone-c71ae9a3bcec75c1", "tupperware-43b7a5fb6b68d568", "cucumber-6aeea0b4cda07699", "bag of salad seeds-7aee574e1c89b8c1", "carrot-9380ebba453069f7", "tupperware2-11d15d017de38b1e", "chopping board-1e555b10192e39fe", "scale-8b9cdc1b75f2b4d7", "bowl-47b88311dd3a369d", "bag of rocket salad-1c72e084f272b6df", "knife-149014d6d9fd1a8f", "peeler-e532d3a65977c3a3", "objects-a0b649f34e9a7d51", "spoon-6931b465c8c2747a", "bag of coriander-cb010b85ff935057", "red onion-a31f12d436a9b1c4", "lid of tupperware2-fa4b3cd03ce89e67", "tupperware3-dd817903fabed8bd", "olive oil bottle-62f05cee3154cb62", "cap of olive oil-9115efcad32f79f3", "spoon2-f2a2584ba5844c70", "bottle of glaze-6bc7fc629fed0ca8", "salt-2af9f7e3701a6229", "pepper shaker-04a9871cab7bae40", "second bag of baguette-91822670d67badbf", "baguette-be78c6e617ae5bba", "knife2-c757600e32597c3c", "slice of bread-7260a59e27402913", "plate-d82a45d215ad08ce", "cloth-bcd9de4970e0eb2b", "fork-c37aedea914bb33c" ], "reasoning": "1. I watched for the user to add cucumber to a bowl and noted it occurring for the first time at 03:10 and the second at 03:20.\n2. I then looked for instances of the user cutting cucumber vertically before 03:20.\n3. I saw the user stand a piece of cucumber on its end and bring the knife (knife-149014d6d9fd1a8f) closer towards it at 02:54.\n4. I saw the user cut the cucumber vertically 3 times from 02:55-02:59.\n5. I saw the user continue cutting cucumber, but did not see them do so with the cucumber standing vertically again before 03:20. Therefore the total is 3.", "referenced_object_masks": [ "knife-149014d6d9fd1a8f" ] }, { "key": "P08-20240613-123547:ffb3dc438fadd230ada39b5e9b582e6ec6918016", "video_id": "P08-20240613-123547", "question": "When I take out 6 items from the fridge in a row, which item do I retrieve last?", "answer_choice_0": "Container of tomatoes", "answer_choice_1": "Cucumber", "answer_choice_2": "Bag of nuts", "answer_choice_3": "Carrot", "answer_choice_4": "Bag of greens", "answer_id": 3, "answer": "Carrot", "question_type": [ "Object Recognition", "Numerical Reasoning", "Action Recognition" ], "objects_of_interest": [ "phone-c71ae9a3bcec75c1", "tupperware-43b7a5fb6b68d568", "cucumber-6aeea0b4cda07699", "bag of salad seeds-7aee574e1c89b8c1", "carrot-9380ebba453069f7", "tupperware2-11d15d017de38b1e", "chopping board-1e555b10192e39fe", "scale-8b9cdc1b75f2b4d7", "bowl-47b88311dd3a369d", "bag of rocket salad-1c72e084f272b6df", "knife-149014d6d9fd1a8f", "peeler-e532d3a65977c3a3", "objects-a0b649f34e9a7d51", "spoon-6931b465c8c2747a", "bag of coriander-cb010b85ff935057", "red onion-a31f12d436a9b1c4", "lid of tupperware2-fa4b3cd03ce89e67", "tupperware3-dd817903fabed8bd", "olive oil bottle-62f05cee3154cb62", "cap of olive oil-9115efcad32f79f3", "spoon2-f2a2584ba5844c70", "bottle of glaze-6bc7fc629fed0ca8", "salt-2af9f7e3701a6229", "pepper shaker-04a9871cab7bae40", "second bag of baguette-91822670d67badbf", "baguette-be78c6e617ae5bba", "knife2-c757600e32597c3c", "slice of bread-7260a59e27402913", "plate-d82a45d215ad08ce", "cloth-bcd9de4970e0eb2b", "fork-c37aedea914bb33c" ], "reasoning": "1. I saw the user open the fridge door at 00:11.\n3. The user first pulls out a bag of greens (bag of rocket salad-1c72e084f272b6df) at 00:14, second, a container of tomatoes (tupperware-43b7a5fb6b68d568) at 00:17, third, 1 cucumber (cucumber-6aeea0b4cda07699) at 00:20, fourth, a bag of herbs (bag of coriander-cb010b85ff935057) at 00:22, fifth, a bag of nuts (bag of salad seeds-7aee574e1c89b8c1) at 00:25, and finally, 1 carrot (carrot-9380ebba453069f7) at 00:28.\n4. Therefore, the last item the user retreived was a carrot (carrot-9380ebba453069f7).", "referenced_object_masks": [ "bag of coriander-cb010b85ff935057", "cucumber-6aeea0b4cda07699", "tupperware-43b7a5fb6b68d568", "carrot-9380ebba453069f7", "bag of rocket salad-1c72e084f272b6df", "bag of salad seeds-7aee574e1c89b8c1" ] }, { "key": "P08-20240614-085000:282b8ee998c7eeab55cd79f1dca65dffc5446a02", "video_id": "P08-20240614-085000", "question": "Where is the refrigerator in relation to me when I am at the stove?", "answer_choice_0": "To my right", "answer_choice_1": "Behind me to my left", "answer_choice_2": "Behind me to my right", "answer_choice_3": "To my left", "answer_choice_4": "In front of me to my right", "answer_id": 1, "answer": "Behind me to my left", "question_type": [ "Action Recognition", "Numerical Reasoning", "Object Recognition", "Egocentric & Spatial Perception", "Temporal Reasoning" ], "objects_of_interest": [ "phone-4ee5418723871673", "small white packet-6000e1e21a78231f", "carton of eggs-9e8bcce2d31af231", "tub of cottage cheese-c745fbd9b1d70f77", "bag of coriander-e3f7e72ac7151ef5", "tray of vegetables-a8d031902952ec58", "plastic box with green chillies-598f2394ec50c47b", "carton of eggs and tub of cottage cheese-0ad1fa13e2bca0d7", "back of lettuce-e129240b53be9003", "small pan-3a286c4233af61b7", "white chopping board-8bcfa5bd1bb0cbee", "knife-5792048276490863", "potato-e04555138e7758e7", "red onion-82a5e12bacbcac20", "weighing scale-0293338bb74fd4ee", "medium knife-19cede612034dbf3", "big knife-bd25e124e25fd56e", "left half of the onion-e5db8e6e7130388f", "right half of the onion-74bfbe726dbea0ed", "layers of onion-d6f48a6dfff95858", "olive oil spray-7d35a1020309e24a", "gray bowl-0e4002cb88aeaee7", "white bowl-8a825276b568e15d", "first egg-70403ff6fa78d64d", "second egg-ff85c39f26558a8b", "tea towel-cb179dea3d39f7d5", "spatula-6c9052df67058bdf", "second spatula-f54d097b337b5baa", "fork-0a72cae135bf2eba", "second fork-3f70b18ac231034f", "eggshells-ae798305e2485513", "grey bowl-f6620b0886e36eea", "grey plate-b4b38dfc25baecbd", "top of cottage cheese tub-24330c3b20ad2bf7", "small plate 1-7d6cbfe6e2ca03d1", "small plate 2-6786f79551369dc8", "spoon-c231bb45b9eba87c", "olive oil-6c8e698760dcbf37", "olive oil bottle cap-bf8e0237ebfaa030", "pepper shaker-afd555a3d2affb9d", "wide glass-478b41989e0cb25e", "salt container-c2f26d875c760b7d", "spice container-d00f4dea11f43d04", "sponge-005a6818405d5faf", "dish soap-b588bf5326d065c7", "second sponge-8b70c6acf6cda2da", "cutlery-4d077c65db7d5cef", "steel scrubber-e580cc55a651b8b0", "set of knives-dda6a8f4e769ebfb", "bag of rocket salad-f7ee935423187fa4", "bowl of vegetables-05f704459ca84ded", "kitchen towels-847b3ed68cfdd6ce" ], "reasoning": "1. I watched the whole video and saw the user at the stove at 00:06.\n2. At 00:09 the user turns left about 180 degrees and walks straight to the refrigerator .\n3. Therefore, the refrigerator is behind the user to their left.", "referenced_object_masks": [] }, { "key": "P08-20240614-085000:2c56e1e4a9e790ca7ec8d3205ee24055f2a0b6df", "video_id": "P08-20240614-085000", "question": "Rounded to the nearest tenth, if I take the weight of the cottage cheese, multiply it by the number of mugs on the mug tree by the white cutting board and divide it by the sum of the number of twists of the salt and pepper grinders, what number do I get?", "answer_choice_0": "30.8", "answer_choice_1": "31.9", "answer_choice_2": "32", "answer_choice_3": "32.8", "answer_choice_4": "33", "answer_id": 3, "answer": "32.8", "question_type": [ "Action Recognition", "Counting", "Event Occurence", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "phone-4ee5418723871673", "small white packet-6000e1e21a78231f", "carton of eggs-9e8bcce2d31af231", "tub of cottage cheese-c745fbd9b1d70f77", "bag of coriander-e3f7e72ac7151ef5", "tray of vegetables-a8d031902952ec58", "plastic box with green chillies-598f2394ec50c47b", "carton of eggs and tub of cottage cheese-0ad1fa13e2bca0d7", "back of lettuce-e129240b53be9003", "small pan-3a286c4233af61b7", "white chopping board-8bcfa5bd1bb0cbee", "knife-5792048276490863", "potato-e04555138e7758e7", "red onion-82a5e12bacbcac20", "weighing scale-0293338bb74fd4ee", "medium knife-19cede612034dbf3", "big knife-bd25e124e25fd56e", "left half of the onion-e5db8e6e7130388f", "right half of the onion-74bfbe726dbea0ed", "layers of onion-d6f48a6dfff95858", "olive oil spray-7d35a1020309e24a", "gray bowl-0e4002cb88aeaee7", "white bowl-8a825276b568e15d", "first egg-70403ff6fa78d64d", "second egg-ff85c39f26558a8b", "tea towel-cb179dea3d39f7d5", "spatula-6c9052df67058bdf", "second spatula-f54d097b337b5baa", "fork-0a72cae135bf2eba", "second fork-3f70b18ac231034f", "eggshells-ae798305e2485513", "grey bowl-f6620b0886e36eea", "grey plate-b4b38dfc25baecbd", "top of cottage cheese tub-24330c3b20ad2bf7", "small plate 1-7d6cbfe6e2ca03d1", "small plate 2-6786f79551369dc8", "spoon-c231bb45b9eba87c", "olive oil-6c8e698760dcbf37", "olive oil bottle cap-bf8e0237ebfaa030", "pepper shaker-afd555a3d2affb9d", "wide glass-478b41989e0cb25e", "salt container-c2f26d875c760b7d", "spice container-d00f4dea11f43d04", "sponge-005a6818405d5faf", "dish soap-b588bf5326d065c7", "second sponge-8b70c6acf6cda2da", "cutlery-4d077c65db7d5cef", "steel scrubber-e580cc55a651b8b0", "set of knives-dda6a8f4e769ebfb", "bag of rocket salad-f7ee935423187fa4", "bowl of vegetables-05f704459ca84ded", "kitchen towels-847b3ed68cfdd6ce" ], "reasoning": "1. I watched the whole video and saw the user weigh the cottage cheese (tub of cottage cheese-c745fbd9b1d70f77) at 09:08.\n2. It weighed 59 g.\n3. At 07:16, the number of mugs on the mug tree by the cutting board (white chopping board-8bcfa5bd1bb0cbee) is 5.\n4. At 14:14, the user does 4 twists of the salt grinder (salt container-c2f26d875c760b7d) and 5 twists of the pepper grinder (pepper shaker-afd555a3d2affb9d).\n5. 5 plus 4 is 9.\n6. 59 * 5 is 295.\n7. 295 divided by 9 is 32.77778.\n8. Rounded to the nearest tenth is 32.8.", "referenced_object_masks": [ "salt container-c2f26d875c760b7d", "pepper shaker-afd555a3d2affb9d", "white chopping board-8bcfa5bd1bb0cbee", "tub of cottage cheese-c745fbd9b1d70f77" ] }, { "key": "P08-20240614-085000:7c6f49c631705252a74d89901e809c85d646fee1", "video_id": "P08-20240614-085000", "question": "What item did I grab while visiting the refrigerator for the second time?", "answer_choice_0": "A carton of eggs", "answer_choice_1": "A bag of coriander", "answer_choice_2": "A box of chili peppers", "answer_choice_3": "A bag of lettuce", "answer_choice_4": "A tub of cottage cheese", "answer_id": 3, "answer": "A bag of lettuce", "question_type": [ "Situational Awareness", "Counting", "Action Recognition", "Egocentric & Spatial Perception", "Object Recognition", "State Changes", "Temporal Reasoning" ], "objects_of_interest": [ "phone-4ee5418723871673", "small white packet-6000e1e21a78231f", "carton of eggs-9e8bcce2d31af231", "tub of cottage cheese-c745fbd9b1d70f77", "bag of coriander-e3f7e72ac7151ef5", "tray of vegetables-a8d031902952ec58", "plastic box with green chillies-598f2394ec50c47b", "carton of eggs and tub of cottage cheese-0ad1fa13e2bca0d7", "back of lettuce-e129240b53be9003", "small pan-3a286c4233af61b7", "white chopping board-8bcfa5bd1bb0cbee", "knife-5792048276490863", "potato-e04555138e7758e7", "red onion-82a5e12bacbcac20", "weighing scale-0293338bb74fd4ee", "medium knife-19cede612034dbf3", "big knife-bd25e124e25fd56e", "left half of the onion-e5db8e6e7130388f", "right half of the onion-74bfbe726dbea0ed", "layers of onion-d6f48a6dfff95858", "olive oil spray-7d35a1020309e24a", "gray bowl-0e4002cb88aeaee7", "white bowl-8a825276b568e15d", "first egg-70403ff6fa78d64d", "second egg-ff85c39f26558a8b", "tea towel-cb179dea3d39f7d5", "spatula-6c9052df67058bdf", "second spatula-f54d097b337b5baa", "fork-0a72cae135bf2eba", "second fork-3f70b18ac231034f", "eggshells-ae798305e2485513", "grey bowl-f6620b0886e36eea", "grey plate-b4b38dfc25baecbd", "top of cottage cheese tub-24330c3b20ad2bf7", "small plate 1-7d6cbfe6e2ca03d1", "small plate 2-6786f79551369dc8", "spoon-c231bb45b9eba87c", "olive oil-6c8e698760dcbf37", "olive oil bottle cap-bf8e0237ebfaa030", "pepper shaker-afd555a3d2affb9d", "wide glass-478b41989e0cb25e", "salt container-c2f26d875c760b7d", "spice container-d00f4dea11f43d04", "sponge-005a6818405d5faf", "dish soap-b588bf5326d065c7", "second sponge-8b70c6acf6cda2da", "cutlery-4d077c65db7d5cef", "steel scrubber-e580cc55a651b8b0", "set of knives-dda6a8f4e769ebfb", "bag of rocket salad-f7ee935423187fa4", "bowl of vegetables-05f704459ca84ded", "kitchen towels-847b3ed68cfdd6ce" ], "reasoning": "1. I watched the video and found the refrigerator at 00:08.\n2. I counted each time the user visited the refrigerator.\n3. I saw the user visit the refrigerator for the first time at 00:09.\n4. I saw the user depart from the refrigerator for the first time at 00:27.\n5. I saw the user visit the refrigerator for the second time at 00:30.\n6. I saw the user grab a bag of lettuce (back of lettuce-e129240b53be9003) from the refrigerator at 00:34.\n7. Therefore, the user grabbed a bag of lettuce (back of lettuce-e129240b53be9003) after visiting the refrigerator for the second time.", "referenced_object_masks": [ "back of lettuce-e129240b53be9003" ] }, { "key": "P08-20240614-085000:aa43ea3f0d44f8c8af132c626d1297fd71c907dc", "video_id": "P08-20240614-085000", "question": "From where I am using the cutting board at 10:12, how many degrees do I have to turn to be at the sink?", "answer_choice_0": "45 degrees", "answer_choice_1": "60 degrees", "answer_choice_2": "90 degrees", "answer_choice_3": "135 degrees", "answer_choice_4": "180 degrees", "answer_id": 2, "answer": "90 degrees", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "phone-4ee5418723871673", "small white packet-6000e1e21a78231f", "carton of eggs-9e8bcce2d31af231", "tub of cottage cheese-c745fbd9b1d70f77", "bag of coriander-e3f7e72ac7151ef5", "tray of vegetables-a8d031902952ec58", "plastic box with green chillies-598f2394ec50c47b", "carton of eggs and tub of cottage cheese-0ad1fa13e2bca0d7", "back of lettuce-e129240b53be9003", "small pan-3a286c4233af61b7", "white chopping board-8bcfa5bd1bb0cbee", "knife-5792048276490863", "potato-e04555138e7758e7", "red onion-82a5e12bacbcac20", "weighing scale-0293338bb74fd4ee", "medium knife-19cede612034dbf3", "big knife-bd25e124e25fd56e", "left half of the onion-e5db8e6e7130388f", "right half of the onion-74bfbe726dbea0ed", "layers of onion-d6f48a6dfff95858", "olive oil spray-7d35a1020309e24a", "gray bowl-0e4002cb88aeaee7", "white bowl-8a825276b568e15d", "first egg-70403ff6fa78d64d", "second egg-ff85c39f26558a8b", "tea towel-cb179dea3d39f7d5", "spatula-6c9052df67058bdf", "second spatula-f54d097b337b5baa", "fork-0a72cae135bf2eba", "second fork-3f70b18ac231034f", "eggshells-ae798305e2485513", "grey bowl-f6620b0886e36eea", "grey plate-b4b38dfc25baecbd", "top of cottage cheese tub-24330c3b20ad2bf7", "small plate 1-7d6cbfe6e2ca03d1", "small plate 2-6786f79551369dc8", "spoon-c231bb45b9eba87c", "olive oil-6c8e698760dcbf37", "olive oil bottle cap-bf8e0237ebfaa030", "pepper shaker-afd555a3d2affb9d", "wide glass-478b41989e0cb25e", "salt container-c2f26d875c760b7d", "spice container-d00f4dea11f43d04", "sponge-005a6818405d5faf", "dish soap-b588bf5326d065c7", "second sponge-8b70c6acf6cda2da", "cutlery-4d077c65db7d5cef", "steel scrubber-e580cc55a651b8b0", "set of knives-dda6a8f4e769ebfb", "bag of rocket salad-f7ee935423187fa4", "bowl of vegetables-05f704459ca84ded", "kitchen towels-847b3ed68cfdd6ce" ], "reasoning": "1. I watched the whole video and saw the user at the cutting board (white chopping board-8bcfa5bd1bb0cbee) at 10:12.\n2. At 10:18, the user turns left to put a spoon (spoon-c231bb45b9eba87c) in the sink . \n3. The user turns 90 degrees to get to the sink .", "referenced_object_masks": [ "spoon-c231bb45b9eba87c", "white chopping board-8bcfa5bd1bb0cbee" ] }, { "key": "P08-20240614-085000:b40c0576d869cfc530eb1ad4d206aab6dea7c9ea", "video_id": "P08-20240614-085000", "question": "If I take the weight of the red onion, add it to the length of time in seconds it took to scan the QR code on the can of olive oil and then multiply by the number of eggs used to make the scrambled eggs, what number do I get?", "answer_choice_0": "160", "answer_choice_1": "170", "answer_choice_2": "174", "answer_choice_3": "166", "answer_choice_4": "162", "answer_id": 3, "answer": "166", "question_type": [ "Action Recognition", "Counting", "Numerical Reasoning", "Object Recognition", "Event Occurence", "Temporal Reasoning" ], "objects_of_interest": [ "phone-4ee5418723871673", "small white packet-6000e1e21a78231f", "carton of eggs-9e8bcce2d31af231", "tub of cottage cheese-c745fbd9b1d70f77", "bag of coriander-e3f7e72ac7151ef5", "tray of vegetables-a8d031902952ec58", "plastic box with green chillies-598f2394ec50c47b", "carton of eggs and tub of cottage cheese-0ad1fa13e2bca0d7", "back of lettuce-e129240b53be9003", "small pan-3a286c4233af61b7", "white chopping board-8bcfa5bd1bb0cbee", "knife-5792048276490863", "potato-e04555138e7758e7", "red onion-82a5e12bacbcac20", "weighing scale-0293338bb74fd4ee", "medium knife-19cede612034dbf3", "big knife-bd25e124e25fd56e", "left half of the onion-e5db8e6e7130388f", "right half of the onion-74bfbe726dbea0ed", "layers of onion-d6f48a6dfff95858", "olive oil spray-7d35a1020309e24a", "gray bowl-0e4002cb88aeaee7", "white bowl-8a825276b568e15d", "first egg-70403ff6fa78d64d", "second egg-ff85c39f26558a8b", "tea towel-cb179dea3d39f7d5", "spatula-6c9052df67058bdf", "second spatula-f54d097b337b5baa", "fork-0a72cae135bf2eba", "second fork-3f70b18ac231034f", "eggshells-ae798305e2485513", "grey bowl-f6620b0886e36eea", "grey plate-b4b38dfc25baecbd", "top of cottage cheese tub-24330c3b20ad2bf7", "small plate 1-7d6cbfe6e2ca03d1", "small plate 2-6786f79551369dc8", "spoon-c231bb45b9eba87c", "olive oil-6c8e698760dcbf37", "olive oil bottle cap-bf8e0237ebfaa030", "pepper shaker-afd555a3d2affb9d", "wide glass-478b41989e0cb25e", "salt container-c2f26d875c760b7d", "spice container-d00f4dea11f43d04", "sponge-005a6818405d5faf", "dish soap-b588bf5326d065c7", "second sponge-8b70c6acf6cda2da", "cutlery-4d077c65db7d5cef", "steel scrubber-e580cc55a651b8b0", "set of knives-dda6a8f4e769ebfb", "bag of rocket salad-f7ee935423187fa4", "bowl of vegetables-05f704459ca84ded", "kitchen towels-847b3ed68cfdd6ce" ], "reasoning": "1. I watched the whole video and saw the user weigh the red onion (right half of the onion-74bfbe726dbea0ed) at 01:19. \n2. It weighed 45 g.\n3. The user tried to scan the can (olive oil spray-7d35a1020309e24a) of olive oil from 03:38 to 04:16.\n4. The scanning took 38 seconds.\n5. At 05:13, the user cracks open 2 eggs to beat them.\n6. 45 + 38 is 83.\n7. 83 * 2 is 166.", "referenced_object_masks": [ "right half of the onion-74bfbe726dbea0ed", "olive oil spray-7d35a1020309e24a" ] }, { "key": "P08-20240614-085000:bccab32bdb779ee3104c51d824498ee71feecae5", "video_id": "P08-20240614-085000", "question": "If I had weighed a whole red onion on the scale, what would the approximate numerical value of its weight have been?", "answer_choice_0": "2", "answer_choice_1": "5", "answer_choice_2": "4", "answer_choice_3": "9", "answer_choice_4": "10", "answer_id": 3, "answer": "9", "question_type": [ "Object Recognition", "Action Recognition", "Cause and Effect", "State Changes", "Reading", "Numerical Reasoning", "Counterfactual" ], "objects_of_interest": [ "phone-4ee5418723871673", "small white packet-6000e1e21a78231f", "carton of eggs-9e8bcce2d31af231", "tub of cottage cheese-c745fbd9b1d70f77", "bag of coriander-e3f7e72ac7151ef5", "tray of vegetables-a8d031902952ec58", "plastic box with green chillies-598f2394ec50c47b", "carton of eggs and tub of cottage cheese-0ad1fa13e2bca0d7", "back of lettuce-e129240b53be9003", "small pan-3a286c4233af61b7", "white chopping board-8bcfa5bd1bb0cbee", "knife-5792048276490863", "potato-e04555138e7758e7", "red onion-82a5e12bacbcac20", "weighing scale-0293338bb74fd4ee", "medium knife-19cede612034dbf3", "big knife-bd25e124e25fd56e", "left half of the onion-e5db8e6e7130388f", "right half of the onion-74bfbe726dbea0ed", "layers of onion-d6f48a6dfff95858", "olive oil spray-7d35a1020309e24a", "gray bowl-0e4002cb88aeaee7", "white bowl-8a825276b568e15d", "first egg-70403ff6fa78d64d", "second egg-ff85c39f26558a8b", "tea towel-cb179dea3d39f7d5", "spatula-6c9052df67058bdf", "second spatula-f54d097b337b5baa", "fork-0a72cae135bf2eba", "second fork-3f70b18ac231034f", "eggshells-ae798305e2485513", "grey bowl-f6620b0886e36eea", "grey plate-b4b38dfc25baecbd", "top of cottage cheese tub-24330c3b20ad2bf7", "small plate 1-7d6cbfe6e2ca03d1", "small plate 2-6786f79551369dc8", "spoon-c231bb45b9eba87c", "olive oil-6c8e698760dcbf37", "olive oil bottle cap-bf8e0237ebfaa030", "pepper shaker-afd555a3d2affb9d", "wide glass-478b41989e0cb25e", "salt container-c2f26d875c760b7d", "spice container-d00f4dea11f43d04", "sponge-005a6818405d5faf", "dish soap-b588bf5326d065c7", "second sponge-8b70c6acf6cda2da", "cutlery-4d077c65db7d5cef", "steel scrubber-e580cc55a651b8b0", "set of knives-dda6a8f4e769ebfb", "bag of rocket salad-f7ee935423187fa4", "bowl of vegetables-05f704459ca84ded", "kitchen towels-847b3ed68cfdd6ce" ], "reasoning": "1. I watched the video and found the scale (weighing scale-0293338bb74fd4ee) at 01:00.\n2. I saw the user cut a whole red onion (red onion-82a5e12bacbcac20) in half with a knife (big knife-bd25e124e25fd56e), resulting in two halves of the onion (left half of the onion-e5db8e6e7130388f, right half of the onion-74bfbe726dbea0ed).\n3. I saw the user weigh half of an onion (right half of the onion-74bfbe726dbea0ed) on the scale (weighing scale-0293338bb74fd4ee) at 01:20.\n4. At 01:20-01:23, I saw the reading of the scale (weighing scale-0293338bb74fd4ee) change from 0.0 to 4.5.\n5. Since the numerical value of the weight of half of an onion (right half of the onion-74bfbe726dbea0ed) is 4.5, I calculated 4.5*2=9 to deduce the approximate numerical value of the weight of the whole onion (red onion-82a5e12bacbcac20).\n6. Therefore, if the user had weighed a whole red onion (red onion-82a5e12bacbcac20) on the scale (weighing scale-0293338bb74fd4ee), the approximate numerical value of its weight would have been 9.", "referenced_object_masks": [ "left half of the onion-e5db8e6e7130388f", "right half of the onion-74bfbe726dbea0ed", "red onion-82a5e12bacbcac20", "weighing scale-0293338bb74fd4ee", "big knife-bd25e124e25fd56e" ] }, { "key": "P08-20240614-085000:e1fbdad1e423d1c662f30752985218b6e1a15e4e", "video_id": "P08-20240614-085000", "question": "What direction would I have turned to look at the stove if it were in the opposite direction while I was dicing the onion?", "answer_choice_0": "90 degrees upward", "answer_choice_1": "135 degrees to the left", "answer_choice_2": "180 degrees around", "answer_choice_3": "90 degrees downward", "answer_choice_4": "135 degrees to the right", "answer_id": 1, "answer": "135 degrees to the left", "question_type": [ "Object Recognition", "Action Recognition", "Situational Awareness", "Egocentric & Spatial Perception", "Cause and Effect", "Counterfactual" ], "objects_of_interest": [ "phone-4ee5418723871673", "small white packet-6000e1e21a78231f", "carton of eggs-9e8bcce2d31af231", "tub of cottage cheese-c745fbd9b1d70f77", "bag of coriander-e3f7e72ac7151ef5", "tray of vegetables-a8d031902952ec58", "plastic box with green chillies-598f2394ec50c47b", "carton of eggs and tub of cottage cheese-0ad1fa13e2bca0d7", "back of lettuce-e129240b53be9003", "small pan-3a286c4233af61b7", "white chopping board-8bcfa5bd1bb0cbee", "knife-5792048276490863", "potato-e04555138e7758e7", "red onion-82a5e12bacbcac20", "weighing scale-0293338bb74fd4ee", "medium knife-19cede612034dbf3", "big knife-bd25e124e25fd56e", "left half of the onion-e5db8e6e7130388f", "right half of the onion-74bfbe726dbea0ed", "layers of onion-d6f48a6dfff95858", "olive oil spray-7d35a1020309e24a", "gray bowl-0e4002cb88aeaee7", "white bowl-8a825276b568e15d", "first egg-70403ff6fa78d64d", "second egg-ff85c39f26558a8b", "tea towel-cb179dea3d39f7d5", "spatula-6c9052df67058bdf", "second spatula-f54d097b337b5baa", "fork-0a72cae135bf2eba", "second fork-3f70b18ac231034f", "eggshells-ae798305e2485513", "grey bowl-f6620b0886e36eea", "grey plate-b4b38dfc25baecbd", "top of cottage cheese tub-24330c3b20ad2bf7", "small plate 1-7d6cbfe6e2ca03d1", "small plate 2-6786f79551369dc8", "spoon-c231bb45b9eba87c", "olive oil-6c8e698760dcbf37", "olive oil bottle cap-bf8e0237ebfaa030", "pepper shaker-afd555a3d2affb9d", "wide glass-478b41989e0cb25e", "salt container-c2f26d875c760b7d", "spice container-d00f4dea11f43d04", "sponge-005a6818405d5faf", "dish soap-b588bf5326d065c7", "second sponge-8b70c6acf6cda2da", "cutlery-4d077c65db7d5cef", "steel scrubber-e580cc55a651b8b0", "set of knives-dda6a8f4e769ebfb", "bag of rocket salad-f7ee935423187fa4", "bowl of vegetables-05f704459ca84ded", "kitchen towels-847b3ed68cfdd6ce" ], "reasoning": "1. I watched the video and saw the user dicing an onion (right half of the onion-74bfbe726dbea0ed) at 02:23-03:29.\n2. I saw the stove at 03:29 when the user turned 135 degrees to the right to look at it.\n3. Since the user turned 135 degrees to the right to look from the diced onion (right half of the onion-74bfbe726dbea0ed) to the stove at 03:29, the stove was consequently behind and to the right of the user while dicing the onion (right half of the onion-74bfbe726dbea0ed).\n4. The opposite of \"to the right\" is \"to the left.\"\n5. Therefore, the user would have turned 135 degrees to the left to look at the stove if it were in the opposite direction while they were dicing the onion (right half of the onion-74bfbe726dbea0ed).", "referenced_object_masks": [ "right half of the onion-74bfbe726dbea0ed" ] }, { "key": "P08-20240614-085000:e57eafef66b9db68c87502769bc4d505bc8dfaf4", "video_id": "P08-20240614-085000", "question": "Where did I move the egg carton to after taking it out of the refrigerator?", "answer_choice_0": "To the kitchen sink", "answer_choice_1": "To a kitchen countertop", "answer_choice_2": "To the kitchen stove", "answer_choice_3": "To the refrigerator", "answer_choice_4": "To a cutting board", "answer_id": 1, "answer": "To a kitchen countertop", "question_type": [ "Situational Awareness", "Object Recognition", "Egocentric & Spatial Perception", "Action Recognition", "State Changes", "Temporal Reasoning" ], "objects_of_interest": [ "phone-4ee5418723871673", "small white packet-6000e1e21a78231f", "carton of eggs-9e8bcce2d31af231", "tub of cottage cheese-c745fbd9b1d70f77", "bag of coriander-e3f7e72ac7151ef5", "tray of vegetables-a8d031902952ec58", "plastic box with green chillies-598f2394ec50c47b", "carton of eggs and tub of cottage cheese-0ad1fa13e2bca0d7", "back of lettuce-e129240b53be9003", "small pan-3a286c4233af61b7", "white chopping board-8bcfa5bd1bb0cbee", "knife-5792048276490863", "potato-e04555138e7758e7", "red onion-82a5e12bacbcac20", "weighing scale-0293338bb74fd4ee", "medium knife-19cede612034dbf3", "big knife-bd25e124e25fd56e", "left half of the onion-e5db8e6e7130388f", "right half of the onion-74bfbe726dbea0ed", "layers of onion-d6f48a6dfff95858", "olive oil spray-7d35a1020309e24a", "gray bowl-0e4002cb88aeaee7", "white bowl-8a825276b568e15d", "first egg-70403ff6fa78d64d", "second egg-ff85c39f26558a8b", "tea towel-cb179dea3d39f7d5", "spatula-6c9052df67058bdf", "second spatula-f54d097b337b5baa", "fork-0a72cae135bf2eba", "second fork-3f70b18ac231034f", "eggshells-ae798305e2485513", "grey bowl-f6620b0886e36eea", "grey plate-b4b38dfc25baecbd", "top of cottage cheese tub-24330c3b20ad2bf7", "small plate 1-7d6cbfe6e2ca03d1", "small plate 2-6786f79551369dc8", "spoon-c231bb45b9eba87c", "olive oil-6c8e698760dcbf37", "olive oil bottle cap-bf8e0237ebfaa030", "pepper shaker-afd555a3d2affb9d", "wide glass-478b41989e0cb25e", "salt container-c2f26d875c760b7d", "spice container-d00f4dea11f43d04", "sponge-005a6818405d5faf", "dish soap-b588bf5326d065c7", "second sponge-8b70c6acf6cda2da", "cutlery-4d077c65db7d5cef", "steel scrubber-e580cc55a651b8b0", "set of knives-dda6a8f4e769ebfb", "bag of rocket salad-f7ee935423187fa4", "bowl of vegetables-05f704459ca84ded", "kitchen towels-847b3ed68cfdd6ce" ], "reasoning": "1. I watched the video and found the refrigerator at 00:08.\n2. I found the egg carton (carton of eggs-9e8bcce2d31af231) inside the refrigerator at 00:11.\n3. At 00:12, I saw the user take the egg carton (carton of eggs-9e8bcce2d31af231) out of the refrigerator.\n4. At 00:29, I saw the user place the egg carton (carton of eggs-9e8bcce2d31af231) on a kitchen countertop.\n5. Therefore, the user moved the egg carton (carton of eggs-9e8bcce2d31af231) to a kitchen countertop after taking it out of the refrigerator.", "referenced_object_masks": [ "carton of eggs-9e8bcce2d31af231" ] }, { "key": "P08-20240614-085000:ed48cdfdf87bb3eeecc91db12065b13dd5d20c0f", "video_id": "P08-20240614-085000", "question": "What is the mathematical difference between the number of items I grabbed from the refrigerator after visiting it for the first and second time?", "answer_choice_0": "3", "answer_choice_1": "4", "answer_choice_2": "1", "answer_choice_3": "2", "answer_choice_4": "5", "answer_id": 0, "answer": "3", "question_type": [ "Situational Awareness", "Counting", "Action Recognition", "Object Recognition", "State Changes", "Egocentric & Spatial Perception", "Temporal Reasoning", "Numerical Reasoning" ], "objects_of_interest": [ "phone-4ee5418723871673", "small white packet-6000e1e21a78231f", "carton of eggs-9e8bcce2d31af231", "tub of cottage cheese-c745fbd9b1d70f77", "bag of coriander-e3f7e72ac7151ef5", "tray of vegetables-a8d031902952ec58", "plastic box with green chillies-598f2394ec50c47b", "carton of eggs and tub of cottage cheese-0ad1fa13e2bca0d7", "back of lettuce-e129240b53be9003", "small pan-3a286c4233af61b7", "white chopping board-8bcfa5bd1bb0cbee", "knife-5792048276490863", "potato-e04555138e7758e7", "red onion-82a5e12bacbcac20", "weighing scale-0293338bb74fd4ee", "medium knife-19cede612034dbf3", "big knife-bd25e124e25fd56e", "left half of the onion-e5db8e6e7130388f", "right half of the onion-74bfbe726dbea0ed", "layers of onion-d6f48a6dfff95858", "olive oil spray-7d35a1020309e24a", "gray bowl-0e4002cb88aeaee7", "white bowl-8a825276b568e15d", "first egg-70403ff6fa78d64d", "second egg-ff85c39f26558a8b", "tea towel-cb179dea3d39f7d5", "spatula-6c9052df67058bdf", "second spatula-f54d097b337b5baa", "fork-0a72cae135bf2eba", "second fork-3f70b18ac231034f", "eggshells-ae798305e2485513", "grey bowl-f6620b0886e36eea", "grey plate-b4b38dfc25baecbd", "top of cottage cheese tub-24330c3b20ad2bf7", "small plate 1-7d6cbfe6e2ca03d1", "small plate 2-6786f79551369dc8", "spoon-c231bb45b9eba87c", "olive oil-6c8e698760dcbf37", "olive oil bottle cap-bf8e0237ebfaa030", "pepper shaker-afd555a3d2affb9d", "wide glass-478b41989e0cb25e", "salt container-c2f26d875c760b7d", "spice container-d00f4dea11f43d04", "sponge-005a6818405d5faf", "dish soap-b588bf5326d065c7", "second sponge-8b70c6acf6cda2da", "cutlery-4d077c65db7d5cef", "steel scrubber-e580cc55a651b8b0", "set of knives-dda6a8f4e769ebfb", "bag of rocket salad-f7ee935423187fa4", "bowl of vegetables-05f704459ca84ded", "kitchen towels-847b3ed68cfdd6ce" ], "reasoning": "1. I watched the video and found the refrigerator at 00:08.\n2. I counted each time the user visited the refrigerator.\n3. I saw the user visit the refrigerator for the first time at 00:09.\n4. I counted each item the user grabbed from the refrigerator during their first time visiting it at 00:09-00:27, calculating a total of 4, which were an egg carton (carton of eggs-9e8bcce2d31af231), a tub of cottage cheese (tub of cottage cheese-c745fbd9b1d70f77), a bag of coriander (bag of coriander-e3f7e72ac7151ef5), and a plastic box of chili peppers (plastic box with green chillies-598f2394ec50c47b).\n5. I saw the user depart from the refrigerator for the first time at 00:27.\n6. I saw the user visit the refrigerator for the second time at 00:30.\n7. I counted each item the user grabbed from the refrigerator during their second time visiting it at 00:30-00:37, calculating a total of 1, which was a bag of lettuce (back of lettuce-e129240b53be9003).\n8. I calculated 4-1=3.\n9. Therefore, the mathematical difference between the number of items the user grabbed from the refrigerator after visiting it for the first and second time is 3.", "referenced_object_masks": [ "plastic box with green chillies-598f2394ec50c47b", "back of lettuce-e129240b53be9003", "bag of coriander-e3f7e72ac7151ef5", "carton of eggs-9e8bcce2d31af231", "tub of cottage cheese-c745fbd9b1d70f77" ] }, { "key": "P08-20240614-085000:fc9d46b3abdb42753a21d2767048592777457c69", "video_id": "P08-20240614-085000", "question": "Listed in chronological order, what 3 textures of eggs are shown in this video?", "answer_choice_0": "Solid, beaten, liquid", "answer_choice_1": "Liquid, scrambled solid, beaten", "answer_choice_2": "Beaten, liquid, scrambled solid", "answer_choice_3": "Liquid, beaten, scrambled solid", "answer_choice_4": "Solid, liquid, beaten", "answer_id": 3, "answer": "Liquid, beaten, scrambled solid", "question_type": [ "Action Recognition", "Event Occurence", "State Changes", "Temporal Reasoning" ], "objects_of_interest": [ "phone-4ee5418723871673", "small white packet-6000e1e21a78231f", "carton of eggs-9e8bcce2d31af231", "tub of cottage cheese-c745fbd9b1d70f77", "bag of coriander-e3f7e72ac7151ef5", "tray of vegetables-a8d031902952ec58", "plastic box with green chillies-598f2394ec50c47b", "carton of eggs and tub of cottage cheese-0ad1fa13e2bca0d7", "back of lettuce-e129240b53be9003", "small pan-3a286c4233af61b7", "white chopping board-8bcfa5bd1bb0cbee", "knife-5792048276490863", "potato-e04555138e7758e7", "red onion-82a5e12bacbcac20", "weighing scale-0293338bb74fd4ee", "medium knife-19cede612034dbf3", "big knife-bd25e124e25fd56e", "left half of the onion-e5db8e6e7130388f", "right half of the onion-74bfbe726dbea0ed", "layers of onion-d6f48a6dfff95858", "olive oil spray-7d35a1020309e24a", "gray bowl-0e4002cb88aeaee7", "white bowl-8a825276b568e15d", "first egg-70403ff6fa78d64d", "second egg-ff85c39f26558a8b", "tea towel-cb179dea3d39f7d5", "spatula-6c9052df67058bdf", "second spatula-f54d097b337b5baa", "fork-0a72cae135bf2eba", "second fork-3f70b18ac231034f", "eggshells-ae798305e2485513", "grey bowl-f6620b0886e36eea", "grey plate-b4b38dfc25baecbd", "top of cottage cheese tub-24330c3b20ad2bf7", "small plate 1-7d6cbfe6e2ca03d1", "small plate 2-6786f79551369dc8", "spoon-c231bb45b9eba87c", "olive oil-6c8e698760dcbf37", "olive oil bottle cap-bf8e0237ebfaa030", "pepper shaker-afd555a3d2affb9d", "wide glass-478b41989e0cb25e", "salt container-c2f26d875c760b7d", "spice container-d00f4dea11f43d04", "sponge-005a6818405d5faf", "dish soap-b588bf5326d065c7", "second sponge-8b70c6acf6cda2da", "cutlery-4d077c65db7d5cef", "steel scrubber-e580cc55a651b8b0", "set of knives-dda6a8f4e769ebfb", "bag of rocket salad-f7ee935423187fa4", "bowl of vegetables-05f704459ca84ded", "kitchen towels-847b3ed68cfdd6ce" ], "reasoning": "1. I watched the whole video and saw the user get a carton of eggs (carton of eggs-9e8bcce2d31af231) from the refrigerator at 00:12.\n2. At 05:15, the user began to crack 2 eggs . This is the liquid texture.\n2. At 11:42, the user began to beat the 2 eggs with a fork (fork-0a72cae135bf2eba). This is the beaten texture.\n3. At 11:42, the user poured the beaten eggs into the pan (small pan-3a286c4233af61b7). By 13:39, the eggs were scrambled solid.", "referenced_object_masks": [ "fork-0a72cae135bf2eba", "carton of eggs-9e8bcce2d31af231", "small pan-3a286c4233af61b7" ] }, { "key": "P08-20240614-125650:1975e53d339a4eebce5c4f54aad0f619bd2c2e28", "video_id": "P08-20240614-125650", "question": "What is the primary color of the oven mitt I use to take things out of the oven?", "answer_choice_0": "Green", "answer_choice_1": "Black", "answer_choice_2": "Blue", "answer_choice_3": "White", "answer_choice_4": "Red", "answer_id": 4, "answer": "Red", "question_type": [ "Object Recognition", "Egocentric & Spatial Perception", "Action Recognition" ], "objects_of_interest": [ "phone-5ffd04b506b53b9c", "baking tray1-147956c80da43fc2", "baking tray2-1e5294196e98a03b", "aluminum foil package-afa16ed3970c1ada", "foil paper piece-1327fa6f3ddacf44", "plastic bag with chicken thigh-08259e9113a28f33", "scale-e885a2e918e911e7", "tea towel-4d18a2f1b95f5543", "salt shaker-4072cc656246f435", "chicken thigh-7ef1f6dbcac6d8b7", "pepper shaker-03f2a01e6f436c4f", "kitchen brush-bfa18d58bd711aae", "olive oil spray-14acad755753e316", "chopping board-900d57cdf92f63bf", "salad bowl-7812e76936f09e9a", "bag of veg1-b8039a9badbcb16d", "bag of veg2-23004bcc3f41c7f7", "yoghurt tub-4c150fd1cedd03bc", "basket-8d3768839e95bfb8", "bag of seeds-8b734610ce337b70", "salad wrap-61f251a3b93140fe", "salad sauce container-f01d60c741b2e113", "bowl-e51b9e55db0814c1", "fork-97971c1fe05c128e", "paper towel-0aed95828d20de25", "cutting board-399c76fa3e80b8b7", "carrot-38d20d10d4e3feb9", "peeler-86e54f7b628161a6", "knife-372134053789cc09", "carrot pieces-45f966cd439de62a", "oven mitt-0cfbb21e4c01ceb5", "kitchen tongs-29bb885641a88bfd", "avocado-847fec247a6b1809", "avocado half-3b456dd667014c9b", "small plate-91023bdac929f2a3", "small spoon-57979b1ff0890896", "avocado skin-9709b0800ca5f589", "tea towel2-2d82d76c7378662c", "pan-6a1fd7c0d40b130a", "bowl2-35430679c1f863c0", "tupperware-34116b9e84bee3fd", "coffee cup-cce2809ca16a7c6b", "top of coffee cup-243e0f96e49d6c6b", "knife5-ed40934fcb199b90", "knife3-74933a149441ba00", "spatulas-04717b621df81ddc", "tupperware2-89bcb463741f9eae", "tupperware lid2-eadc477c58c9e076", "tupperware3-a0e2e7f4be98ed7c", "tupperware4-7d797fb35ed7d34c", "small plate2-e21bfc195bd6cd66", "wine glass-d99dd0528a89697d", "cocktail mixer-4631f7ce7c1398bc", "utensils-58a427d5c84042af", "cocktail mixer part-84c109d936ca00d4", "cocktail mixer part2-eb9256108921f996", "cocktail mixer part3-b56220d415e3d4ce", "top of cocktail mixer-780362e075bf04d7", "fork2-0f3a164b492bed79", "spoon2-a244cee1efa6ceda", "sponge-13e9c2572a6aba5c", "washing up liquid bottle-ae579e7db2f0e519", "knife4-9408afbf9bd45759", "empty food container-27be1f90ab24a743", "head of kitchen brush-b54dde5748c92781", "kitchen towel roll-61d8606600acfca8", "knife holder-271c919bb46b4387", "mugs holder-ecfa28dea2a4348f", "clear bowl of vegetables-fe58b329441753ed" ], "reasoning": "1. As I watched the video, I saw the user turn the oven on from 00:09-00:12.\n2. I continued to watch the video, and during the first 12 monites, the user does things away from the oven.\n3. Then, at 12:22-12:15, I observed the user moving towards the oven and opening a drawer below it to grab an oven mitt (oven mitt-0cfbb21e4c01ceb5). In this span, it looks like the mitt (oven mitt-0cfbb21e4c01ceb5) is red in color, but it is not super clear.\n4. I continued to watch from 12:30-12:37, in which I clearly saw that the oven mitt (oven mitt-0cfbb21e4c01ceb5) is primarily red.", "referenced_object_masks": [ "oven mitt-0cfbb21e4c01ceb5" ] }, { "key": "P08-20240614-125650:2f5b3bd8238b2373a0c886408c1ce9149978c46e", "video_id": "P08-20240614-125650", "question": "What is the product of the weight of the chicken thigh and the number of mugs hanging on the mug holder?", "answer_choice_0": "1520", "answer_choice_1": "1240", "answer_choice_2": "1750", "answer_choice_3": "1660", "answer_choice_4": "2000", "answer_id": 0, "answer": "1520", "question_type": [ "Reading", "Numerical Reasoning", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "phone-5ffd04b506b53b9c", "baking tray1-147956c80da43fc2", "baking tray2-1e5294196e98a03b", "aluminum foil package-afa16ed3970c1ada", "foil paper piece-1327fa6f3ddacf44", "plastic bag with chicken thigh-08259e9113a28f33", "scale-e885a2e918e911e7", "tea towel-4d18a2f1b95f5543", "salt shaker-4072cc656246f435", "chicken thigh-7ef1f6dbcac6d8b7", "pepper shaker-03f2a01e6f436c4f", "kitchen brush-bfa18d58bd711aae", "olive oil spray-14acad755753e316", "chopping board-900d57cdf92f63bf", "salad bowl-7812e76936f09e9a", "bag of veg1-b8039a9badbcb16d", "bag of veg2-23004bcc3f41c7f7", "yoghurt tub-4c150fd1cedd03bc", "basket-8d3768839e95bfb8", "bag of seeds-8b734610ce337b70", "salad wrap-61f251a3b93140fe", "salad sauce container-f01d60c741b2e113", "bowl-e51b9e55db0814c1", "fork-97971c1fe05c128e", "paper towel-0aed95828d20de25", "cutting board-399c76fa3e80b8b7", "carrot-38d20d10d4e3feb9", "peeler-86e54f7b628161a6", "knife-372134053789cc09", "carrot pieces-45f966cd439de62a", "oven mitt-0cfbb21e4c01ceb5", "kitchen tongs-29bb885641a88bfd", "avocado-847fec247a6b1809", "avocado half-3b456dd667014c9b", "small plate-91023bdac929f2a3", "small spoon-57979b1ff0890896", "avocado skin-9709b0800ca5f589", "tea towel2-2d82d76c7378662c", "pan-6a1fd7c0d40b130a", "bowl2-35430679c1f863c0", "tupperware-34116b9e84bee3fd", "coffee cup-cce2809ca16a7c6b", "top of coffee cup-243e0f96e49d6c6b", "knife5-ed40934fcb199b90", "knife3-74933a149441ba00", "spatulas-04717b621df81ddc", "tupperware2-89bcb463741f9eae", "tupperware lid2-eadc477c58c9e076", "tupperware3-a0e2e7f4be98ed7c", "tupperware4-7d797fb35ed7d34c", "small plate2-e21bfc195bd6cd66", "wine glass-d99dd0528a89697d", "cocktail mixer-4631f7ce7c1398bc", "utensils-58a427d5c84042af", "cocktail mixer part-84c109d936ca00d4", "cocktail mixer part2-eb9256108921f996", "cocktail mixer part3-b56220d415e3d4ce", "top of cocktail mixer-780362e075bf04d7", "fork2-0f3a164b492bed79", "spoon2-a244cee1efa6ceda", "sponge-13e9c2572a6aba5c", "washing up liquid bottle-ae579e7db2f0e519", "knife4-9408afbf9bd45759", "empty food container-27be1f90ab24a743", "head of kitchen brush-b54dde5748c92781", "kitchen towel roll-61d8606600acfca8", "knife holder-271c919bb46b4387", "mugs holder-ecfa28dea2a4348f", "clear bowl of vegetables-fe58b329441753ed" ], "reasoning": "1. I watched the video to see the user take the chicken thigh (plastic bag with chicken thigh-08259e9113a28f33) out of the refrigerator from 00:43-00:46.\n2. Then, I observed the user take a scale (scale-e885a2e918e911e7) out, placing the chicken thigh (chicken thigh-7ef1f6dbcac6d8b7) on it from 01:10-01:12.\n3. At 01:12, I looked at the scale and read that it weighed \"304.\"\n4. I continued to watch the video until 01:23, when the mug holder is clearly in view.\n5. At this point, I counted 5 mugs on the holder.\n6. Then, I did the math, which is 304 x 5 = 1520.", "referenced_object_masks": [ "chicken thigh-7ef1f6dbcac6d8b7", "scale-e885a2e918e911e7", "plastic bag with chicken thigh-08259e9113a28f33" ] }, { "key": "P08-20240614-125650:45588ccf2b8bd63998417a6bb3601bc5a02dd9af", "video_id": "P08-20240614-125650", "question": "Not including the item the user is holding at the video's open, what is the second item I grab with my left hand?", "answer_choice_0": "Cell phone", "answer_choice_1": "Baking Tray 1", "answer_choice_2": "Oven door", "answer_choice_3": "Plate", "answer_choice_4": "Scale", "answer_id": 1, "answer": "Baking Tray 1", "question_type": [ "Egocentric & Spatial Perception", "Action Recognition", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "phone-5ffd04b506b53b9c", "baking tray1-147956c80da43fc2", "baking tray2-1e5294196e98a03b", "aluminum foil package-afa16ed3970c1ada", "foil paper piece-1327fa6f3ddacf44", "plastic bag with chicken thigh-08259e9113a28f33", "scale-e885a2e918e911e7", "tea towel-4d18a2f1b95f5543", "salt shaker-4072cc656246f435", "chicken thigh-7ef1f6dbcac6d8b7", "pepper shaker-03f2a01e6f436c4f", "kitchen brush-bfa18d58bd711aae", "olive oil spray-14acad755753e316", "chopping board-900d57cdf92f63bf", "salad bowl-7812e76936f09e9a", "bag of veg1-b8039a9badbcb16d", "bag of veg2-23004bcc3f41c7f7", "yoghurt tub-4c150fd1cedd03bc", "basket-8d3768839e95bfb8", "bag of seeds-8b734610ce337b70", "salad wrap-61f251a3b93140fe", "salad sauce container-f01d60c741b2e113", "bowl-e51b9e55db0814c1", "fork-97971c1fe05c128e", "paper towel-0aed95828d20de25", "cutting board-399c76fa3e80b8b7", "carrot-38d20d10d4e3feb9", "peeler-86e54f7b628161a6", "knife-372134053789cc09", "carrot pieces-45f966cd439de62a", "oven mitt-0cfbb21e4c01ceb5", "kitchen tongs-29bb885641a88bfd", "avocado-847fec247a6b1809", "avocado half-3b456dd667014c9b", "small plate-91023bdac929f2a3", "small spoon-57979b1ff0890896", "avocado skin-9709b0800ca5f589", "tea towel2-2d82d76c7378662c", "pan-6a1fd7c0d40b130a", "bowl2-35430679c1f863c0", "tupperware-34116b9e84bee3fd", "coffee cup-cce2809ca16a7c6b", "top of coffee cup-243e0f96e49d6c6b", "knife5-ed40934fcb199b90", "knife3-74933a149441ba00", "spatulas-04717b621df81ddc", "tupperware2-89bcb463741f9eae", "tupperware lid2-eadc477c58c9e076", "tupperware3-a0e2e7f4be98ed7c", "tupperware4-7d797fb35ed7d34c", "small plate2-e21bfc195bd6cd66", "wine glass-d99dd0528a89697d", "cocktail mixer-4631f7ce7c1398bc", "utensils-58a427d5c84042af", "cocktail mixer part-84c109d936ca00d4", "cocktail mixer part2-eb9256108921f996", "cocktail mixer part3-b56220d415e3d4ce", "top of cocktail mixer-780362e075bf04d7", "fork2-0f3a164b492bed79", "spoon2-a244cee1efa6ceda", "sponge-13e9c2572a6aba5c", "washing up liquid bottle-ae579e7db2f0e519", "knife4-9408afbf9bd45759", "empty food container-27be1f90ab24a743", "head of kitchen brush-b54dde5748c92781", "kitchen towel roll-61d8606600acfca8", "knife holder-271c919bb46b4387", "mugs holder-ecfa28dea2a4348f", "clear bowl of vegetables-fe58b329441753ed" ], "reasoning": "1. At 00:00, I observed the user holding their cell phone (phone-5ffd04b506b53b9c) in the left hand. Because they are holding it at the beginning of the video, this does not count.\n2. The user then walks forward, holding the phone (phone-5ffd04b506b53b9c) in the left hand from 00:00-00:06.\n3. I then saw the user turn to the right and use their right hand to flip a switch , turn 2 oven knobs , and reach into the oven , opening it with the right hand from 00:07-00:12.\n4. At 00:12, I briefly noticed the user touching the oven door with their left hand (visible in bottom left of frame). This is then the first item touched with the left hand.\n5. After opening the oven , the user grabs the first baking tray (baking tray1-147956c80da43fc2) with the right hand before transferring it to the left hand from 00:13-00:15.\nSince the user first touched the oven door with the left hand, this makes baking tray 1 (baking tray1-147956c80da43fc2) the second item touched with the left hand.", "referenced_object_masks": [ "phone-5ffd04b506b53b9c", "baking tray1-147956c80da43fc2" ] }, { "key": "P08-20240614-125650:49b511619277dc173a2285ec9303051ccb6d72d4", "video_id": "P08-20240614-125650", "question": "After I weigh the carrot, how many times do I peel the carrot before I move the scale?", "answer_choice_0": "7", "answer_choice_1": "3", "answer_choice_2": "5", "answer_choice_3": "4", "answer_choice_4": "2", "answer_id": 3, "answer": "4", "question_type": [ "Action Recognition", "Counting", "Event Occurence", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "phone-5ffd04b506b53b9c", "baking tray1-147956c80da43fc2", "baking tray2-1e5294196e98a03b", "aluminum foil package-afa16ed3970c1ada", "foil paper piece-1327fa6f3ddacf44", "plastic bag with chicken thigh-08259e9113a28f33", "scale-e885a2e918e911e7", "tea towel-4d18a2f1b95f5543", "salt shaker-4072cc656246f435", "chicken thigh-7ef1f6dbcac6d8b7", "pepper shaker-03f2a01e6f436c4f", "kitchen brush-bfa18d58bd711aae", "olive oil spray-14acad755753e316", "chopping board-900d57cdf92f63bf", "salad bowl-7812e76936f09e9a", "bag of veg1-b8039a9badbcb16d", "bag of veg2-23004bcc3f41c7f7", "yoghurt tub-4c150fd1cedd03bc", "basket-8d3768839e95bfb8", "bag of seeds-8b734610ce337b70", "salad wrap-61f251a3b93140fe", "salad sauce container-f01d60c741b2e113", "bowl-e51b9e55db0814c1", "fork-97971c1fe05c128e", "paper towel-0aed95828d20de25", "cutting board-399c76fa3e80b8b7", "carrot-38d20d10d4e3feb9", "peeler-86e54f7b628161a6", "knife-372134053789cc09", "carrot pieces-45f966cd439de62a", "oven mitt-0cfbb21e4c01ceb5", "kitchen tongs-29bb885641a88bfd", "avocado-847fec247a6b1809", "avocado half-3b456dd667014c9b", "small plate-91023bdac929f2a3", "small spoon-57979b1ff0890896", "avocado skin-9709b0800ca5f589", "tea towel2-2d82d76c7378662c", "pan-6a1fd7c0d40b130a", "bowl2-35430679c1f863c0", "tupperware-34116b9e84bee3fd", "coffee cup-cce2809ca16a7c6b", "top of coffee cup-243e0f96e49d6c6b", "knife5-ed40934fcb199b90", "knife3-74933a149441ba00", "spatulas-04717b621df81ddc", "tupperware2-89bcb463741f9eae", "tupperware lid2-eadc477c58c9e076", "tupperware3-a0e2e7f4be98ed7c", "tupperware4-7d797fb35ed7d34c", "small plate2-e21bfc195bd6cd66", "wine glass-d99dd0528a89697d", "cocktail mixer-4631f7ce7c1398bc", "utensils-58a427d5c84042af", "cocktail mixer part-84c109d936ca00d4", "cocktail mixer part2-eb9256108921f996", "cocktail mixer part3-b56220d415e3d4ce", "top of cocktail mixer-780362e075bf04d7", "fork2-0f3a164b492bed79", "spoon2-a244cee1efa6ceda", "sponge-13e9c2572a6aba5c", "washing up liquid bottle-ae579e7db2f0e519", "knife4-9408afbf9bd45759", "empty food container-27be1f90ab24a743", "head of kitchen brush-b54dde5748c92781", "kitchen towel roll-61d8606600acfca8", "knife holder-271c919bb46b4387", "mugs holder-ecfa28dea2a4348f", "clear bowl of vegetables-fe58b329441753ed" ], "reasoning": "1. I noticed the carrot (carrot-38d20d10d4e3feb9) on the scale (scale-e885a2e918e911e7) at 09:01.\n2. I noticed the user begin to peel the carrot (carrot-38d20d10d4e3feb9) with the peeler (peeler-86e54f7b628161a6) at 09:04 for the 1st time.\n3. I noticed the user peel the carrot (carrot-38d20d10d4e3feb9) with the peeler (peeler-86e54f7b628161a6) at 09:06 for the 2nd time.\n4. I noticed the user peel the carrot (carrot-38d20d10d4e3feb9) with the peeler (peeler-86e54f7b628161a6) at 09:08 for the 3rd time.\n5. I noticed the user peel the carrot (carrot-38d20d10d4e3feb9) with the peeler (peeler-86e54f7b628161a6) at 09:11 for the 4th time.\n6. I noticed the user move the scale (scale-e885a2e918e911e7) at 09:14, leaving us with a final answer of 4.", "referenced_object_masks": [ "carrot-38d20d10d4e3feb9", "peeler-86e54f7b628161a6", "scale-e885a2e918e911e7" ] }, { "key": "P08-20240614-125650:53bced260cf186a803fbdc1a6a4e5e8afea41b6e", "video_id": "P08-20240614-125650", "question": "In the last minute of the video, in what order do I touch the leftover plastic wrap, the trash bin lid, and the hand towel?", "answer_choice_0": "Trash bin lid, hand towel, plastic wrap", "answer_choice_1": "Hand towel, plastic wrap, trash bin lid", "answer_choice_2": "Plastic wrap, hand towel, trash bin lid", "answer_choice_3": "Plastic wrap, trash bin lid, hand towel", "answer_choice_4": "Hand towel, trash bin lid, plastic wrap", "answer_id": 4, "answer": "Hand towel, trash bin lid, plastic wrap", "question_type": [ "Action Recognition", "Counting", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "phone-5ffd04b506b53b9c", "baking tray1-147956c80da43fc2", "baking tray2-1e5294196e98a03b", "aluminum foil package-afa16ed3970c1ada", "foil paper piece-1327fa6f3ddacf44", "plastic bag with chicken thigh-08259e9113a28f33", "scale-e885a2e918e911e7", "tea towel-4d18a2f1b95f5543", "salt shaker-4072cc656246f435", "chicken thigh-7ef1f6dbcac6d8b7", "pepper shaker-03f2a01e6f436c4f", "kitchen brush-bfa18d58bd711aae", "olive oil spray-14acad755753e316", "chopping board-900d57cdf92f63bf", "salad bowl-7812e76936f09e9a", "bag of veg1-b8039a9badbcb16d", "bag of veg2-23004bcc3f41c7f7", "yoghurt tub-4c150fd1cedd03bc", "basket-8d3768839e95bfb8", "bag of seeds-8b734610ce337b70", "salad wrap-61f251a3b93140fe", "salad sauce container-f01d60c741b2e113", "bowl-e51b9e55db0814c1", "fork-97971c1fe05c128e", "paper towel-0aed95828d20de25", "cutting board-399c76fa3e80b8b7", "carrot-38d20d10d4e3feb9", "peeler-86e54f7b628161a6", "knife-372134053789cc09", "carrot pieces-45f966cd439de62a", "oven mitt-0cfbb21e4c01ceb5", "kitchen tongs-29bb885641a88bfd", "avocado-847fec247a6b1809", "avocado half-3b456dd667014c9b", "small plate-91023bdac929f2a3", "small spoon-57979b1ff0890896", "avocado skin-9709b0800ca5f589", "tea towel2-2d82d76c7378662c", "pan-6a1fd7c0d40b130a", "bowl2-35430679c1f863c0", "tupperware-34116b9e84bee3fd", "coffee cup-cce2809ca16a7c6b", "top of coffee cup-243e0f96e49d6c6b", "knife5-ed40934fcb199b90", "knife3-74933a149441ba00", "spatulas-04717b621df81ddc", "tupperware2-89bcb463741f9eae", "tupperware lid2-eadc477c58c9e076", "tupperware3-a0e2e7f4be98ed7c", "tupperware4-7d797fb35ed7d34c", "small plate2-e21bfc195bd6cd66", "wine glass-d99dd0528a89697d", "cocktail mixer-4631f7ce7c1398bc", "utensils-58a427d5c84042af", "cocktail mixer part-84c109d936ca00d4", "cocktail mixer part2-eb9256108921f996", "cocktail mixer part3-b56220d415e3d4ce", "top of cocktail mixer-780362e075bf04d7", "fork2-0f3a164b492bed79", "spoon2-a244cee1efa6ceda", "sponge-13e9c2572a6aba5c", "washing up liquid bottle-ae579e7db2f0e519", "knife4-9408afbf9bd45759", "empty food container-27be1f90ab24a743", "head of kitchen brush-b54dde5748c92781", "kitchen towel roll-61d8606600acfca8", "knife holder-271c919bb46b4387", "mugs holder-ecfa28dea2a4348f", "clear bowl of vegetables-fe58b329441753ed" ], "reasoning": "1. I noticed the user grab a piece of leftover plastic wrap at 20:06.\n2. I noticed the user touch a trash bin lid at 20:09.\n3. I noticed the user touch the hand towel (tea towel2-2d82d76c7378662c) at 20:13.", "referenced_object_masks": [ "tea towel2-2d82d76c7378662c" ] }, { "key": "P08-20240614-125650:5dfa74af1e3ae2cff171b622337ea1925473c3fa", "video_id": "P08-20240614-125650", "question": "Once I cut the avocado in half, how many times do I slice the avocado further?", "answer_choice_0": "9 times veritcally 6 times horizontally", "answer_choice_1": "9 times diagonally, 3 times vertically", "answer_choice_2": "5 times vertically, 6 times diagonally", "answer_choice_3": "9 times vertically, 2 times horizonatally", "answer_choice_4": "8 times diagonally, 2 times vertically", "answer_id": 1, "answer": "9 times diagonally, 3 times vertically", "question_type": [ "Counting", "Action Recognition", "Event Occurence", "Situational Awareness", "State Changes" ], "objects_of_interest": [ "phone-5ffd04b506b53b9c", "baking tray1-147956c80da43fc2", "baking tray2-1e5294196e98a03b", "aluminum foil package-afa16ed3970c1ada", "foil paper piece-1327fa6f3ddacf44", "plastic bag with chicken thigh-08259e9113a28f33", "scale-e885a2e918e911e7", "tea towel-4d18a2f1b95f5543", "salt shaker-4072cc656246f435", "chicken thigh-7ef1f6dbcac6d8b7", "pepper shaker-03f2a01e6f436c4f", "kitchen brush-bfa18d58bd711aae", "olive oil spray-14acad755753e316", "chopping board-900d57cdf92f63bf", "salad bowl-7812e76936f09e9a", "bag of veg1-b8039a9badbcb16d", "bag of veg2-23004bcc3f41c7f7", "yoghurt tub-4c150fd1cedd03bc", "basket-8d3768839e95bfb8", "bag of seeds-8b734610ce337b70", "salad wrap-61f251a3b93140fe", "salad sauce container-f01d60c741b2e113", "bowl-e51b9e55db0814c1", "fork-97971c1fe05c128e", "paper towel-0aed95828d20de25", "cutting board-399c76fa3e80b8b7", "carrot-38d20d10d4e3feb9", "peeler-86e54f7b628161a6", "knife-372134053789cc09", "carrot pieces-45f966cd439de62a", "oven mitt-0cfbb21e4c01ceb5", "kitchen tongs-29bb885641a88bfd", "avocado-847fec247a6b1809", "avocado half-3b456dd667014c9b", "small plate-91023bdac929f2a3", "small spoon-57979b1ff0890896", "avocado skin-9709b0800ca5f589", "tea towel2-2d82d76c7378662c", "pan-6a1fd7c0d40b130a", "bowl2-35430679c1f863c0", "tupperware-34116b9e84bee3fd", "coffee cup-cce2809ca16a7c6b", "top of coffee cup-243e0f96e49d6c6b", "knife5-ed40934fcb199b90", "knife3-74933a149441ba00", "spatulas-04717b621df81ddc", "tupperware2-89bcb463741f9eae", "tupperware lid2-eadc477c58c9e076", "tupperware3-a0e2e7f4be98ed7c", "tupperware4-7d797fb35ed7d34c", "small plate2-e21bfc195bd6cd66", "wine glass-d99dd0528a89697d", "cocktail mixer-4631f7ce7c1398bc", "utensils-58a427d5c84042af", "cocktail mixer part-84c109d936ca00d4", "cocktail mixer part2-eb9256108921f996", "cocktail mixer part3-b56220d415e3d4ce", "top of cocktail mixer-780362e075bf04d7", "fork2-0f3a164b492bed79", "spoon2-a244cee1efa6ceda", "sponge-13e9c2572a6aba5c", "washing up liquid bottle-ae579e7db2f0e519", "knife4-9408afbf9bd45759", "empty food container-27be1f90ab24a743", "head of kitchen brush-b54dde5748c92781", "kitchen towel roll-61d8606600acfca8", "knife holder-271c919bb46b4387", "mugs holder-ecfa28dea2a4348f", "clear bowl of vegetables-fe58b329441753ed" ], "reasoning": "1. I noticed the user cut the avocado (avocado-847fec247a6b1809) in half at 13:39.\n2. I noticed the user cut the avocado half (avocado half-3b456dd667014c9b) 9 times diagonally from 13:41-13:49.\n3. I noticed the user cut the avocado half (avocado half-3b456dd667014c9b) 3 times vertically from 13:50-13:53.", "referenced_object_masks": [ "avocado half-3b456dd667014c9b", "avocado-847fec247a6b1809" ] }, { "key": "P08-20240614-125650:5ed7741e46571fd40ebad3cddd2ba869d4bd5f93", "video_id": "P08-20240614-125650", "question": "Which ingredient do I add to the salad immediately after I add the avocado?", "answer_choice_0": "Lettuce", "answer_choice_1": "Cheese", "answer_choice_2": "Salt", "answer_choice_3": "Pepper", "answer_choice_4": "Chicken", "answer_id": 2, "answer": "Salt", "question_type": [ "Egocentric & Spatial Perception", "Action Recognition", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "phone-5ffd04b506b53b9c", "baking tray1-147956c80da43fc2", "baking tray2-1e5294196e98a03b", "aluminum foil package-afa16ed3970c1ada", "foil paper piece-1327fa6f3ddacf44", "plastic bag with chicken thigh-08259e9113a28f33", "scale-e885a2e918e911e7", "tea towel-4d18a2f1b95f5543", "salt shaker-4072cc656246f435", "chicken thigh-7ef1f6dbcac6d8b7", "pepper shaker-03f2a01e6f436c4f", "kitchen brush-bfa18d58bd711aae", "olive oil spray-14acad755753e316", "chopping board-900d57cdf92f63bf", "salad bowl-7812e76936f09e9a", "bag of veg1-b8039a9badbcb16d", "bag of veg2-23004bcc3f41c7f7", "yoghurt tub-4c150fd1cedd03bc", "basket-8d3768839e95bfb8", "bag of seeds-8b734610ce337b70", "salad wrap-61f251a3b93140fe", "salad sauce container-f01d60c741b2e113", "bowl-e51b9e55db0814c1", "fork-97971c1fe05c128e", "paper towel-0aed95828d20de25", "cutting board-399c76fa3e80b8b7", "carrot-38d20d10d4e3feb9", "peeler-86e54f7b628161a6", "knife-372134053789cc09", "carrot pieces-45f966cd439de62a", "oven mitt-0cfbb21e4c01ceb5", "kitchen tongs-29bb885641a88bfd", "avocado-847fec247a6b1809", "avocado half-3b456dd667014c9b", "small plate-91023bdac929f2a3", "small spoon-57979b1ff0890896", "avocado skin-9709b0800ca5f589", "tea towel2-2d82d76c7378662c", "pan-6a1fd7c0d40b130a", "bowl2-35430679c1f863c0", "tupperware-34116b9e84bee3fd", "coffee cup-cce2809ca16a7c6b", "top of coffee cup-243e0f96e49d6c6b", "knife5-ed40934fcb199b90", "knife3-74933a149441ba00", "spatulas-04717b621df81ddc", "tupperware2-89bcb463741f9eae", "tupperware lid2-eadc477c58c9e076", "tupperware3-a0e2e7f4be98ed7c", "tupperware4-7d797fb35ed7d34c", "small plate2-e21bfc195bd6cd66", "wine glass-d99dd0528a89697d", "cocktail mixer-4631f7ce7c1398bc", "utensils-58a427d5c84042af", "cocktail mixer part-84c109d936ca00d4", "cocktail mixer part2-eb9256108921f996", "cocktail mixer part3-b56220d415e3d4ce", "top of cocktail mixer-780362e075bf04d7", "fork2-0f3a164b492bed79", "spoon2-a244cee1efa6ceda", "sponge-13e9c2572a6aba5c", "washing up liquid bottle-ae579e7db2f0e519", "knife4-9408afbf9bd45759", "empty food container-27be1f90ab24a743", "head of kitchen brush-b54dde5748c92781", "kitchen towel roll-61d8606600acfca8", "knife holder-271c919bb46b4387", "mugs holder-ecfa28dea2a4348f", "clear bowl of vegetables-fe58b329441753ed" ], "reasoning": "1. While watching the video, I observed the user put the avocado (avocado-847fec247a6b1809) in the bowl on top of the salad (salad bowl-7812e76936f09e9a) from 14:43-14:50.\n2. I then continued to watch and see what ingredient the user added next. 3. From 14:50-15:21, the user moves various dishes to the sink .\n4. Then from 15:21-15:25, I observed the user grab the salt grinder (salt shaker-4072cc656246f435) and grind salt onto the salad. This means that salt is the ingredient added right after the avocado is added.", "referenced_object_masks": [ "salt shaker-4072cc656246f435", "salad bowl-7812e76936f09e9a", "avocado-847fec247a6b1809" ] }, { "key": "P08-20240614-125650:d75c208b09cff47b1532b5bdde4778c12877776f", "video_id": "P08-20240614-125650", "question": "How many more times do I grind the pepper than I do the salt?", "answer_choice_0": "2", "answer_choice_1": "5", "answer_choice_2": "1", "answer_choice_3": "3", "answer_choice_4": "4", "answer_id": 3, "answer": "3", "question_type": [ "Action Recognition", "Object Recognition", "Numerical Reasoning", "Counting" ], "objects_of_interest": [ "phone-5ffd04b506b53b9c", "baking tray1-147956c80da43fc2", "baking tray2-1e5294196e98a03b", "aluminum foil package-afa16ed3970c1ada", "foil paper piece-1327fa6f3ddacf44", "plastic bag with chicken thigh-08259e9113a28f33", "scale-e885a2e918e911e7", "tea towel-4d18a2f1b95f5543", "salt shaker-4072cc656246f435", "chicken thigh-7ef1f6dbcac6d8b7", "pepper shaker-03f2a01e6f436c4f", "kitchen brush-bfa18d58bd711aae", "olive oil spray-14acad755753e316", "chopping board-900d57cdf92f63bf", "salad bowl-7812e76936f09e9a", "bag of veg1-b8039a9badbcb16d", "bag of veg2-23004bcc3f41c7f7", "yoghurt tub-4c150fd1cedd03bc", "basket-8d3768839e95bfb8", "bag of seeds-8b734610ce337b70", "salad wrap-61f251a3b93140fe", "salad sauce container-f01d60c741b2e113", "bowl-e51b9e55db0814c1", "fork-97971c1fe05c128e", "paper towel-0aed95828d20de25", "cutting board-399c76fa3e80b8b7", "carrot-38d20d10d4e3feb9", "peeler-86e54f7b628161a6", "knife-372134053789cc09", "carrot pieces-45f966cd439de62a", "oven mitt-0cfbb21e4c01ceb5", "kitchen tongs-29bb885641a88bfd", "avocado-847fec247a6b1809", "avocado half-3b456dd667014c9b", "small plate-91023bdac929f2a3", "small spoon-57979b1ff0890896", "avocado skin-9709b0800ca5f589", "tea towel2-2d82d76c7378662c", "pan-6a1fd7c0d40b130a", "bowl2-35430679c1f863c0", "tupperware-34116b9e84bee3fd", "coffee cup-cce2809ca16a7c6b", "top of coffee cup-243e0f96e49d6c6b", "knife5-ed40934fcb199b90", "knife3-74933a149441ba00", "spatulas-04717b621df81ddc", "tupperware2-89bcb463741f9eae", "tupperware lid2-eadc477c58c9e076", "tupperware3-a0e2e7f4be98ed7c", "tupperware4-7d797fb35ed7d34c", "small plate2-e21bfc195bd6cd66", "wine glass-d99dd0528a89697d", "cocktail mixer-4631f7ce7c1398bc", "utensils-58a427d5c84042af", "cocktail mixer part-84c109d936ca00d4", "cocktail mixer part2-eb9256108921f996", "cocktail mixer part3-b56220d415e3d4ce", "top of cocktail mixer-780362e075bf04d7", "fork2-0f3a164b492bed79", "spoon2-a244cee1efa6ceda", "sponge-13e9c2572a6aba5c", "washing up liquid bottle-ae579e7db2f0e519", "knife4-9408afbf9bd45759", "empty food container-27be1f90ab24a743", "head of kitchen brush-b54dde5748c92781", "kitchen towel roll-61d8606600acfca8", "knife holder-271c919bb46b4387", "mugs holder-ecfa28dea2a4348f", "clear bowl of vegetables-fe58b329441753ed" ], "reasoning": "1. I notice the user grab the salt shaker (salt shaker-4072cc656246f435) at 15:25. \n2. I noticed the user grind the salt shaker (salt shaker-4072cc656246f435) twice at 15:27.\n3. I noticed the user grab the pepper shaker (pepper shaker-03f2a01e6f436c4f) at 15:27.\n4. I noticed the user grind the pepper shaker (pepper shaker-03f2a01e6f436c4f) 5 times at 15:30.\n5. Subtracting 5-2, I noticed the user grind the pepper shaker (pepper shaker-03f2a01e6f436c4f) 3 more times than the salt (salt shaker-4072cc656246f435).", "referenced_object_masks": [ "salt shaker-4072cc656246f435", "pepper shaker-03f2a01e6f436c4f" ] }, { "key": "P08-20240614-125650:fad74ac7165df31e0cae7b2d6fa7c89598fa2fa0", "video_id": "P08-20240614-125650", "question": "While standing at the kitchen sink, what item would I walk into if I turned 90 degrees to the left and walked forward?", "answer_choice_0": "Door", "answer_choice_1": "Refrigerator", "answer_choice_2": "Couch", "answer_choice_3": "Dishwasher", "answer_choice_4": "Island", "answer_id": 2, "answer": "Couch", "question_type": [ "Counterfactual", "Egocentric & Spatial Perception", "Object Recognition" ], "objects_of_interest": [ "phone-5ffd04b506b53b9c", "baking tray1-147956c80da43fc2", "baking tray2-1e5294196e98a03b", "aluminum foil package-afa16ed3970c1ada", "foil paper piece-1327fa6f3ddacf44", "plastic bag with chicken thigh-08259e9113a28f33", "scale-e885a2e918e911e7", "tea towel-4d18a2f1b95f5543", "salt shaker-4072cc656246f435", "chicken thigh-7ef1f6dbcac6d8b7", "pepper shaker-03f2a01e6f436c4f", "kitchen brush-bfa18d58bd711aae", "olive oil spray-14acad755753e316", "chopping board-900d57cdf92f63bf", "salad bowl-7812e76936f09e9a", "bag of veg1-b8039a9badbcb16d", "bag of veg2-23004bcc3f41c7f7", "yoghurt tub-4c150fd1cedd03bc", "basket-8d3768839e95bfb8", "bag of seeds-8b734610ce337b70", "salad wrap-61f251a3b93140fe", "salad sauce container-f01d60c741b2e113", "bowl-e51b9e55db0814c1", "fork-97971c1fe05c128e", "paper towel-0aed95828d20de25", "cutting board-399c76fa3e80b8b7", "carrot-38d20d10d4e3feb9", "peeler-86e54f7b628161a6", "knife-372134053789cc09", "carrot pieces-45f966cd439de62a", "oven mitt-0cfbb21e4c01ceb5", "kitchen tongs-29bb885641a88bfd", "avocado-847fec247a6b1809", "avocado half-3b456dd667014c9b", "small plate-91023bdac929f2a3", "small spoon-57979b1ff0890896", "avocado skin-9709b0800ca5f589", "tea towel2-2d82d76c7378662c", "pan-6a1fd7c0d40b130a", "bowl2-35430679c1f863c0", "tupperware-34116b9e84bee3fd", "coffee cup-cce2809ca16a7c6b", "top of coffee cup-243e0f96e49d6c6b", "knife5-ed40934fcb199b90", "knife3-74933a149441ba00", "spatulas-04717b621df81ddc", "tupperware2-89bcb463741f9eae", "tupperware lid2-eadc477c58c9e076", "tupperware3-a0e2e7f4be98ed7c", "tupperware4-7d797fb35ed7d34c", "small plate2-e21bfc195bd6cd66", "wine glass-d99dd0528a89697d", "cocktail mixer-4631f7ce7c1398bc", "utensils-58a427d5c84042af", "cocktail mixer part-84c109d936ca00d4", "cocktail mixer part2-eb9256108921f996", "cocktail mixer part3-b56220d415e3d4ce", "top of cocktail mixer-780362e075bf04d7", "fork2-0f3a164b492bed79", "spoon2-a244cee1efa6ceda", "sponge-13e9c2572a6aba5c", "washing up liquid bottle-ae579e7db2f0e519", "knife4-9408afbf9bd45759", "empty food container-27be1f90ab24a743", "head of kitchen brush-b54dde5748c92781", "kitchen towel roll-61d8606600acfca8", "knife holder-271c919bb46b4387", "mugs holder-ecfa28dea2a4348f", "clear bowl of vegetables-fe58b329441753ed" ], "reasoning": "1. I watched the video and noted that from 00:40-00:45, the user walks past the sink on the right to the refrigerator , which is to the left of the sink.\n2. During this span, a couch is visible on the far left side of the room. \n3. I continued watching to confirm the spatial layout of the room, getting to 14:55, in which the user is standing near the sink and the couch is visible 90 degrees to the left.", "referenced_object_masks": [] }, { "key": "P08-20240614-125650:fcf34254f6d635c6c07c936cf5eb45a9dd7a071a", "video_id": "P08-20240614-125650", "question": "After I take the glass cup from the plate holder, how many items do I put into the metal cup?", "answer_choice_0": "5", "answer_choice_1": "6", "answer_choice_2": "3", "answer_choice_3": "7", "answer_choice_4": "4", "answer_id": 4, "answer": "4", "question_type": [ "Action Recognition", "Counting", "Event Occurence", "Numerical Reasoning" ], "objects_of_interest": [ "phone-5ffd04b506b53b9c", "baking tray1-147956c80da43fc2", "baking tray2-1e5294196e98a03b", "aluminum foil package-afa16ed3970c1ada", "foil paper piece-1327fa6f3ddacf44", "plastic bag with chicken thigh-08259e9113a28f33", "scale-e885a2e918e911e7", "tea towel-4d18a2f1b95f5543", "salt shaker-4072cc656246f435", "chicken thigh-7ef1f6dbcac6d8b7", "pepper shaker-03f2a01e6f436c4f", "kitchen brush-bfa18d58bd711aae", "olive oil spray-14acad755753e316", "chopping board-900d57cdf92f63bf", "salad bowl-7812e76936f09e9a", "bag of veg1-b8039a9badbcb16d", "bag of veg2-23004bcc3f41c7f7", "yoghurt tub-4c150fd1cedd03bc", "basket-8d3768839e95bfb8", "bag of seeds-8b734610ce337b70", "salad wrap-61f251a3b93140fe", "salad sauce container-f01d60c741b2e113", "bowl-e51b9e55db0814c1", "fork-97971c1fe05c128e", "paper towel-0aed95828d20de25", "cutting board-399c76fa3e80b8b7", "carrot-38d20d10d4e3feb9", "peeler-86e54f7b628161a6", "knife-372134053789cc09", "carrot pieces-45f966cd439de62a", "oven mitt-0cfbb21e4c01ceb5", "kitchen tongs-29bb885641a88bfd", "avocado-847fec247a6b1809", "avocado half-3b456dd667014c9b", "small plate-91023bdac929f2a3", "small spoon-57979b1ff0890896", "avocado skin-9709b0800ca5f589", "tea towel2-2d82d76c7378662c", "pan-6a1fd7c0d40b130a", "bowl2-35430679c1f863c0", "tupperware-34116b9e84bee3fd", "coffee cup-cce2809ca16a7c6b", "top of coffee cup-243e0f96e49d6c6b", "knife5-ed40934fcb199b90", "knife3-74933a149441ba00", "spatulas-04717b621df81ddc", "tupperware2-89bcb463741f9eae", "tupperware lid2-eadc477c58c9e076", "tupperware3-a0e2e7f4be98ed7c", "tupperware4-7d797fb35ed7d34c", "small plate2-e21bfc195bd6cd66", "wine glass-d99dd0528a89697d", "cocktail mixer-4631f7ce7c1398bc", "utensils-58a427d5c84042af", "cocktail mixer part-84c109d936ca00d4", "cocktail mixer part2-eb9256108921f996", "cocktail mixer part3-b56220d415e3d4ce", "top of cocktail mixer-780362e075bf04d7", "fork2-0f3a164b492bed79", "spoon2-a244cee1efa6ceda", "sponge-13e9c2572a6aba5c", "washing up liquid bottle-ae579e7db2f0e519", "knife4-9408afbf9bd45759", "empty food container-27be1f90ab24a743", "head of kitchen brush-b54dde5748c92781", "kitchen towel roll-61d8606600acfca8", "knife holder-271c919bb46b4387", "mugs holder-ecfa28dea2a4348f", "clear bowl of vegetables-fe58b329441753ed" ], "reasoning": "1. I noticed the user grab the wine glass (wine glass-d99dd0528a89697d) at 16:33. \n2. I noticed the user grab the metal cocktail mixer cup (cocktail mixer-4631f7ce7c1398bc) at 16:35. \n3. I noticed the user place the metal stick (utensils-58a427d5c84042af) into the metal cocktail mixer cup (cocktail mixer-4631f7ce7c1398bc) at 16:37. \n4. I noticed the user place the metal shot glass measure (cocktail mixer part2-eb9256108921f996) at 16:39 into the metal cocktail mixer cup (cocktail mixer-4631f7ce7c1398bc).\n5. I noticed the user place the metal stirrer (cocktail mixer part3-b56220d415e3d4ce) in the metal cocktail mixer cup (cocktail mixer-4631f7ce7c1398bc) at 16:41.\n6. I noticed the user place the fourth item, a round metal object (cocktail mixer part-84c109d936ca00d4), into the metal cocktail mixer cup (cocktail mixer-4631f7ce7c1398bc) at 16:43.", "referenced_object_masks": [ "cocktail mixer part3-b56220d415e3d4ce", "cocktail mixer part-84c109d936ca00d4", "utensils-58a427d5c84042af", "wine glass-d99dd0528a89697d", "cocktail mixer-4631f7ce7c1398bc", "cocktail mixer part2-eb9256108921f996" ] }, { "key": "P08-20240614-132938:2efe8b665db63132eab8bf8a1afcce21cf8b23ac", "video_id": "P08-20240614-132938", "question": "If I am standing at the stove, what what clock position is the trash?", "answer_choice_0": "1 o'clock", "answer_choice_1": "4 o'clock", "answer_choice_2": "12 o'clock", "answer_choice_3": "9 o'clock", "answer_choice_4": "6 o'clock", "answer_id": 1, "answer": "4 o'clock", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Counterfactual" ], "objects_of_interest": [ "oven mitt-1dcc670ebdf24101", "tea towel-43403be64a3b2df2", "baking tray-4611ccdc9ddf0128", "tongs-5ecc768d7d9a4e43", "knife-db1b9cf5af6977f3", "knife2-ecb0e4a36bb0e552", "fork-6c584052bf04d697", "plate-8a2f77da5ab2ec0f", "phone-3ecbf76b522b5511", "salad bowl-41ee27e6a12dd68e", "foil-c44d7b16a1aeb969", "sponge-544b494ea81e9a6b" ], "reasoning": "1. I watched the video to find what position that user is at when they are standing at the stove and found this position at 00:17.\n2. I then continued watching the video to find out where the trash can was and found this position at 01:21.\n3. It appears that when the user moves between their position at the stove at 01:18 and their position at the trash can at 01:21, they move to the right and slightly down, indicating that the trash is a 4 o'clock position relative to the stove (OB).", "referenced_object_masks": [] }, { "key": "P08-20240614-132938:3a55ba1e29cd0051e5ca27a1add686f1a545b51c", "video_id": "P08-20240614-132938", "question": "If facing the sink represents north, where is the oven mitt in relation to the sheet of foil while I'm washing the tongs?", "answer_choice_0": "South", "answer_choice_1": "North", "answer_choice_2": "Northwest", "answer_choice_3": "Southeast", "answer_choice_4": "East", "answer_id": 1, "answer": "North", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Counterfactual" ], "objects_of_interest": [ "oven mitt-1dcc670ebdf24101", "tea towel-43403be64a3b2df2", "baking tray-4611ccdc9ddf0128", "tongs-5ecc768d7d9a4e43", "knife-db1b9cf5af6977f3", "knife2-ecb0e4a36bb0e552", "fork-6c584052bf04d697", "plate-8a2f77da5ab2ec0f", "phone-3ecbf76b522b5511", "salad bowl-41ee27e6a12dd68e", "foil-c44d7b16a1aeb969", "sponge-544b494ea81e9a6b" ], "reasoning": "1. The user washes the tongs (tongs-5ecc768d7d9a4e43) in the sink from 02:01 to 02:14.\n2. I watched the video before this segment in order to find the location of the oven mitt (oven mitt-1dcc670ebdf24101) while the user washes the tongs. At 01:31, the user sets down the oven mitt (oven mitt-1dcc670ebdf24101) on the left edge of the stove.\n3. I watched the video before the user washes the tongs (tongs-5ecc768d7d9a4e43) to find the last location of the sheet of foil (foil-c44d7b16a1aeb969). From 01:18 to 01:21, the user walks to the trash can to throw away the sheet of foil (foil-c44d7b16a1aeb969). The trash can is located to the right of the stove.\n4. Next, I oriented the location of the sink to the stove. At 01:59, I noted that the user walked left from the stove to reach the sink.\n5. If facing the sink represented north, the stove where the oven mitt (oven mitt-1dcc670ebdf24101) is located would be east of the user. The foil (foil-c44d7b16a1aeb969), located in the trashcan when the user is washing the tongs (tongs-5ecc768d7d9a4e43), would be located south of the oven mitt (oven mitt-1dcc670ebdf24101). This means the oven mitt (oven mitt-1dcc670ebdf24101) is located north of the foil (foil-c44d7b16a1aeb969) during the time the user washes the tongs (tongs-5ecc768d7d9a4e43).", "referenced_object_masks": [ "foil-c44d7b16a1aeb969", "oven mitt-1dcc670ebdf24101", "tongs-5ecc768d7d9a4e43" ] }, { "key": "P08-20240614-132938:b1d71bc8d8408981f5b405ca4204f26c6b920b61", "video_id": "P08-20240614-132938", "question": "If I was standing facing the sink and I did a 180 degree turn, what would I be looking at?", "answer_choice_0": "The fridge", "answer_choice_1": "The trash can", "answer_choice_2": "The cabinet", "answer_choice_3": "The stove", "answer_choice_4": "The dishwasher", "answer_id": 1, "answer": "The trash can", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Counterfactual" ], "objects_of_interest": [ "oven mitt-1dcc670ebdf24101", "tea towel-43403be64a3b2df2", "baking tray-4611ccdc9ddf0128", "tongs-5ecc768d7d9a4e43", "knife-db1b9cf5af6977f3", "knife2-ecb0e4a36bb0e552", "fork-6c584052bf04d697", "plate-8a2f77da5ab2ec0f", "phone-3ecbf76b522b5511", "salad bowl-41ee27e6a12dd68e", "foil-c44d7b16a1aeb969", "sponge-544b494ea81e9a6b" ], "reasoning": "1. I watched the video to see what position that the user is at when they are standing at the sink and found this position at 02:00.\n2. I continued watching the video to see what would be opposite the sink if the user were to do a 180 degree turn and saw this angle at 01:19.\n3. The trash can is directly opposite the sink , so that's what the user would be looking at if they did a 180 degree turn.", "referenced_object_masks": [] }, { "key": "P08-20240614-132938:b876765761df60f49b9ddbf29430dd1c0cd3af45", "video_id": "P08-20240614-132938", "question": "If I switch the 8th magnet on top of the microwave with the 3rd magnet at the bottom of the microwave, what would be the color sequence of the magnets at the bottom?", "answer_choice_0": "White, yellow, brown, green, blue", "answer_choice_1": "White, yellow, black, green, blue", "answer_choice_2": "Black, yellow, black, blue, green", "answer_choice_3": "Brown, brown, black, yellow, green", "answer_choice_4": "Black, red, yellow, green, blue", "answer_id": 1, "answer": "White, yellow, black, green, blue", "question_type": [ "Counting", "Object Recognition", "Numerical Reasoning" ], "objects_of_interest": [ "oven mitt-1dcc670ebdf24101", "tea towel-43403be64a3b2df2", "baking tray-4611ccdc9ddf0128", "tongs-5ecc768d7d9a4e43", "knife-db1b9cf5af6977f3", "knife2-ecb0e4a36bb0e552", "fork-6c584052bf04d697", "plate-8a2f77da5ab2ec0f", "phone-3ecbf76b522b5511", "salad bowl-41ee27e6a12dd68e", "foil-c44d7b16a1aeb969", "sponge-544b494ea81e9a6b" ], "reasoning": "1. I watched the video from the beginning to locate the microwave in the kitchen, which I saw immediately at 00:00. \n2. I noticed the magnets on the microwave's top and bottom borders at 00:00, but only a few magnets were visible in the frame.\n3. I continued to watch, searching for a better view of the microwave , which I found at 01:22, when the user is standing in front of it. I paused the video and zoomed in. \n4. I counted the number of magnets at the top of the microwave to find the 8th one. The 8th magnet is of a black car. \n5. I counted the number of magnets at the bottom of the microwave to locate the 3rd one, which is an \"I LOVE NY\" magnet with a predominantly white background. \n6. If the user switches the black car magnet with the \"I LOVE NY\" magnet, then the black car magnet would be the 3rd magnet at the bottom of the microwave .\n7. I concluded that the predominant colors of the magnets at the bottom of the microwave if the black car magnet is switched with the \"I LOVE NY\" magnet would be: White, Yellow, Black, Green, and Blue.", "referenced_object_masks": [] }, { "key": "P08-20240614-132938:f6112c5ad02d5e8e7742fe26429c037300b20ca4", "video_id": "P08-20240614-132938", "question": "How many times did I use my oven mitt to move something hot?", "answer_choice_0": "Thrice", "answer_choice_1": "4 times", "answer_choice_2": "Twice", "answer_choice_3": "Once", "answer_choice_4": "5 times", "answer_id": 2, "answer": "Twice", "question_type": [ "Counting", "Object Recognition" ], "objects_of_interest": [ "oven mitt-1dcc670ebdf24101", "tea towel-43403be64a3b2df2", "baking tray-4611ccdc9ddf0128", "tongs-5ecc768d7d9a4e43", "knife-db1b9cf5af6977f3", "knife2-ecb0e4a36bb0e552", "fork-6c584052bf04d697", "plate-8a2f77da5ab2ec0f", "phone-3ecbf76b522b5511", "salad bowl-41ee27e6a12dd68e", "foil-c44d7b16a1aeb969", "sponge-544b494ea81e9a6b" ], "reasoning": "1. I watched the video to identify the oven mitt (oven mitt-1dcc670ebdf24101) and found it at 00:02.\n2. The first time I observed the user using the oven mitt (oven mitt-1dcc670ebdf24101) to move something hot was when the user used it to remove the baking tray (baking tray-4611ccdc9ddf0128) from the oven at 00:10.\n3. The second time I observed the user using the oven mitt (oven mitt-1dcc670ebdf24101) to move something hot was when the user used it to move the still hot baking tray (baking tray-4611ccdc9ddf0128) over at 01:27.\n4. Therefore, the user used the oven mitt (oven mitt-1dcc670ebdf24101) to move the baking tray (baking tray-4611ccdc9ddf0128) twice.", "referenced_object_masks": [ "oven mitt-1dcc670ebdf24101", "baking tray-4611ccdc9ddf0128" ] }, { "key": "P08-20240614-184945:06ec109bd5d5cbe6e913379698c49c054d3d707c", "video_id": "P08-20240614-184945", "question": "What am I most likely doing when I gesture with my hand right before I rinse out the small bowl and leave it in the sink?", "answer_choice_0": "Showing my guest the way to the bathroom", "answer_choice_1": "Counting the books on the shelf above the TV", "answer_choice_2": "Checking my finger for any cuts", "answer_choice_3": "Communicating with my cat", "answer_choice_4": "Pointing at my brand new big screen TV", "answer_id": 3, "answer": "Communicating with my cat", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Object Recognition", "Goal Reasoning" ], "objects_of_interest": [ "salmon skewer packaging-36dd2f2bb37b566d", "pouch of food-1e0bf176e3e6739a", "baking tray-de9529d43407b24d", "box of kitchen foil-b3321cc93c9e41c6", "baking tray2-eaf65a93cb24242c", "phone-fa69c89bd937f66f", "cardboard sleeve-36b0d8e8ecd28dd7", "piece of foil-a6b3010a8c96fd68", "knife-b820c749f54a6766", "skewer1-8d7ad8cf8ef75acd", "skewer2-f6e0d2b9e2dddb20", "skewer3-269db417e9ddfa2c", "air fryer-fedb08ae74b125c3", "bag of frozen broccoli-e1978bb4ed53e367", "weighing scales-7cda857c0ddeb106", "scissors-0fca98c44bb56811", "broccoli1-698ab4482752b8b1", "broccoli2-76b2eb14b6b98bca", "broccoli3-ec252c7f100e00d6", "broccoli4-859f245ed08e8772", "broccoli5-b4d4bc1c9cc3c303", "air fryer drum-4c0d92179d8e6e77", "piece of plastic-bb26188a50aa9dd0", "paper towel piece-4b035530e7d77e07", "piece of plastic2-c70d83d41e315ce0", "piece of plastic3-65f68ec067da9227", "bowl-0c30320f366aabec", "fork-d538b70af2985b1d", "plate-99745b85dc4011d0", "clip-8142619a137c8692", "salt shaker-20a8dbf856cdb383", "pepper grinder-d84a5b71b1bfe460", "oil-64e976c2baa181ed", "oven mitt-5452742ece417cba", "brush-25c89c842b5f82b3", "sponge-ae72e981a1155186", "bottle of washing up liquid-2bb58495cb26336d", "piece of plastic4-545e03bdf8a4b6c3", "chopping board-7124bc6ce86b9c0e", "tongs-ac68fcd3de9449df", "spoon-3d3eb2c1fbfe560c", "knife2-3a5f16f9f890e73a", "small plate-d76568994bd60227", "peeler-c81dd2a0c2adc4c0", "flask-383c84b3294aa17a", "seasoning-f8f9407553280ef9", "small spoon-1f5f9573a64410f9", "lid of seasoning-59abf75a850cdbbe", "vent-6c5495c3031dd6f6", "tea towel-28427e92b7a003c3", "cutlery tray-72dca017d6c19fd2" ], "reasoning": "1. I began by looking for when the user gives a hand gesture right before rinsing a bowl (bowl-0c30320f366aabec) and leaving it in the sink. This happens at 06:55-06:58 and doesn't happen again in the video.\n2. From here, I studied the hand gesture at 06:55. The answer options all point to objects in the frame with the exception of any \"guest\" save the cat. The user is also pointing in the general direction of the cat at that moment. \n3. Finally, I watched around that time stamp to look for clues as to what the hand gesture could mean. At 07:11 the cat can be seen walking away, and at 06:31, 06:36 and 06:47 increasingly louder and more persistent meowing can be heard in the background while the user prepares dinner. The user is thus most likely communicating something to the cat.", "referenced_object_masks": [ "bowl-0c30320f366aabec" ] }, { "key": "P08-20240614-184945:22a38f29eb6b400e16f7fa0f6d47aca83f8c2d82", "video_id": "P08-20240614-184945", "question": "Between opening the fridge and adjusting the oven dial, how many unique items do I touch with my right hand?", "answer_choice_0": "6", "answer_choice_1": "3", "answer_choice_2": "4", "answer_choice_3": "7", "answer_choice_4": "5", "answer_id": 2, "answer": "4", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counting" ], "objects_of_interest": [ "salmon skewer packaging-36dd2f2bb37b566d", "pouch of food-1e0bf176e3e6739a", "baking tray-de9529d43407b24d", "box of kitchen foil-b3321cc93c9e41c6", "baking tray2-eaf65a93cb24242c", "phone-fa69c89bd937f66f", "cardboard sleeve-36b0d8e8ecd28dd7", "piece of foil-a6b3010a8c96fd68", "knife-b820c749f54a6766", "skewer1-8d7ad8cf8ef75acd", "skewer2-f6e0d2b9e2dddb20", "skewer3-269db417e9ddfa2c", "air fryer-fedb08ae74b125c3", "bag of frozen broccoli-e1978bb4ed53e367", "weighing scales-7cda857c0ddeb106", "scissors-0fca98c44bb56811", "broccoli1-698ab4482752b8b1", "broccoli2-76b2eb14b6b98bca", "broccoli3-ec252c7f100e00d6", "broccoli4-859f245ed08e8772", "broccoli5-b4d4bc1c9cc3c303", "air fryer drum-4c0d92179d8e6e77", "piece of plastic-bb26188a50aa9dd0", "paper towel piece-4b035530e7d77e07", "piece of plastic2-c70d83d41e315ce0", "piece of plastic3-65f68ec067da9227", "bowl-0c30320f366aabec", "fork-d538b70af2985b1d", "plate-99745b85dc4011d0", "clip-8142619a137c8692", "salt shaker-20a8dbf856cdb383", "pepper grinder-d84a5b71b1bfe460", "oil-64e976c2baa181ed", "oven mitt-5452742ece417cba", "brush-25c89c842b5f82b3", "sponge-ae72e981a1155186", "bottle of washing up liquid-2bb58495cb26336d", "piece of plastic4-545e03bdf8a4b6c3", "chopping board-7124bc6ce86b9c0e", "tongs-ac68fcd3de9449df", "spoon-3d3eb2c1fbfe560c", "knife2-3a5f16f9f890e73a", "small plate-d76568994bd60227", "peeler-c81dd2a0c2adc4c0", "flask-383c84b3294aa17a", "seasoning-f8f9407553280ef9", "small spoon-1f5f9573a64410f9", "lid of seasoning-59abf75a850cdbbe", "vent-6c5495c3031dd6f6", "tea towel-28427e92b7a003c3", "cutlery tray-72dca017d6c19fd2" ], "reasoning": "1. I observe the user opening the fridge in front of them and slightly to the right at 00:07. \n2. At 00:08, I observe the user remove both the salmon (salmon skewer packaging-36dd2f2bb37b566d) and the pouch of food (pouch of food-1e0bf176e3e6739a) from the fridge with their right hand.\n3. At 0:17, I observe the user flipping a switch above the countertop and grazing a whisk with their right hand.\n4. At 0:19, I observe the user adjust the oven dial .\n5. I determine that the user touches 4 unique objects with their right hand between opening the fridge and adjusting the oven dial .", "referenced_object_masks": [ "salmon skewer packaging-36dd2f2bb37b566d", "pouch of food-1e0bf176e3e6739a" ] }, { "key": "P08-20240614-184945:256429669e221789cc006a6d4a5ed6b09f890b7d", "video_id": "P08-20240614-184945", "question": "Based on where the skewer position points, in what order do I place the salmon on the baking tray?", "answer_choice_0": "down, down, up, down", "answer_choice_1": "up, down, up, down", "answer_choice_2": "down, down, up, up", "answer_choice_3": "up, up, down, up", "answer_choice_4": "down, up, down, up", "answer_id": 2, "answer": "down, down, up, up", "question_type": [ "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counting" ], "objects_of_interest": [ "salmon skewer packaging-36dd2f2bb37b566d", "pouch of food-1e0bf176e3e6739a", "baking tray-de9529d43407b24d", "box of kitchen foil-b3321cc93c9e41c6", "baking tray2-eaf65a93cb24242c", "phone-fa69c89bd937f66f", "cardboard sleeve-36b0d8e8ecd28dd7", "piece of foil-a6b3010a8c96fd68", "knife-b820c749f54a6766", "skewer1-8d7ad8cf8ef75acd", "skewer2-f6e0d2b9e2dddb20", "skewer3-269db417e9ddfa2c", "air fryer-fedb08ae74b125c3", "bag of frozen broccoli-e1978bb4ed53e367", "weighing scales-7cda857c0ddeb106", "scissors-0fca98c44bb56811", "broccoli1-698ab4482752b8b1", "broccoli2-76b2eb14b6b98bca", "broccoli3-ec252c7f100e00d6", "broccoli4-859f245ed08e8772", "broccoli5-b4d4bc1c9cc3c303", "air fryer drum-4c0d92179d8e6e77", "piece of plastic-bb26188a50aa9dd0", "paper towel piece-4b035530e7d77e07", "piece of plastic2-c70d83d41e315ce0", "piece of plastic3-65f68ec067da9227", "bowl-0c30320f366aabec", "fork-d538b70af2985b1d", "plate-99745b85dc4011d0", "clip-8142619a137c8692", "salt shaker-20a8dbf856cdb383", "pepper grinder-d84a5b71b1bfe460", "oil-64e976c2baa181ed", "oven mitt-5452742ece417cba", "brush-25c89c842b5f82b3", "sponge-ae72e981a1155186", "bottle of washing up liquid-2bb58495cb26336d", "piece of plastic4-545e03bdf8a4b6c3", "chopping board-7124bc6ce86b9c0e", "tongs-ac68fcd3de9449df", "spoon-3d3eb2c1fbfe560c", "knife2-3a5f16f9f890e73a", "small plate-d76568994bd60227", "peeler-c81dd2a0c2adc4c0", "flask-383c84b3294aa17a", "seasoning-f8f9407553280ef9", "small spoon-1f5f9573a64410f9", "lid of seasoning-59abf75a850cdbbe", "vent-6c5495c3031dd6f6", "tea towel-28427e92b7a003c3", "cutlery tray-72dca017d6c19fd2" ], "reasoning": "1. From 01:38 to 01:56, I observe the user transfer the contents of the package of salmon (salmon skewer packaging-36dd2f2bb37b566d) to the baking tray (baking tray2-eaf65a93cb24242c).\n2. At 01:38, I observe the user place the first salmon skewer (skewer1-8d7ad8cf8ef75acd) with the skewer pointed downwards.\n3. At 01:43, I observe the user place the second salmon skewer (skewer2-f6e0d2b9e2dddb20) with the skewer pointed downwards.\n4. From 01:47 to 01:56, I observe the user place the third and fourth salmon skewers (skewer3-269db417e9ddfa2c) with their skewers pointed upwards.\n5. I determine that the placement order based on skewer direction is down, down, up, up.", "referenced_object_masks": [ "baking tray2-eaf65a93cb24242c", "skewer2-f6e0d2b9e2dddb20", "salmon skewer packaging-36dd2f2bb37b566d", "skewer3-269db417e9ddfa2c", "skewer1-8d7ad8cf8ef75acd" ] }, { "key": "P08-20240614-184945:5e653fc0e0a020ec5194bb43b7860ce5340f2496", "video_id": "P08-20240614-184945", "question": "Assuming that I kept tea leaves in the right cabinet above the stove, how many turns would I have to take to get the tea leaves, a cup, the electric tea kettle and water in that order if I were standing in front of the trash bin on the right?", "answer_choice_0": "3", "answer_choice_1": "5", "answer_choice_2": "4", "answer_choice_3": "6", "answer_choice_4": "7", "answer_id": 2, "answer": "4", "question_type": [ "Counterfactual", "Counting", "Egocentric & Spatial Perception", "Goal Reasoning", "Object Recognition" ], "objects_of_interest": [ "salmon skewer packaging-36dd2f2bb37b566d", "pouch of food-1e0bf176e3e6739a", "baking tray-de9529d43407b24d", "box of kitchen foil-b3321cc93c9e41c6", "baking tray2-eaf65a93cb24242c", "phone-fa69c89bd937f66f", "cardboard sleeve-36b0d8e8ecd28dd7", "piece of foil-a6b3010a8c96fd68", "knife-b820c749f54a6766", "skewer1-8d7ad8cf8ef75acd", "skewer2-f6e0d2b9e2dddb20", "skewer3-269db417e9ddfa2c", "air fryer-fedb08ae74b125c3", "bag of frozen broccoli-e1978bb4ed53e367", "weighing scales-7cda857c0ddeb106", "scissors-0fca98c44bb56811", "broccoli1-698ab4482752b8b1", "broccoli2-76b2eb14b6b98bca", "broccoli3-ec252c7f100e00d6", "broccoli4-859f245ed08e8772", "broccoli5-b4d4bc1c9cc3c303", "air fryer drum-4c0d92179d8e6e77", "piece of plastic-bb26188a50aa9dd0", "paper towel piece-4b035530e7d77e07", "piece of plastic2-c70d83d41e315ce0", "piece of plastic3-65f68ec067da9227", "bowl-0c30320f366aabec", "fork-d538b70af2985b1d", "plate-99745b85dc4011d0", "clip-8142619a137c8692", "salt shaker-20a8dbf856cdb383", "pepper grinder-d84a5b71b1bfe460", "oil-64e976c2baa181ed", "oven mitt-5452742ece417cba", "brush-25c89c842b5f82b3", "sponge-ae72e981a1155186", "bottle of washing up liquid-2bb58495cb26336d", "piece of plastic4-545e03bdf8a4b6c3", "chopping board-7124bc6ce86b9c0e", "tongs-ac68fcd3de9449df", "spoon-3d3eb2c1fbfe560c", "knife2-3a5f16f9f890e73a", "small plate-d76568994bd60227", "peeler-c81dd2a0c2adc4c0", "flask-383c84b3294aa17a", "seasoning-f8f9407553280ef9", "small spoon-1f5f9573a64410f9", "lid of seasoning-59abf75a850cdbbe", "vent-6c5495c3031dd6f6", "tea towel-28427e92b7a003c3", "cutlery tray-72dca017d6c19fd2" ], "reasoning": "1. I began by looking for a cup , the electric kettle and the sink for water, and noted where these were located in relation to the trash can and the imaginary tea leaves. At 04:39 I could see that cups and the electric kettle were located in the same corner to the right of the sink.\n2. I then looked for a clear view of the trash bin in question, which I saw clearly at 10:15.\n3. From here I imagined the steps and turns the user would have to take to get the cupboard with the imaginary tea leaves, then to a cup and the electric kettle , and finally to water from the tap: 1.) make a sharp left to face the cupboard with the leaves, 2.) turn left to get a cup and the electric kettle , 3.) turn left again toward the sink, 4.) turn right to face the sink to fill the electric kettle. These are all the steps necessary for collecting the ingredients, dishes and cookware specified, meaning that the user would need to make 4 total turns.", "referenced_object_masks": [] }, { "key": "P08-20240614-184945:5ec6eab1ea0c590c799614b46634e60b48969d74", "video_id": "P08-20240614-184945", "question": "Where are the paper towels in relation to me when I put the skewers in the oven?", "answer_choice_0": "Behind and to the left", "answer_choice_1": "Above and to the left", "answer_choice_2": "Below and to the left", "answer_choice_3": "Below and to the right", "answer_choice_4": "Behind and to the right", "answer_id": 0, "answer": "Behind and to the left", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Object Recognition" ], "objects_of_interest": [ "salmon skewer packaging-36dd2f2bb37b566d", "pouch of food-1e0bf176e3e6739a", "baking tray-de9529d43407b24d", "box of kitchen foil-b3321cc93c9e41c6", "baking tray2-eaf65a93cb24242c", "phone-fa69c89bd937f66f", "cardboard sleeve-36b0d8e8ecd28dd7", "piece of foil-a6b3010a8c96fd68", "knife-b820c749f54a6766", "skewer1-8d7ad8cf8ef75acd", "skewer2-f6e0d2b9e2dddb20", "skewer3-269db417e9ddfa2c", "air fryer-fedb08ae74b125c3", "bag of frozen broccoli-e1978bb4ed53e367", "weighing scales-7cda857c0ddeb106", "scissors-0fca98c44bb56811", "broccoli1-698ab4482752b8b1", "broccoli2-76b2eb14b6b98bca", "broccoli3-ec252c7f100e00d6", "broccoli4-859f245ed08e8772", "broccoli5-b4d4bc1c9cc3c303", "air fryer drum-4c0d92179d8e6e77", "piece of plastic-bb26188a50aa9dd0", "paper towel piece-4b035530e7d77e07", "piece of plastic2-c70d83d41e315ce0", "piece of plastic3-65f68ec067da9227", "bowl-0c30320f366aabec", "fork-d538b70af2985b1d", "plate-99745b85dc4011d0", "clip-8142619a137c8692", "salt shaker-20a8dbf856cdb383", "pepper grinder-d84a5b71b1bfe460", "oil-64e976c2baa181ed", "oven mitt-5452742ece417cba", "brush-25c89c842b5f82b3", "sponge-ae72e981a1155186", "bottle of washing up liquid-2bb58495cb26336d", "piece of plastic4-545e03bdf8a4b6c3", "chopping board-7124bc6ce86b9c0e", "tongs-ac68fcd3de9449df", "spoon-3d3eb2c1fbfe560c", "knife2-3a5f16f9f890e73a", "small plate-d76568994bd60227", "peeler-c81dd2a0c2adc4c0", "flask-383c84b3294aa17a", "seasoning-f8f9407553280ef9", "small spoon-1f5f9573a64410f9", "lid of seasoning-59abf75a850cdbbe", "vent-6c5495c3031dd6f6", "tea towel-28427e92b7a003c3", "cutlery tray-72dca017d6c19fd2" ], "reasoning": "1. I began by looking for where the user keeps the paper towels (paper towel piece-4b035530e7d77e07). At 04:42 the paper towels are clearly visible between the sink and fridge.\n2. I then waited for the user to put the skewers (skewer1-8d7ad8cf8ef75acd, skewer2-f6e0d2b9e2dddb20, skewer3-269db417e9ddfa2c, piece of foil-a6b3010a8c96fd68, baking tray-de9529d43407b24d) into the oven, which happens at 02:12.\n3. From here, I visualized where the paper towels (paper towel piece-4b035530e7d77e07) were in relation to the user standing directly in front of the oven: behind and to the left.", "referenced_object_masks": [ "skewer1-8d7ad8cf8ef75acd", "skewer2-f6e0d2b9e2dddb20", "skewer3-269db417e9ddfa2c", "piece of foil-a6b3010a8c96fd68", "baking tray-de9529d43407b24d", "paper towel piece-4b035530e7d77e07" ] }, { "key": "P08-20240614-184945:6e30563f1579abe291c02db1c259ca8e7edfd511", "video_id": "P08-20240614-184945", "question": "After plugging the air fryer into the outlet, what is the third object I touch with my left hand?", "answer_choice_0": "Countertop", "answer_choice_1": "Fridge drawer", "answer_choice_2": "Broccoli", "answer_choice_3": "Package of salmon", "answer_choice_4": "Air fryer", "answer_id": 1, "answer": "Fridge drawer", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counting" ], "objects_of_interest": [ "salmon skewer packaging-36dd2f2bb37b566d", "pouch of food-1e0bf176e3e6739a", "baking tray-de9529d43407b24d", "box of kitchen foil-b3321cc93c9e41c6", "baking tray2-eaf65a93cb24242c", "phone-fa69c89bd937f66f", "cardboard sleeve-36b0d8e8ecd28dd7", "piece of foil-a6b3010a8c96fd68", "knife-b820c749f54a6766", "skewer1-8d7ad8cf8ef75acd", "skewer2-f6e0d2b9e2dddb20", "skewer3-269db417e9ddfa2c", "air fryer-fedb08ae74b125c3", "bag of frozen broccoli-e1978bb4ed53e367", "weighing scales-7cda857c0ddeb106", "scissors-0fca98c44bb56811", "broccoli1-698ab4482752b8b1", "broccoli2-76b2eb14b6b98bca", "broccoli3-ec252c7f100e00d6", "broccoli4-859f245ed08e8772", "broccoli5-b4d4bc1c9cc3c303", "air fryer drum-4c0d92179d8e6e77", "piece of plastic-bb26188a50aa9dd0", "paper towel piece-4b035530e7d77e07", "piece of plastic2-c70d83d41e315ce0", "piece of plastic3-65f68ec067da9227", "bowl-0c30320f366aabec", "fork-d538b70af2985b1d", "plate-99745b85dc4011d0", "clip-8142619a137c8692", "salt shaker-20a8dbf856cdb383", "pepper grinder-d84a5b71b1bfe460", "oil-64e976c2baa181ed", "oven mitt-5452742ece417cba", "brush-25c89c842b5f82b3", "sponge-ae72e981a1155186", "bottle of washing up liquid-2bb58495cb26336d", "piece of plastic4-545e03bdf8a4b6c3", "chopping board-7124bc6ce86b9c0e", "tongs-ac68fcd3de9449df", "spoon-3d3eb2c1fbfe560c", "knife2-3a5f16f9f890e73a", "small plate-d76568994bd60227", "peeler-c81dd2a0c2adc4c0", "flask-383c84b3294aa17a", "seasoning-f8f9407553280ef9", "small spoon-1f5f9573a64410f9", "lid of seasoning-59abf75a850cdbbe", "vent-6c5495c3031dd6f6", "tea towel-28427e92b7a003c3", "cutlery tray-72dca017d6c19fd2" ], "reasoning": "1. At 02:30, I observe the user plug the air fryer (air fryer-fedb08ae74b125c3) into the outlet.\n2. At 02:31, I observe the user push against the air fryer (air fryer-fedb08ae74b125c3) with their left hand.\n3. At 02:34, I observe the user touching the fridge door handle with their left hand.\n4. At 02:39, the user pushes in the fridge drawer with their left hand. This is the third object the user touches with their left hand.", "referenced_object_masks": [ "air fryer-fedb08ae74b125c3" ] }, { "key": "P08-20240614-184945:70e1a6bcb025899170d6bba5d18aad241676bc8d", "video_id": "P08-20240614-184945", "question": "How many times does the spoon tap against the opened seasoning jar right before I measure the seasoning to be added to the broccoli after taking it out of the air fryer for the fourth time?", "answer_choice_0": "2 times", "answer_choice_1": "6 times", "answer_choice_2": "4 times", "answer_choice_3": "5 times", "answer_choice_4": "3 times", "answer_id": 2, "answer": "4 times", "question_type": [ "Action Recognition", "Counting", "Object Recognition", "State Changes", "Audio Reasoning", "Event Occurence" ], "objects_of_interest": [ "salmon skewer packaging-36dd2f2bb37b566d", "pouch of food-1e0bf176e3e6739a", "baking tray-de9529d43407b24d", "box of kitchen foil-b3321cc93c9e41c6", "baking tray2-eaf65a93cb24242c", "phone-fa69c89bd937f66f", "cardboard sleeve-36b0d8e8ecd28dd7", "piece of foil-a6b3010a8c96fd68", "knife-b820c749f54a6766", "skewer1-8d7ad8cf8ef75acd", "skewer2-f6e0d2b9e2dddb20", "skewer3-269db417e9ddfa2c", "air fryer-fedb08ae74b125c3", "bag of frozen broccoli-e1978bb4ed53e367", "weighing scales-7cda857c0ddeb106", "scissors-0fca98c44bb56811", "broccoli1-698ab4482752b8b1", "broccoli2-76b2eb14b6b98bca", "broccoli3-ec252c7f100e00d6", "broccoli4-859f245ed08e8772", "broccoli5-b4d4bc1c9cc3c303", "air fryer drum-4c0d92179d8e6e77", "piece of plastic-bb26188a50aa9dd0", "paper towel piece-4b035530e7d77e07", "piece of plastic2-c70d83d41e315ce0", "piece of plastic3-65f68ec067da9227", "bowl-0c30320f366aabec", "fork-d538b70af2985b1d", "plate-99745b85dc4011d0", "clip-8142619a137c8692", "salt shaker-20a8dbf856cdb383", "pepper grinder-d84a5b71b1bfe460", "oil-64e976c2baa181ed", "oven mitt-5452742ece417cba", "brush-25c89c842b5f82b3", "sponge-ae72e981a1155186", "bottle of washing up liquid-2bb58495cb26336d", "piece of plastic4-545e03bdf8a4b6c3", "chopping board-7124bc6ce86b9c0e", "tongs-ac68fcd3de9449df", "spoon-3d3eb2c1fbfe560c", "knife2-3a5f16f9f890e73a", "small plate-d76568994bd60227", "peeler-c81dd2a0c2adc4c0", "flask-383c84b3294aa17a", "seasoning-f8f9407553280ef9", "small spoon-1f5f9573a64410f9", "lid of seasoning-59abf75a850cdbbe", "vent-6c5495c3031dd6f6", "tea towel-28427e92b7a003c3", "cutlery tray-72dca017d6c19fd2" ], "reasoning": "1. I began by looking for when the user takes the broccoli (broccoli1-698ab4482752b8b1, broccoli2-76b2eb14b6b98bca, broccoli3-ec252c7f100e00d6, broccoli4-859f245ed08e8772, broccoli5-b4d4bc1c9cc3c303) out of the air fryer (air fryer-fedb08ae74b125c3, air fryer drum-4c0d92179d8e6e77) for the first four times, which happens at 07:40, 09:35, 10:52 and 11:43.\n2. From here I waited for the user to measure seasoning (seasoning-f8f9407553280ef9) with a spoon (small spoon-1f5f9573a64410f9), which happens at 12:00.\n3. I then slowed the video down to listen for the number of times the spoon (small spoon-1f5f9573a64410f9) taps against the jar (seasoning-f8f9407553280ef9) once opened: I heard 2 taps at 12:00 and 2 taps at 12:01, which is also when the user begins to measure the seasoning (seasoning-f8f9407553280ef9): hence 4 taps total.", "referenced_object_masks": [ "small spoon-1f5f9573a64410f9", "seasoning-f8f9407553280ef9", "air fryer-fedb08ae74b125c3", "air fryer drum-4c0d92179d8e6e77", "broccoli1-698ab4482752b8b1", "broccoli2-76b2eb14b6b98bca", "broccoli3-ec252c7f100e00d6", "broccoli4-859f245ed08e8772", "broccoli5-b4d4bc1c9cc3c303" ] }, { "key": "P08-20240614-184945:7142f0a1726bff37f45a161fe639c6e59765dd42", "video_id": "P08-20240614-184945", "question": "What is the difference between the number on the oven's display when I put the salmon in the oven for the last time and the number on the oven's display right before placing the oven tray on the stove for the first time?", "answer_choice_0": "5", "answer_choice_1": "7", "answer_choice_2": "10", "answer_choice_3": "12", "answer_choice_4": "4", "answer_id": 3, "answer": "12", "question_type": [ "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counting", "Numerical Reasoning" ], "objects_of_interest": [ "salmon skewer packaging-36dd2f2bb37b566d", "pouch of food-1e0bf176e3e6739a", "baking tray-de9529d43407b24d", "box of kitchen foil-b3321cc93c9e41c6", "baking tray2-eaf65a93cb24242c", "phone-fa69c89bd937f66f", "cardboard sleeve-36b0d8e8ecd28dd7", "piece of foil-a6b3010a8c96fd68", "knife-b820c749f54a6766", "skewer1-8d7ad8cf8ef75acd", "skewer2-f6e0d2b9e2dddb20", "skewer3-269db417e9ddfa2c", "air fryer-fedb08ae74b125c3", "bag of frozen broccoli-e1978bb4ed53e367", "weighing scales-7cda857c0ddeb106", "scissors-0fca98c44bb56811", "broccoli1-698ab4482752b8b1", "broccoli2-76b2eb14b6b98bca", "broccoli3-ec252c7f100e00d6", "broccoli4-859f245ed08e8772", "broccoli5-b4d4bc1c9cc3c303", "air fryer drum-4c0d92179d8e6e77", "piece of plastic-bb26188a50aa9dd0", "paper towel piece-4b035530e7d77e07", "piece of plastic2-c70d83d41e315ce0", "piece of plastic3-65f68ec067da9227", "bowl-0c30320f366aabec", "fork-d538b70af2985b1d", "plate-99745b85dc4011d0", "clip-8142619a137c8692", "salt shaker-20a8dbf856cdb383", "pepper grinder-d84a5b71b1bfe460", "oil-64e976c2baa181ed", "oven mitt-5452742ece417cba", "brush-25c89c842b5f82b3", "sponge-ae72e981a1155186", "bottle of washing up liquid-2bb58495cb26336d", "piece of plastic4-545e03bdf8a4b6c3", "chopping board-7124bc6ce86b9c0e", "tongs-ac68fcd3de9449df", "spoon-3d3eb2c1fbfe560c", "knife2-3a5f16f9f890e73a", "small plate-d76568994bd60227", "peeler-c81dd2a0c2adc4c0", "flask-383c84b3294aa17a", "seasoning-f8f9407553280ef9", "small spoon-1f5f9573a64410f9", "lid of seasoning-59abf75a850cdbbe", "vent-6c5495c3031dd6f6", "tea towel-28427e92b7a003c3", "cutlery tray-72dca017d6c19fd2" ], "reasoning": "1. At 02:12, I observe the user place the baking tray (baking tray-de9529d43407b24d) filled with salmon skewers (skewer1-8d7ad8cf8ef75acd, skewer2-f6e0d2b9e2dddb20, skewer3-269db417e9ddfa2c) into the oven for the first time. \n2. I observe the user place the baking tray (baking tray-de9529d43407b24d) filled with salmon skewers (skewer1-8d7ad8cf8ef75acd, skewer2-f6e0d2b9e2dddb20, skewer3-269db417e9ddfa2c) into the oven for the second time at 09:28 and the final time at 13:29. \n3. At 13:29, I observe that the number on the oven display reads 12:12.\n4. At 00:28, I observe the user place both the first baking tray (baking tray-de9529d43407b24d) and the second baking tray (baking tray2-eaf65a93cb24242c) on the stove .\n5. I rewind from 00:28 to determine the last number on the oven display .\n6. At 00:27, I observe that the number on the oven display is 12:00.\n7. I subtract 1212 from 1200 to get 12.", "referenced_object_masks": [ "skewer1-8d7ad8cf8ef75acd", "skewer2-f6e0d2b9e2dddb20", "skewer3-269db417e9ddfa2c", "baking tray-de9529d43407b24d", "baking tray2-eaf65a93cb24242c" ] }, { "key": "P08-20240614-184945:ad7fb49cc594ba4eb1fd53f603fcb0da5570cc14", "video_id": "P08-20240614-184945", "question": "If I were to multiply the number of magnets above and to the right of my microwave by the number of coffee mugs hanging near my espresso machine and then divide it by the number of skewers I put in the oven, what number would I get?", "answer_choice_0": "12.5", "answer_choice_1": "24", "answer_choice_2": "13.75", "answer_choice_3": "20", "answer_choice_4": "16.5", "answer_id": 4, "answer": "16.5", "question_type": [ "Action Recognition", "Counting", "Numerical Reasoning", "Object Recognition" ], "objects_of_interest": [ "salmon skewer packaging-36dd2f2bb37b566d", "pouch of food-1e0bf176e3e6739a", "baking tray-de9529d43407b24d", "box of kitchen foil-b3321cc93c9e41c6", "baking tray2-eaf65a93cb24242c", "phone-fa69c89bd937f66f", "cardboard sleeve-36b0d8e8ecd28dd7", "piece of foil-a6b3010a8c96fd68", "knife-b820c749f54a6766", "skewer1-8d7ad8cf8ef75acd", "skewer2-f6e0d2b9e2dddb20", "skewer3-269db417e9ddfa2c", "air fryer-fedb08ae74b125c3", "bag of frozen broccoli-e1978bb4ed53e367", "weighing scales-7cda857c0ddeb106", "scissors-0fca98c44bb56811", "broccoli1-698ab4482752b8b1", "broccoli2-76b2eb14b6b98bca", "broccoli3-ec252c7f100e00d6", "broccoli4-859f245ed08e8772", "broccoli5-b4d4bc1c9cc3c303", "air fryer drum-4c0d92179d8e6e77", "piece of plastic-bb26188a50aa9dd0", "paper towel piece-4b035530e7d77e07", "piece of plastic2-c70d83d41e315ce0", "piece of plastic3-65f68ec067da9227", "bowl-0c30320f366aabec", "fork-d538b70af2985b1d", "plate-99745b85dc4011d0", "clip-8142619a137c8692", "salt shaker-20a8dbf856cdb383", "pepper grinder-d84a5b71b1bfe460", "oil-64e976c2baa181ed", "oven mitt-5452742ece417cba", "brush-25c89c842b5f82b3", "sponge-ae72e981a1155186", "bottle of washing up liquid-2bb58495cb26336d", "piece of plastic4-545e03bdf8a4b6c3", "chopping board-7124bc6ce86b9c0e", "tongs-ac68fcd3de9449df", "spoon-3d3eb2c1fbfe560c", "knife2-3a5f16f9f890e73a", "small plate-d76568994bd60227", "peeler-c81dd2a0c2adc4c0", "flask-383c84b3294aa17a", "seasoning-f8f9407553280ef9", "small spoon-1f5f9573a64410f9", "lid of seasoning-59abf75a850cdbbe", "vent-6c5495c3031dd6f6", "tea towel-28427e92b7a003c3", "cutlery tray-72dca017d6c19fd2" ], "reasoning": "1. I began by looking for a clear shot of the magnets on the user's microwave. At 02:11 I saw that there were 11 above and to the right of the microwave.\n2. I then looked for a clear view of the coffee cups by the espresso maker and saw at 01:52 that there were 6.\n3. From here I calculated the product of magnets and coffee cups : 11*6=66.\n4. Finally, I divided this figure by the number of skewers (skewer1-8d7ad8cf8ef75acd, skewer2-f6e0d2b9e2dddb20, skewer3-269db417e9ddfa2c) placed into the oven at 02:12 which was 4: 66/4=16.5.", "referenced_object_masks": [ "skewer1-8d7ad8cf8ef75acd", "skewer2-f6e0d2b9e2dddb20", "skewer3-269db417e9ddfa2c" ] }, { "key": "P08-20240614-184945:f822addb1f2c0704dee80846288a8faa562ea849", "video_id": "P08-20240614-184945", "question": "After I line the baking tray with tin foil, what is the second unique object I touch with my right hand?", "answer_choice_0": "Oven", "answer_choice_1": "Countertop", "answer_choice_2": "Knife", "answer_choice_3": "Salmon skewer", "answer_choice_4": "Phone", "answer_id": 2, "answer": "Knife", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counting" ], "objects_of_interest": [ "salmon skewer packaging-36dd2f2bb37b566d", "pouch of food-1e0bf176e3e6739a", "baking tray-de9529d43407b24d", "box of kitchen foil-b3321cc93c9e41c6", "baking tray2-eaf65a93cb24242c", "phone-fa69c89bd937f66f", "cardboard sleeve-36b0d8e8ecd28dd7", "piece of foil-a6b3010a8c96fd68", "knife-b820c749f54a6766", "skewer1-8d7ad8cf8ef75acd", "skewer2-f6e0d2b9e2dddb20", "skewer3-269db417e9ddfa2c", "air fryer-fedb08ae74b125c3", "bag of frozen broccoli-e1978bb4ed53e367", "weighing scales-7cda857c0ddeb106", "scissors-0fca98c44bb56811", "broccoli1-698ab4482752b8b1", "broccoli2-76b2eb14b6b98bca", "broccoli3-ec252c7f100e00d6", "broccoli4-859f245ed08e8772", "broccoli5-b4d4bc1c9cc3c303", "air fryer drum-4c0d92179d8e6e77", "piece of plastic-bb26188a50aa9dd0", "paper towel piece-4b035530e7d77e07", "piece of plastic2-c70d83d41e315ce0", "piece of plastic3-65f68ec067da9227", "bowl-0c30320f366aabec", "fork-d538b70af2985b1d", "plate-99745b85dc4011d0", "clip-8142619a137c8692", "salt shaker-20a8dbf856cdb383", "pepper grinder-d84a5b71b1bfe460", "oil-64e976c2baa181ed", "oven mitt-5452742ece417cba", "brush-25c89c842b5f82b3", "sponge-ae72e981a1155186", "bottle of washing up liquid-2bb58495cb26336d", "piece of plastic4-545e03bdf8a4b6c3", "chopping board-7124bc6ce86b9c0e", "tongs-ac68fcd3de9449df", "spoon-3d3eb2c1fbfe560c", "knife2-3a5f16f9f890e73a", "small plate-d76568994bd60227", "peeler-c81dd2a0c2adc4c0", "flask-383c84b3294aa17a", "seasoning-f8f9407553280ef9", "small spoon-1f5f9573a64410f9", "lid of seasoning-59abf75a850cdbbe", "vent-6c5495c3031dd6f6", "tea towel-28427e92b7a003c3", "cutlery tray-72dca017d6c19fd2" ], "reasoning": "1. From 01:16 to 01:19, I observe the user pull a piece of tin foil (piece of foil-a6b3010a8c96fd68) from the tin foil box (box of kitchen foil-b3321cc93c9e41c6).\n2. From 01:24 to 01:27, I observe the user line the baking tray (baking tray2-eaf65a93cb24242c) with the tin foil (piece of foil-a6b3010a8c96fd68).\n3. At 01:29, I observe the user touch the package of salmon (salmon skewer packaging-36dd2f2bb37b566d) with their right hand.\n4. At 01:30, I observe the user grab a knife (knife-b820c749f54a6766) from the knife block with their right hand.\n5. I determine that the second unique object the user touches with their right hand is the knife (knife-b820c749f54a6766).", "referenced_object_masks": [ "box of kitchen foil-b3321cc93c9e41c6", "knife-b820c749f54a6766", "baking tray2-eaf65a93cb24242c", "salmon skewer packaging-36dd2f2bb37b566d", "piece of foil-a6b3010a8c96fd68" ] }, { "key": "P08-20240617-130401:1b25f6dcef5ab03916fa88bb004bebd0786b4ff2", "video_id": "P08-20240617-130401", "question": "How many times do I tap on the phone screen after weighing the pasta?", "answer_choice_0": "4 times", "answer_choice_1": "5 times", "answer_choice_2": "3 times", "answer_choice_3": "Once", "answer_choice_4": "Twice", "answer_id": 1, "answer": "5 times", "question_type": [ "Object Recognition", "Action Recognition", "Counting", "Event Occurence" ], "objects_of_interest": [ "phone-85724d9e5d94148c", "tea towel-427614b9e081b7aa", "piece of paper-d613d383a5f78cb8", "coffee grinder-45637c4bff924e2e", "pot lid-258d022cd42bb7d6", "chopping board-f5b35f085b577243", "cucumber-a3816067e6a02c7e", "bell peper-5a94bfdd100b14d2", "small tupperware-dafe051e20c99254", "tomato container-b3686c57f082a560", "bag of spinach-94fa6852d43ba7d3", "bag of coriander-9acfec1e3b217815", "vegetables3-84e9781ba0e9feeb", "refrigerator drawer-7b9ec08f24848bbd", "vegetables5-eb27e0c1ab0e9b05", "food packet-43914a347655d5fb", "pot-cfda9ec34cb81ff2", "top of pot-16072779d839ebbb", "plastic container-d31edf0e2295e132", "beer can-807a1ee36fe8a920", "kitchen scale-7e60a8366c417004", "knife holder-4b9f706781e22ed1", "onion-14d2cff3d51668eb", "vegetable basket-c133eead82decd11", "measuring cups-4822959e8b96b839", "salad bowl-a6730b4ebe7722b3", "plastic film-f42ed0a200f039fd", "knife-7c00b19f7197bc8a", "smaller knife-d5aeabbc2652fa5c", "cherry tomatoes-6a474b3d9ba6efe7", "new tomato1-200289016707d2bf", "knife2-cf2ad08a4e381d0e", "knife3-5f4ebcb05f43e15c", "bag of celery-cb8bd824383db776", "celery-10ffde68e50c1964", "vegetables4-ec244ab253d03441", "paper towel-c0ae417842762647", "can of beans-fabced34945a4ecc", "pasta sauce-21f067e8fb677428", "some can-e36ab72107ebfa4a", "can of tuna-30e76b3f7006aeee", "peeler-0ac0790a4a392de6", "spoon-0a26c5d695214bab", "large spoon-fdd604e711dcc288", "carrots-c0ec32ee12321e86", "can opener-c4587da967c63301", "bowl-6420618f1eb48874", "spoon2-6b1f15c0179017d7", "butter-14ab53559b0eaf05", "sauce bottle-e3131925669430af", "single pasta-bc3ade7b93c01e88", "small plate-a38e1b44926880a3", "fork-fa19eb1c0295a587", "aluminium foil-a785ab50dfbc3d25", "pack of aluminum foil-45d0c1ffaf749c02", "pack of plastic sheets-879c88e871c58955", "cleaning cloth-c5ea3975c7fb613c", "cleaning sponge-b460d8fda33273cd", "bottle of dishwashing liquid-b3a1618fe39a6003", "bowl2-0e7adbb5d4d969cc", "aluminum tin-0e724679ec386af9", "pan-38b2fd4336ec4d62", "bottle-196950f1857db3cb", "tin lid-d09e123d69254881", "bottle oil-2a696f22a7d81899", "box-1770a3d4b1a741ac", "jar-5b4d2f8825b08700", "glass container-9db809eac769b24c", "lid-93bb46c08d0089fa", "spoon3-a3ac5412d2e808ce", "bottle2-40a912b668615655", "glass-6ded6ed7a2319bdc", "paper roll-8dd136af95c41246" ], "reasoning": "1. I located the moment where the user weights the dry pasta , which happens at 02:55. \n2. At 03:01, the user grabs the cell phone (phone-85724d9e5d94148c) and taps on the screen once at 03:02.\n3. At 03:03, the user taps the phone (phone-85724d9e5d94148c) screen twice.\n4. At 03:04, the user taps the phone (phone-85724d9e5d94148c) screen once.\n5. The user taps the phone (phone-85724d9e5d94148c) screen once more at 03:07. \n6. The user puts the phone (phone-85724d9e5d94148c) down at 03:10. Therefore, the user taps the phone (phone-85724d9e5d94148c) screen 5 times after weighing the dry pasta .", "referenced_object_masks": [ "phone-85724d9e5d94148c" ] }, { "key": "P08-20240617-130401:321f4551537b28b67cc6ce8d35d75c701d778f2e", "video_id": "P08-20240617-130401", "question": "Same question can be used", "answer_choice_0": "Onion", "answer_choice_1": "Tomato", "answer_choice_2": "Red bell pepper", "answer_choice_3": "Cucumber", "answer_choice_4": "Yellow bell pepper", "answer_id": 2, "answer": "Red bell pepper", "question_type": [ "Object Recognition", "Action Recognition", "Counting", "Event Occurence" ], "objects_of_interest": [ "phone-85724d9e5d94148c", "tea towel-427614b9e081b7aa", "piece of paper-d613d383a5f78cb8", "coffee grinder-45637c4bff924e2e", "pot lid-258d022cd42bb7d6", "chopping board-f5b35f085b577243", "cucumber-a3816067e6a02c7e", "bell peper-5a94bfdd100b14d2", "small tupperware-dafe051e20c99254", "tomato container-b3686c57f082a560", "bag of spinach-94fa6852d43ba7d3", "bag of coriander-9acfec1e3b217815", "vegetables3-84e9781ba0e9feeb", "refrigerator drawer-7b9ec08f24848bbd", "vegetables5-eb27e0c1ab0e9b05", "food packet-43914a347655d5fb", "pot-cfda9ec34cb81ff2", "top of pot-16072779d839ebbb", "plastic container-d31edf0e2295e132", "beer can-807a1ee36fe8a920", "kitchen scale-7e60a8366c417004", "knife holder-4b9f706781e22ed1", "onion-14d2cff3d51668eb", "vegetable basket-c133eead82decd11", "measuring cups-4822959e8b96b839", "salad bowl-a6730b4ebe7722b3", "plastic film-f42ed0a200f039fd", "knife-7c00b19f7197bc8a", "smaller knife-d5aeabbc2652fa5c", "cherry tomatoes-6a474b3d9ba6efe7", "new tomato1-200289016707d2bf", "knife2-cf2ad08a4e381d0e", "knife3-5f4ebcb05f43e15c", "bag of celery-cb8bd824383db776", "celery-10ffde68e50c1964", "vegetables4-ec244ab253d03441", "paper towel-c0ae417842762647", "can of beans-fabced34945a4ecc", "pasta sauce-21f067e8fb677428", "some can-e36ab72107ebfa4a", "can of tuna-30e76b3f7006aeee", "peeler-0ac0790a4a392de6", "spoon-0a26c5d695214bab", "large spoon-fdd604e711dcc288", "carrots-c0ec32ee12321e86", "can opener-c4587da967c63301", "bowl-6420618f1eb48874", "spoon2-6b1f15c0179017d7", "butter-14ab53559b0eaf05", "sauce bottle-e3131925669430af", "single pasta-bc3ade7b93c01e88", "small plate-a38e1b44926880a3", "fork-fa19eb1c0295a587", "aluminium foil-a785ab50dfbc3d25", "pack of aluminum foil-45d0c1ffaf749c02", "pack of plastic sheets-879c88e871c58955", "cleaning cloth-c5ea3975c7fb613c", "cleaning sponge-b460d8fda33273cd", "bottle of dishwashing liquid-b3a1618fe39a6003", "bowl2-0e7adbb5d4d969cc", "aluminum tin-0e724679ec386af9", "pan-38b2fd4336ec4d62", "bottle-196950f1857db3cb", "tin lid-d09e123d69254881", "bottle oil-2a696f22a7d81899", "box-1770a3d4b1a741ac", "jar-5b4d2f8825b08700", "glass container-9db809eac769b24c", "lid-93bb46c08d0089fa", "spoon3-a3ac5412d2e808ce", "bottle2-40a912b668615655", "glass-6ded6ed7a2319bdc", "paper roll-8dd136af95c41246" ], "reasoning": "1. I looked for the moment when the user opened the fridge for the first time, which happens at 00:38.\n2. The first item the user touches with the right hand after opening the fridge is a cucumber (cucumber-a3816067e6a02c7e) at 00:39. \n3. The second item the user touches with the right hand after the cucumber (cucumber-a3816067e6a02c7e) is the plastic container that is holding the peppers at 0:40. \n4. The third item the user touches after opening the fridge is the red bell pepper at 00:42.", "referenced_object_masks": [ "cucumber-a3816067e6a02c7e" ] }, { "key": "P08-20240617-130401:521c83e72d8249aae4a02bcc66679bd07944f375", "video_id": "P08-20240617-130401", "question": "What are the first and second ingredients I weigh on the digital scale?", "answer_choice_0": "Tomatoes and cucumbers", "answer_choice_1": "Onions and pasta", "answer_choice_2": "Cucumber and carrots", "answer_choice_3": "Pasta and cucumber", "answer_choice_4": "Tomatoes and pasta", "answer_id": 3, "answer": "Pasta and cucumber", "question_type": [ "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counting" ], "objects_of_interest": [ "phone-85724d9e5d94148c", "tea towel-427614b9e081b7aa", "piece of paper-d613d383a5f78cb8", "coffee grinder-45637c4bff924e2e", "pot lid-258d022cd42bb7d6", "chopping board-f5b35f085b577243", "cucumber-a3816067e6a02c7e", "bell peper-5a94bfdd100b14d2", "small tupperware-dafe051e20c99254", "tomato container-b3686c57f082a560", "bag of spinach-94fa6852d43ba7d3", "bag of coriander-9acfec1e3b217815", "vegetables3-84e9781ba0e9feeb", "refrigerator drawer-7b9ec08f24848bbd", "vegetables5-eb27e0c1ab0e9b05", "food packet-43914a347655d5fb", "pot-cfda9ec34cb81ff2", "top of pot-16072779d839ebbb", "plastic container-d31edf0e2295e132", "beer can-807a1ee36fe8a920", "kitchen scale-7e60a8366c417004", "knife holder-4b9f706781e22ed1", "onion-14d2cff3d51668eb", "vegetable basket-c133eead82decd11", "measuring cups-4822959e8b96b839", "salad bowl-a6730b4ebe7722b3", "plastic film-f42ed0a200f039fd", "knife-7c00b19f7197bc8a", "smaller knife-d5aeabbc2652fa5c", "cherry tomatoes-6a474b3d9ba6efe7", "new tomato1-200289016707d2bf", "knife2-cf2ad08a4e381d0e", "knife3-5f4ebcb05f43e15c", "bag of celery-cb8bd824383db776", "celery-10ffde68e50c1964", "vegetables4-ec244ab253d03441", "paper towel-c0ae417842762647", "can of beans-fabced34945a4ecc", "pasta sauce-21f067e8fb677428", "some can-e36ab72107ebfa4a", "can of tuna-30e76b3f7006aeee", "peeler-0ac0790a4a392de6", "spoon-0a26c5d695214bab", "large spoon-fdd604e711dcc288", "carrots-c0ec32ee12321e86", "can opener-c4587da967c63301", "bowl-6420618f1eb48874", "spoon2-6b1f15c0179017d7", "butter-14ab53559b0eaf05", "sauce bottle-e3131925669430af", "single pasta-bc3ade7b93c01e88", "small plate-a38e1b44926880a3", "fork-fa19eb1c0295a587", "aluminium foil-a785ab50dfbc3d25", "pack of aluminum foil-45d0c1ffaf749c02", "pack of plastic sheets-879c88e871c58955", "cleaning cloth-c5ea3975c7fb613c", "cleaning sponge-b460d8fda33273cd", "bottle of dishwashing liquid-b3a1618fe39a6003", "bowl2-0e7adbb5d4d969cc", "aluminum tin-0e724679ec386af9", "pan-38b2fd4336ec4d62", "bottle-196950f1857db3cb", "tin lid-d09e123d69254881", "bottle oil-2a696f22a7d81899", "box-1770a3d4b1a741ac", "jar-5b4d2f8825b08700", "glass container-9db809eac769b24c", "lid-93bb46c08d0089fa", "spoon3-a3ac5412d2e808ce", "bottle2-40a912b668615655", "glass-6ded6ed7a2319bdc", "paper roll-8dd136af95c41246" ], "reasoning": "1. At 01:58, I observe the user pick up the digital scale (kitchen scale-7e60a8366c417004) from behind the knife hold (knife holder-4b9f706781e22ed1).\n2. At 03:00, I observe the user find a measurement for the pasta .\n3. At 04:13, I observe the user find a measurement for the cucumber (cucumber-a3816067e6a02c7e).\n4. I determine that the first and second ingredients weighed are pasta and cucumber (cucumber-a3816067e6a02c7e).", "referenced_object_masks": [ "cucumber-a3816067e6a02c7e", "knife holder-4b9f706781e22ed1", "kitchen scale-7e60a8366c417004" ] }, { "key": "P08-20240617-130401:6dabd332a36d8fe6ecd0d7fb14c993262e9c1d5c", "video_id": "P08-20240617-130401", "question": "Which of the first 3 items I remove from the fridge do I place furthest to my right on the countertop?", "answer_choice_0": "Tomatoes", "answer_choice_1": "Carrots", "answer_choice_2": "Bell pepper", "answer_choice_3": "Cucumber", "answer_choice_4": "Celery", "answer_id": 0, "answer": "Tomatoes", "question_type": [ "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counting", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "phone-85724d9e5d94148c", "tea towel-427614b9e081b7aa", "piece of paper-d613d383a5f78cb8", "coffee grinder-45637c4bff924e2e", "pot lid-258d022cd42bb7d6", "chopping board-f5b35f085b577243", "cucumber-a3816067e6a02c7e", "bell peper-5a94bfdd100b14d2", "small tupperware-dafe051e20c99254", "tomato container-b3686c57f082a560", "bag of spinach-94fa6852d43ba7d3", "bag of coriander-9acfec1e3b217815", "vegetables3-84e9781ba0e9feeb", "refrigerator drawer-7b9ec08f24848bbd", "vegetables5-eb27e0c1ab0e9b05", "food packet-43914a347655d5fb", "pot-cfda9ec34cb81ff2", "top of pot-16072779d839ebbb", "plastic container-d31edf0e2295e132", "beer can-807a1ee36fe8a920", "kitchen scale-7e60a8366c417004", "knife holder-4b9f706781e22ed1", "onion-14d2cff3d51668eb", "vegetable basket-c133eead82decd11", "measuring cups-4822959e8b96b839", "salad bowl-a6730b4ebe7722b3", "plastic film-f42ed0a200f039fd", "knife-7c00b19f7197bc8a", "smaller knife-d5aeabbc2652fa5c", "cherry tomatoes-6a474b3d9ba6efe7", "new tomato1-200289016707d2bf", "knife2-cf2ad08a4e381d0e", "knife3-5f4ebcb05f43e15c", "bag of celery-cb8bd824383db776", "celery-10ffde68e50c1964", "vegetables4-ec244ab253d03441", "paper towel-c0ae417842762647", "can of beans-fabced34945a4ecc", "pasta sauce-21f067e8fb677428", "some can-e36ab72107ebfa4a", "can of tuna-30e76b3f7006aeee", "peeler-0ac0790a4a392de6", "spoon-0a26c5d695214bab", "large spoon-fdd604e711dcc288", "carrots-c0ec32ee12321e86", "can opener-c4587da967c63301", "bowl-6420618f1eb48874", "spoon2-6b1f15c0179017d7", "butter-14ab53559b0eaf05", "sauce bottle-e3131925669430af", "single pasta-bc3ade7b93c01e88", "small plate-a38e1b44926880a3", "fork-fa19eb1c0295a587", "aluminium foil-a785ab50dfbc3d25", "pack of aluminum foil-45d0c1ffaf749c02", "pack of plastic sheets-879c88e871c58955", "cleaning cloth-c5ea3975c7fb613c", "cleaning sponge-b460d8fda33273cd", "bottle of dishwashing liquid-b3a1618fe39a6003", "bowl2-0e7adbb5d4d969cc", "aluminum tin-0e724679ec386af9", "pan-38b2fd4336ec4d62", "bottle-196950f1857db3cb", "tin lid-d09e123d69254881", "bottle oil-2a696f22a7d81899", "box-1770a3d4b1a741ac", "jar-5b4d2f8825b08700", "glass container-9db809eac769b24c", "lid-93bb46c08d0089fa", "spoon3-a3ac5412d2e808ce", "bottle2-40a912b668615655", "glass-6ded6ed7a2319bdc", "paper roll-8dd136af95c41246" ], "reasoning": "1. I observe the user place the cutting board (chopping board-f5b35f085b577243) onto the counter at 00:36.\n2. I observe the user open the fridge at 00:37 and close the fridge at 01:11.\n3. Between 00:37 and 0:47, I observe the user remove the cucumber (cucumber-a3816067e6a02c7e), the bell pepper (bell peper-5a94bfdd100b14d2), and the tomatoes (tomato container-b3686c57f082a560).\n4. I observe the user place the cucumber (cucumber-a3816067e6a02c7e), the bell pepper (bell peper-5a94bfdd100b14d2), and the tomatoes (tomato container-b3686c57f082a560) onto the counter at 00:50. \n5. I determine that the tomatoes (tomato container-b3686c57f082a560) are furthest to the user's right.", "referenced_object_masks": [ "cucumber-a3816067e6a02c7e", "chopping board-f5b35f085b577243", "bell peper-5a94bfdd100b14d2", "tomato container-b3686c57f082a560" ] }, { "key": "P08-20240617-130401:7142f0a1726bff37f45a161fe639c6e59765dd42", "video_id": "P08-20240617-130401", "question": "Where is my veggie peeler in relation to me during the separate instances when I'm rinsing the thermos, dicing the cucumbers, and tasting the pasta?", "answer_choice_0": "On the counter to my right; in the drawer to my right; on the counter to my left", "answer_choice_1": "In the sink to my left; in the dishwasher to my left; on the counter to my left", "answer_choice_2": "On the counter to my left; in the sink to my left; on the counter to my left", "answer_choice_3": "In the sink to my right; in the dishwasher to my right; in the drawer to my left", "answer_choice_4": "In the drawer to my left; in the dishwasher to my right; on the counter to my right", "answer_id": 0, "answer": "On the counter to my right; in the drawer to my right; on the counter to my left", "question_type": [ "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counting", "Numerical Reasoning" ], "objects_of_interest": [ "phone-85724d9e5d94148c", "tea towel-427614b9e081b7aa", "piece of paper-d613d383a5f78cb8", "coffee grinder-45637c4bff924e2e", "pot lid-258d022cd42bb7d6", "chopping board-f5b35f085b577243", "cucumber-a3816067e6a02c7e", "bell peper-5a94bfdd100b14d2", "small tupperware-dafe051e20c99254", "tomato container-b3686c57f082a560", "bag of spinach-94fa6852d43ba7d3", "bag of coriander-9acfec1e3b217815", "vegetables3-84e9781ba0e9feeb", "refrigerator drawer-7b9ec08f24848bbd", "vegetables5-eb27e0c1ab0e9b05", "food packet-43914a347655d5fb", "pot-cfda9ec34cb81ff2", "top of pot-16072779d839ebbb", "plastic container-d31edf0e2295e132", "beer can-807a1ee36fe8a920", "kitchen scale-7e60a8366c417004", "knife holder-4b9f706781e22ed1", "onion-14d2cff3d51668eb", "vegetable basket-c133eead82decd11", "measuring cups-4822959e8b96b839", "salad bowl-a6730b4ebe7722b3", "plastic film-f42ed0a200f039fd", "knife-7c00b19f7197bc8a", "smaller knife-d5aeabbc2652fa5c", "cherry tomatoes-6a474b3d9ba6efe7", "new tomato1-200289016707d2bf", "knife2-cf2ad08a4e381d0e", "knife3-5f4ebcb05f43e15c", "bag of celery-cb8bd824383db776", "celery-10ffde68e50c1964", "vegetables4-ec244ab253d03441", "paper towel-c0ae417842762647", "can of beans-fabced34945a4ecc", "pasta sauce-21f067e8fb677428", "some can-e36ab72107ebfa4a", "can of tuna-30e76b3f7006aeee", "peeler-0ac0790a4a392de6", "spoon-0a26c5d695214bab", "large spoon-fdd604e711dcc288", "carrots-c0ec32ee12321e86", "can opener-c4587da967c63301", "bowl-6420618f1eb48874", "spoon2-6b1f15c0179017d7", "butter-14ab53559b0eaf05", "sauce bottle-e3131925669430af", "single pasta-bc3ade7b93c01e88", "small plate-a38e1b44926880a3", "fork-fa19eb1c0295a587", "aluminium foil-a785ab50dfbc3d25", "pack of aluminum foil-45d0c1ffaf749c02", "pack of plastic sheets-879c88e871c58955", "cleaning cloth-c5ea3975c7fb613c", "cleaning sponge-b460d8fda33273cd", "bottle of dishwashing liquid-b3a1618fe39a6003", "bowl2-0e7adbb5d4d969cc", "aluminum tin-0e724679ec386af9", "pan-38b2fd4336ec4d62", "bottle-196950f1857db3cb", "tin lid-d09e123d69254881", "bottle oil-2a696f22a7d81899", "box-1770a3d4b1a741ac", "jar-5b4d2f8825b08700", "glass container-9db809eac769b24c", "lid-93bb46c08d0089fa", "spoon3-a3ac5412d2e808ce", "bottle2-40a912b668615655", "glass-6ded6ed7a2319bdc", "paper roll-8dd136af95c41246" ], "reasoning": "1. First, I locate the timestamp where the user rinses the thermos (bottle-196950f1857db3cb). This occurs at 28:20.\n2. I keep watching to determine the user's next steps and as they move about, I notice the vegetable peeler (peeler-0ac0790a4a392de6) on the counter, to the user's right, positioned in front of the bowl (salad bowl-a6730b4ebe7722b3) at 28:55 until the user picks up the peeler (peeler-0ac0790a4a392de6) at 28:56. \n3. Next, I locate the timestamp where the user dices the cucumbers (cucumber-a3816067e6a02c7e). This occurs between 04:34-05:07. \n4. I keep watching and determine the peeler (peeler-0ac0790a4a392de6) is not retrieved from its storage place until 10:57. The user then retrieves the peeler (peeler-0ac0790a4a392de6) from a drawer on the user's right. \n5. Finally, I locate the timestamp where the user tastes the pasta (single pasta-bc3ade7b93c01e88). This occurs at 22:25.\n6. While the user stands at the stove to taste the boiling pasta (single pasta-bc3ade7b93c01e88), the peeler (peeler-0ac0790a4a392de6) is visible on the counter, to the user's left, in front of the bowl (salad bowl-a6730b4ebe7722b3) and next to the user's cellphone (phone-85724d9e5d94148c).\n7. Therefore, in each of the 3 instances, the vegetable peeler (peeler-0ac0790a4a392de6) is positioned on the counter to the user's right, in the drawer to the user's right, and on the counter to the user's left.", "referenced_object_masks": [ "salad bowl-a6730b4ebe7722b3", "phone-85724d9e5d94148c", "cucumber-a3816067e6a02c7e", "peeler-0ac0790a4a392de6", "single pasta-bc3ade7b93c01e88", "bottle-196950f1857db3cb" ] }, { "key": "P08-20240617-130401:723fcda1d3b68f91143893b9f67dbb71947f890a", "video_id": "P08-20240617-130401", "question": "What is the fourth item I store in the refrigerator after the celery?", "answer_choice_0": "Onion", "answer_choice_1": "Bag of carrots", "answer_choice_2": "Pepper", "answer_choice_3": "Butter", "answer_choice_4": "Bowl", "answer_id": 1, "answer": "Bag of carrots", "question_type": [ "Object Recognition", "Action Recognition", "Counting", "Event Occurence" ], "objects_of_interest": [ "phone-85724d9e5d94148c", "tea towel-427614b9e081b7aa", "piece of paper-d613d383a5f78cb8", "coffee grinder-45637c4bff924e2e", "pot lid-258d022cd42bb7d6", "chopping board-f5b35f085b577243", "cucumber-a3816067e6a02c7e", "bell peper-5a94bfdd100b14d2", "small tupperware-dafe051e20c99254", "tomato container-b3686c57f082a560", "bag of spinach-94fa6852d43ba7d3", "bag of coriander-9acfec1e3b217815", "vegetables3-84e9781ba0e9feeb", "refrigerator drawer-7b9ec08f24848bbd", "vegetables5-eb27e0c1ab0e9b05", "food packet-43914a347655d5fb", "pot-cfda9ec34cb81ff2", "top of pot-16072779d839ebbb", "plastic container-d31edf0e2295e132", "beer can-807a1ee36fe8a920", "kitchen scale-7e60a8366c417004", "knife holder-4b9f706781e22ed1", "onion-14d2cff3d51668eb", "vegetable basket-c133eead82decd11", "measuring cups-4822959e8b96b839", "salad bowl-a6730b4ebe7722b3", "plastic film-f42ed0a200f039fd", "knife-7c00b19f7197bc8a", "smaller knife-d5aeabbc2652fa5c", "cherry tomatoes-6a474b3d9ba6efe7", "new tomato1-200289016707d2bf", "knife2-cf2ad08a4e381d0e", "knife3-5f4ebcb05f43e15c", "bag of celery-cb8bd824383db776", "celery-10ffde68e50c1964", "vegetables4-ec244ab253d03441", "paper towel-c0ae417842762647", "can of beans-fabced34945a4ecc", "pasta sauce-21f067e8fb677428", "some can-e36ab72107ebfa4a", "can of tuna-30e76b3f7006aeee", "peeler-0ac0790a4a392de6", "spoon-0a26c5d695214bab", "large spoon-fdd604e711dcc288", "carrots-c0ec32ee12321e86", "can opener-c4587da967c63301", "bowl-6420618f1eb48874", "spoon2-6b1f15c0179017d7", "butter-14ab53559b0eaf05", "sauce bottle-e3131925669430af", "single pasta-bc3ade7b93c01e88", "small plate-a38e1b44926880a3", "fork-fa19eb1c0295a587", "aluminium foil-a785ab50dfbc3d25", "pack of aluminum foil-45d0c1ffaf749c02", "pack of plastic sheets-879c88e871c58955", "cleaning cloth-c5ea3975c7fb613c", "cleaning sponge-b460d8fda33273cd", "bottle of dishwashing liquid-b3a1618fe39a6003", "bowl2-0e7adbb5d4d969cc", "aluminum tin-0e724679ec386af9", "pan-38b2fd4336ec4d62", "bottle-196950f1857db3cb", "tin lid-d09e123d69254881", "bottle oil-2a696f22a7d81899", "box-1770a3d4b1a741ac", "jar-5b4d2f8825b08700", "glass container-9db809eac769b24c", "lid-93bb46c08d0089fa", "spoon3-a3ac5412d2e808ce", "bottle2-40a912b668615655", "glass-6ded6ed7a2319bdc", "paper roll-8dd136af95c41246" ], "reasoning": "1. I located the moment the user stores the celery (bag of celery-cb8bd824383db776) in the refrigerator at 20:11. \n2. The first item the user stores in the refrigerator after the celery (bag of celery-cb8bd824383db776) is a white bowl (bowl-6420618f1eb48874) at 20:16.\n3. The second item the user stores in the refrigerator after the celery (bag of celery-cb8bd824383db776) is a piece of red bell pepper (bell peper-5a94bfdd100b14d2) at 20:19. \n4. The third item the user stores in the refrigerator after the celery (bag of celery-cb8bd824383db776) is a piece of purple onion (onion-14d2cff3d51668eb) at 20:23.\n5. Finally, the fourth item the user stores in the refrigerator after the celery (bag of celery-cb8bd824383db776) is a bag of carrots (carrots-c0ec32ee12321e86) at 20:26. Therefore, the correct answer is a bag of carrots (carrots-c0ec32ee12321e86).", "referenced_object_masks": [ "carrots-c0ec32ee12321e86", "bag of celery-cb8bd824383db776", "bowl-6420618f1eb48874", "bell peper-5a94bfdd100b14d2", "onion-14d2cff3d51668eb" ] }, { "key": "P08-20240617-130401:740087cb67b26bfe3a6ce2cec1692dd2c6a3cbdd", "video_id": "P08-20240617-130401", "question": "How many cut vegetables did I add to a bowl in between the 1st and 3rd times that I handled the can of beans?", "answer_choice_0": "3", "answer_choice_1": "2", "answer_choice_2": "5", "answer_choice_3": "1", "answer_choice_4": "4", "answer_id": 0, "answer": "3", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Event Occurence" ], "objects_of_interest": [ "phone-85724d9e5d94148c", "tea towel-427614b9e081b7aa", "piece of paper-d613d383a5f78cb8", "coffee grinder-45637c4bff924e2e", "pot lid-258d022cd42bb7d6", "chopping board-f5b35f085b577243", "cucumber-a3816067e6a02c7e", "bell peper-5a94bfdd100b14d2", "small tupperware-dafe051e20c99254", "tomato container-b3686c57f082a560", "bag of spinach-94fa6852d43ba7d3", "bag of coriander-9acfec1e3b217815", "vegetables3-84e9781ba0e9feeb", "refrigerator drawer-7b9ec08f24848bbd", "vegetables5-eb27e0c1ab0e9b05", "food packet-43914a347655d5fb", "pot-cfda9ec34cb81ff2", "top of pot-16072779d839ebbb", "plastic container-d31edf0e2295e132", "beer can-807a1ee36fe8a920", "kitchen scale-7e60a8366c417004", "knife holder-4b9f706781e22ed1", "onion-14d2cff3d51668eb", "vegetable basket-c133eead82decd11", "measuring cups-4822959e8b96b839", "salad bowl-a6730b4ebe7722b3", "plastic film-f42ed0a200f039fd", "knife-7c00b19f7197bc8a", "smaller knife-d5aeabbc2652fa5c", "cherry tomatoes-6a474b3d9ba6efe7", "new tomato1-200289016707d2bf", "knife2-cf2ad08a4e381d0e", "knife3-5f4ebcb05f43e15c", "bag of celery-cb8bd824383db776", "celery-10ffde68e50c1964", "vegetables4-ec244ab253d03441", "paper towel-c0ae417842762647", "can of beans-fabced34945a4ecc", "pasta sauce-21f067e8fb677428", "some can-e36ab72107ebfa4a", "can of tuna-30e76b3f7006aeee", "peeler-0ac0790a4a392de6", "spoon-0a26c5d695214bab", "large spoon-fdd604e711dcc288", "carrots-c0ec32ee12321e86", "can opener-c4587da967c63301", "bowl-6420618f1eb48874", "spoon2-6b1f15c0179017d7", "butter-14ab53559b0eaf05", "sauce bottle-e3131925669430af", "single pasta-bc3ade7b93c01e88", "small plate-a38e1b44926880a3", "fork-fa19eb1c0295a587", "aluminium foil-a785ab50dfbc3d25", "pack of aluminum foil-45d0c1ffaf749c02", "pack of plastic sheets-879c88e871c58955", "cleaning cloth-c5ea3975c7fb613c", "cleaning sponge-b460d8fda33273cd", "bottle of dishwashing liquid-b3a1618fe39a6003", "bowl2-0e7adbb5d4d969cc", "aluminum tin-0e724679ec386af9", "pan-38b2fd4336ec4d62", "bottle-196950f1857db3cb", "tin lid-d09e123d69254881", "bottle oil-2a696f22a7d81899", "box-1770a3d4b1a741ac", "jar-5b4d2f8825b08700", "glass container-9db809eac769b24c", "lid-93bb46c08d0089fa", "spoon3-a3ac5412d2e808ce", "bottle2-40a912b668615655", "glass-6ded6ed7a2319bdc", "paper roll-8dd136af95c41246" ], "reasoning": "1. I watched the video and looked for the times when the user handled a can of beans (can of beans-fabced34945a4ecc). I found the first occurrence at 10:15, when the user pulled it out of a cabinet.\n2. From 10:15, I watched to see how many cut vegetables are added to a bowl noting that I would stop counting when the user handled the can of beans (can of beans-fabced34945a4ecc) a 2nd time.\n3. From 13:08-13:12, the user added carrots (carrots-c0ec32ee12321e86) to a bowl (salad bowl-a6730b4ebe7722b3). I counted this as the 1st cut vegetable.\n4. From 13:25-13:26, I noticed the user briefly handled the can of beans (can of beans-fabced34945a4ecc). I counted this as the 2nd occurrence of handling the can of beans (can of beans-fabced34945a4ecc).\n5. From 16:06-16:08, the user added a cut red bell pepper (bell peper-5a94bfdd100b14d2) to the same bowl (salad bowl-a6730b4ebe7722b3). I counted this as the 2nd cut vegetable.\n6. From 17:51-17:58, the user added spinach from a bag of spinach (bag of spinach-94fa6852d43ba7d3) to the same bowl (salad bowl-a6730b4ebe7722b3). I counted this as the 3rd cut vegetable.\n7. At 18:07, I noticed the user picked up the can of beans (can of beans-fabced34945a4ecc). I counted this as the 3rd occurrence of handling the can of beans (can of beans-fabced34945a4ecc).\n8. Since this was the 3rd occurrence, I found a final answer of 3 cut vegetables added to a bowl between the 1st and 3rd occurrence of handling the can of beans (can of beans-fabced34945a4ecc).", "referenced_object_masks": [ "salad bowl-a6730b4ebe7722b3", "can of beans-fabced34945a4ecc", "carrots-c0ec32ee12321e86", "bag of spinach-94fa6852d43ba7d3", "bell peper-5a94bfdd100b14d2" ] }, { "key": "P08-20240617-130401:bdfab31a54f027cd6292ed9f00973f906f55ce31", "video_id": "P08-20240617-130401", "question": "After placing the cutting board on the countertop, how many items above the bottom drawer in the fridge do I touch, but not remove?", "answer_choice_0": "2", "answer_choice_1": "1", "answer_choice_2": "5", "answer_choice_3": "3", "answer_choice_4": "4", "answer_id": 0, "answer": "2", "question_type": [ "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counting" ], "objects_of_interest": [ "phone-85724d9e5d94148c", "tea towel-427614b9e081b7aa", "piece of paper-d613d383a5f78cb8", "coffee grinder-45637c4bff924e2e", "pot lid-258d022cd42bb7d6", "chopping board-f5b35f085b577243", "cucumber-a3816067e6a02c7e", "bell peper-5a94bfdd100b14d2", "small tupperware-dafe051e20c99254", "tomato container-b3686c57f082a560", "bag of spinach-94fa6852d43ba7d3", "bag of coriander-9acfec1e3b217815", "vegetables3-84e9781ba0e9feeb", "refrigerator drawer-7b9ec08f24848bbd", "vegetables5-eb27e0c1ab0e9b05", "food packet-43914a347655d5fb", "pot-cfda9ec34cb81ff2", "top of pot-16072779d839ebbb", "plastic container-d31edf0e2295e132", "beer can-807a1ee36fe8a920", "kitchen scale-7e60a8366c417004", "knife holder-4b9f706781e22ed1", "onion-14d2cff3d51668eb", "vegetable basket-c133eead82decd11", "measuring cups-4822959e8b96b839", "salad bowl-a6730b4ebe7722b3", "plastic film-f42ed0a200f039fd", "knife-7c00b19f7197bc8a", "smaller knife-d5aeabbc2652fa5c", "cherry tomatoes-6a474b3d9ba6efe7", "new tomato1-200289016707d2bf", "knife2-cf2ad08a4e381d0e", "knife3-5f4ebcb05f43e15c", "bag of celery-cb8bd824383db776", "celery-10ffde68e50c1964", "vegetables4-ec244ab253d03441", "paper towel-c0ae417842762647", "can of beans-fabced34945a4ecc", "pasta sauce-21f067e8fb677428", "some can-e36ab72107ebfa4a", "can of tuna-30e76b3f7006aeee", "peeler-0ac0790a4a392de6", "spoon-0a26c5d695214bab", "large spoon-fdd604e711dcc288", "carrots-c0ec32ee12321e86", "can opener-c4587da967c63301", "bowl-6420618f1eb48874", "spoon2-6b1f15c0179017d7", "butter-14ab53559b0eaf05", "sauce bottle-e3131925669430af", "single pasta-bc3ade7b93c01e88", "small plate-a38e1b44926880a3", "fork-fa19eb1c0295a587", "aluminium foil-a785ab50dfbc3d25", "pack of aluminum foil-45d0c1ffaf749c02", "pack of plastic sheets-879c88e871c58955", "cleaning cloth-c5ea3975c7fb613c", "cleaning sponge-b460d8fda33273cd", "bottle of dishwashing liquid-b3a1618fe39a6003", "bowl2-0e7adbb5d4d969cc", "aluminum tin-0e724679ec386af9", "pan-38b2fd4336ec4d62", "bottle-196950f1857db3cb", "tin lid-d09e123d69254881", "bottle oil-2a696f22a7d81899", "box-1770a3d4b1a741ac", "jar-5b4d2f8825b08700", "glass container-9db809eac769b24c", "lid-93bb46c08d0089fa", "spoon3-a3ac5412d2e808ce", "bottle2-40a912b668615655", "glass-6ded6ed7a2319bdc", "paper roll-8dd136af95c41246" ], "reasoning": "1. I observe the user place the cutting board (chopping board-f5b35f085b577243) onto the counter at 00:36.\n2. I observe the user open the fridge at 00:37 and close the fridge at 01:11.\n3. Between 00:37 and 01:11, I observe the user touch the small tupperware (small tupperware-dafe051e20c99254) and the pepper container .\n4. I determine that the user touched, but did not remove, 2 items from the fridge .", "referenced_object_masks": [ "chopping board-f5b35f085b577243", "small tupperware-dafe051e20c99254" ] }, { "key": "P08-20240617-130401:e9c493a266ba7c6a65a0e574f59289f53c97739f", "video_id": "P08-20240617-130401", "question": "After putting the pot on the stove, what number would I get if I added the number of stove-top clicks and the number of times I ground salt?", "answer_choice_0": "14", "answer_choice_1": "5", "answer_choice_2": "19", "answer_choice_3": "3", "answer_choice_4": "11", "answer_id": 0, "answer": "14", "question_type": [ "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counting", "Audio Reasoning", "Numerical Reasoning" ], "objects_of_interest": [ "phone-85724d9e5d94148c", "tea towel-427614b9e081b7aa", "piece of paper-d613d383a5f78cb8", "coffee grinder-45637c4bff924e2e", "pot lid-258d022cd42bb7d6", "chopping board-f5b35f085b577243", "cucumber-a3816067e6a02c7e", "bell peper-5a94bfdd100b14d2", "small tupperware-dafe051e20c99254", "tomato container-b3686c57f082a560", "bag of spinach-94fa6852d43ba7d3", "bag of coriander-9acfec1e3b217815", "vegetables3-84e9781ba0e9feeb", "refrigerator drawer-7b9ec08f24848bbd", "vegetables5-eb27e0c1ab0e9b05", "food packet-43914a347655d5fb", "pot-cfda9ec34cb81ff2", "top of pot-16072779d839ebbb", "plastic container-d31edf0e2295e132", "beer can-807a1ee36fe8a920", "kitchen scale-7e60a8366c417004", "knife holder-4b9f706781e22ed1", "onion-14d2cff3d51668eb", "vegetable basket-c133eead82decd11", "measuring cups-4822959e8b96b839", "salad bowl-a6730b4ebe7722b3", "plastic film-f42ed0a200f039fd", "knife-7c00b19f7197bc8a", "smaller knife-d5aeabbc2652fa5c", "cherry tomatoes-6a474b3d9ba6efe7", "new tomato1-200289016707d2bf", "knife2-cf2ad08a4e381d0e", "knife3-5f4ebcb05f43e15c", "bag of celery-cb8bd824383db776", "celery-10ffde68e50c1964", "vegetables4-ec244ab253d03441", "paper towel-c0ae417842762647", "can of beans-fabced34945a4ecc", "pasta sauce-21f067e8fb677428", "some can-e36ab72107ebfa4a", "can of tuna-30e76b3f7006aeee", "peeler-0ac0790a4a392de6", "spoon-0a26c5d695214bab", "large spoon-fdd604e711dcc288", "carrots-c0ec32ee12321e86", "can opener-c4587da967c63301", "bowl-6420618f1eb48874", "spoon2-6b1f15c0179017d7", "butter-14ab53559b0eaf05", "sauce bottle-e3131925669430af", "single pasta-bc3ade7b93c01e88", "small plate-a38e1b44926880a3", "fork-fa19eb1c0295a587", "aluminium foil-a785ab50dfbc3d25", "pack of aluminum foil-45d0c1ffaf749c02", "pack of plastic sheets-879c88e871c58955", "cleaning cloth-c5ea3975c7fb613c", "cleaning sponge-b460d8fda33273cd", "bottle of dishwashing liquid-b3a1618fe39a6003", "bowl2-0e7adbb5d4d969cc", "aluminum tin-0e724679ec386af9", "pan-38b2fd4336ec4d62", "bottle-196950f1857db3cb", "tin lid-d09e123d69254881", "bottle oil-2a696f22a7d81899", "box-1770a3d4b1a741ac", "jar-5b4d2f8825b08700", "glass container-9db809eac769b24c", "lid-93bb46c08d0089fa", "spoon3-a3ac5412d2e808ce", "bottle2-40a912b668615655", "glass-6ded6ed7a2319bdc", "paper roll-8dd136af95c41246" ], "reasoning": "1. At 01:41, I observe the user place the pot (pot-cfda9ec34cb81ff2) on top of the stove.\n2. At 01:43, I observe the user turn the knob to adjust the heat.\n3. From 01:43 to 01:46, I hear the igniter click 11 times.\n4. From 01:51 to 01:53, I observe the user grind the salt 3 times over the pot (pot-cfda9ec34cb81ff2).\n5. I add 11 and 3 to get 14.", "referenced_object_masks": [ "pot-cfda9ec34cb81ff2" ] }, { "key": "P08-20240617-130401:eaa4e39f5c4249102fe512bafdb63276e46000d4", "video_id": "P08-20240617-130401", "question": "When I open the drawer to remove the measuring cups, what item is the closest to my left hand?", "answer_choice_0": "Spoon", "answer_choice_1": "Can opener", "answer_choice_2": "Knife", "answer_choice_3": "Spatula", "answer_choice_4": "Kitchen scissors", "answer_id": 4, "answer": "Kitchen scissors", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "phone-85724d9e5d94148c", "tea towel-427614b9e081b7aa", "piece of paper-d613d383a5f78cb8", "coffee grinder-45637c4bff924e2e", "pot lid-258d022cd42bb7d6", "chopping board-f5b35f085b577243", "cucumber-a3816067e6a02c7e", "bell peper-5a94bfdd100b14d2", "small tupperware-dafe051e20c99254", "tomato container-b3686c57f082a560", "bag of spinach-94fa6852d43ba7d3", "bag of coriander-9acfec1e3b217815", "vegetables3-84e9781ba0e9feeb", "refrigerator drawer-7b9ec08f24848bbd", "vegetables5-eb27e0c1ab0e9b05", "food packet-43914a347655d5fb", "pot-cfda9ec34cb81ff2", "top of pot-16072779d839ebbb", "plastic container-d31edf0e2295e132", "beer can-807a1ee36fe8a920", "kitchen scale-7e60a8366c417004", "knife holder-4b9f706781e22ed1", "onion-14d2cff3d51668eb", "vegetable basket-c133eead82decd11", "measuring cups-4822959e8b96b839", "salad bowl-a6730b4ebe7722b3", "plastic film-f42ed0a200f039fd", "knife-7c00b19f7197bc8a", "smaller knife-d5aeabbc2652fa5c", "cherry tomatoes-6a474b3d9ba6efe7", "new tomato1-200289016707d2bf", "knife2-cf2ad08a4e381d0e", "knife3-5f4ebcb05f43e15c", "bag of celery-cb8bd824383db776", "celery-10ffde68e50c1964", "vegetables4-ec244ab253d03441", "paper towel-c0ae417842762647", "can of beans-fabced34945a4ecc", "pasta sauce-21f067e8fb677428", "some can-e36ab72107ebfa4a", "can of tuna-30e76b3f7006aeee", "peeler-0ac0790a4a392de6", "spoon-0a26c5d695214bab", "large spoon-fdd604e711dcc288", "carrots-c0ec32ee12321e86", "can opener-c4587da967c63301", "bowl-6420618f1eb48874", "spoon2-6b1f15c0179017d7", "butter-14ab53559b0eaf05", "sauce bottle-e3131925669430af", "single pasta-bc3ade7b93c01e88", "small plate-a38e1b44926880a3", "fork-fa19eb1c0295a587", "aluminium foil-a785ab50dfbc3d25", "pack of aluminum foil-45d0c1ffaf749c02", "pack of plastic sheets-879c88e871c58955", "cleaning cloth-c5ea3975c7fb613c", "cleaning sponge-b460d8fda33273cd", "bottle of dishwashing liquid-b3a1618fe39a6003", "bowl2-0e7adbb5d4d969cc", "aluminum tin-0e724679ec386af9", "pan-38b2fd4336ec4d62", "bottle-196950f1857db3cb", "tin lid-d09e123d69254881", "bottle oil-2a696f22a7d81899", "box-1770a3d4b1a741ac", "jar-5b4d2f8825b08700", "glass container-9db809eac769b24c", "lid-93bb46c08d0089fa", "spoon3-a3ac5412d2e808ce", "bottle2-40a912b668615655", "glass-6ded6ed7a2319bdc", "paper roll-8dd136af95c41246" ], "reasoning": "1. I looked for the moment the user opened the drawer to pick the measuring cups (measuring cups-4822959e8b96b839) at 02:34. \n2. I paused the video to assess what other items are inside the drawer. The user has their left hand on the tray inside the drawer, right on top of a potato peeler (peeler-0ac0790a4a392de6).\n3. Immediately to the left of the user's left hand is a pair of white kitchen scissors . Therefore, the object closest to the user's left when they open the drawer are the kitchen scissors .", "referenced_object_masks": [ "peeler-0ac0790a4a392de6", "measuring cups-4822959e8b96b839" ] }, { "key": "P08-20240617-184909:17438df2fdcad9adbcbe98274439322732e24881", "video_id": "P08-20240617-184909", "question": "What is the difference between how many times I cut the chilli after cutting off the end and how much the chilli weighs?", "answer_choice_0": "14", "answer_choice_1": "11", "answer_choice_2": "12", "answer_choice_3": "13", "answer_choice_4": "10", "answer_id": 2, "answer": "12", "question_type": [ "Action Recognition", "Counting", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "phone-42c87742240840b5", "plate-361fb404563eead5", "baking tray-b3ad1cbdafc0c517", "package of aluminum foil-aacf65d8c081894a", "foil-232806f05ae312b8", "first single fish cake-eef19b89390c8d4f", "knife-1ea848f8ac021aab", "packaging-0c652fa7861c6d4e", "fish cake2-e59c823a262bf319", "fish cake3-4e7a66437dab735d", "water bottle-353075a5dd1cb9fd", "fish cake packaging-19034de1b82cb506", "bag of vegetables-ada8975f7d9b7828", "another bag of vegetables-c2b80aabc9b7c5d2", "chopping board-5038a9a5137f3473", "air fryer-8eb47c13480c4726", "clear bowl-e5d6058cbfc51868", "knife holder-60a00c7b18c37453", "kitchen scale-744ffc4507f71aad", "bowl-cbf69d89dc48a148", "paper towel-2a88dfe85b274d1f", "bell pepper-38a8fbd533f963ff", "onion-d662a09583449534", "tomato bowl-ccb9d5303f95dbfc", "chilli-cf9f47c8a2cacab7", "bag of couscous-20826eedd286affe", "kettle-fc60b79e3a1ade3f", "pot-5de34c4f5af16197", "baking tray2-b9066a55cb65e9f6", "measuring spoons-99f7c2e0b06b248e", "couscous bag-53243da314f150d0", "spoon-fe400a2753b8d3df", "pan-39323c0daccefa4f", "garlic-c324b0433066a029", "oven mitt-44c62254b0bb306d", "spatula2-cc1f179a93614be6", "package of garlic-7c818e1faadc4798", "bottle of olive oil-4f15f428e6c84f51", "spoon2-a73cfe81343f16a5", "top of pot-06f8cfdf733b2b2a", "colander-ebb5cac0adcc4521", "small pan-b0cea205b713ca5f", "pot2-5ea1ae8cfc84ec9f", "pot top2-1fa05adb6453c57e", "small coffee pot-2300905f719a9571", "knife2-b0e12860e818c31e", "spatula-de7371adc061725d", "spoon3-871a33554ef3cd62", "peeler-7739174f9e0f6874", "tongs-f39cde91d6e92b9a", "spoon4-976725ae159c5c1d", "small plate-724aef7ad6f5f2ec", "bottle-61367938b822e3fd", "top of tupperware-0f3ffebcd8a1ecfc", "small bowl-c0ac0b5a68fdf5bb", "air fryer tray-f9f88e22a176ca58", "salt grinder-ed7009b5c28257d2", "pepper grinder-b1476962739ce5bf", "bottle of sesame oil-07137ca60d286e0f", "sesame oil bottle cap-44ea37b8fb7230dd", "sponge-f95a76572317480d", "bottle of dish soap-c24e2894bc584300", "air fryer vents-41c76fe48d17f1b0", "mug-d7a005ac3e09718b", "tea towel-4ed28f5e7d510235", "small glass tupperware-26cbef907d4e5064", "paper towel2-a3db5be1a45512d2", "waste-844bdf381b55ede4", "glass-58b2885c8bbc8c7b", "bowl2-693355edb2ab5b55", "plate2-be91db8ab97e6471", "plastic container-95b0e25bed999af9", "pasta spoon-34f4b8a5b86e05f8", "plate3-fc2153c189f37dfe", "dish washer bottle-1451b29537682bd5", "sponge2-4220d04e5ee109f0", "dishrag-bbe2df7b38660613", "fork-c8c2853a78129edb", "fork2-ae8c8e02238f62e6", "knife3-e101e654d4a57fa7", "knife4-e8308928a3c6425f" ], "reasoning": "1. I watched the whole video and saw the user weigh 1 chili (chilli-cf9f47c8a2cacab7) on the kitchen scale (kitchen scale-744ffc4507f71aad) at 08:16.\n2. The chilli (chilli-cf9f47c8a2cacab7) weighed 4 g at 08:16. \n3. At 08:21, the user starts to cut the chilli (chilli-cf9f47c8a2cacab7) into pieces on the chopping board (chopping board-5038a9a5137f3473) after cutting off the end.\n4. The user cut the chilli (chilli-cf9f47c8a2cacab7) into 16 pieces.\n5. The difference between the number of times the user cut the chilli (chilli-cf9f47c8a2cacab7), 16 and the how much the chilli (chilli-cf9f47c8a2cacab7) weighed, 4 is 12.", "referenced_object_masks": [ "kitchen scale-744ffc4507f71aad", "chilli-cf9f47c8a2cacab7", "chopping board-5038a9a5137f3473" ] }, { "key": "P08-20240617-184909:21d890beb9e861a85a25d191c2e4634580382bc3", "video_id": "P08-20240617-184909", "question": "How many times did I flip a fish cake?", "answer_choice_0": "3", "answer_choice_1": "5", "answer_choice_2": "9", "answer_choice_3": "2", "answer_choice_4": "6", "answer_id": 4, "answer": "6", "question_type": [ "Action Recognition", "Counting", "Object Recognition" ], "objects_of_interest": [ "knife4-e8308928a3c6425f", "knife3-e101e654d4a57fa7", "fork2-ae8c8e02238f62e6", "fork-c8c2853a78129edb", "dishrag-bbe2df7b38660613", "sponge2-4220d04e5ee109f0", "dish washer bottle-1451b29537682bd5", "plate3-fc2153c189f37dfe", "pasta spoon-34f4b8a5b86e05f8", "plastic container-95b0e25bed999af9", "plate2-be91db8ab97e6471", "bowl2-693355edb2ab5b55", "glass-58b2885c8bbc8c7b", "waste-844bdf381b55ede4", "paper towel2-a3db5be1a45512d2", "small glass tupperware-26cbef907d4e5064", "tea towel-4ed28f5e7d510235", "mug-d7a005ac3e09718b", "air fryer vents-41c76fe48d17f1b0", "bottle of dish soap-c24e2894bc584300", "sponge-f95a76572317480d", "sesame oil bottle cap-44ea37b8fb7230dd", "bottle of sesame oil-07137ca60d286e0f", "pepper grinder-b1476962739ce5bf", "salt grinder-ed7009b5c28257d2", "air fryer tray-f9f88e22a176ca58", "small bowl-c0ac0b5a68fdf5bb", "top of tupperware-0f3ffebcd8a1ecfc", "bottle-61367938b822e3fd", "small plate-724aef7ad6f5f2ec", "spoon4-976725ae159c5c1d", "tongs-f39cde91d6e92b9a", "peeler-7739174f9e0f6874", "spoon3-871a33554ef3cd62", "spatula2-cc1f179a93614be6", "knife2-b0e12860e818c31e", "small coffee pot-2300905f719a9571", "pot top2-1fa05adb6453c57e", "pot2-5ea1ae8cfc84ec9f", "small pan-b0cea205b713ca5f", "colander-ebb5cac0adcc4521", "spoon2-a73cfe81343f16a5", "bottle of olive oil-4f15f428e6c84f51", "spatula-de7371adc061725d", "oven mitt-44c62254b0bb306d", "garlic-c324b0433066a029", "pan-39323c0daccefa4f", "top of pot-06f8cfdf733b2b2a", "spoon-fe400a2753b8d3df", "couscous bag-53243da314f150d0", "measuring spoons-99f7c2e0b06b248e", "baking tray2-b9066a55cb65e9f6", "pot-5de34c4f5af16197", "kettle-fc60b79e3a1ade3f", "bag of couscous-20826eedd286affe", "chilli-cf9f47c8a2cacab7", "tomato bowl-ccb9d5303f95dbfc", "onion-d662a09583449534", "bell pepper-38a8fbd533f963ff", "paper towel-2a88dfe85b274d1f", "bowl-cbf69d89dc48a148", "kitchen scale-744ffc4507f71aad", "knife holder-60a00c7b18c37453", "clear bowl-e5d6058cbfc51868", "air fryer-8eb47c13480c4726", "chopping board-5038a9a5137f3473", "another bag of vegetables-c2b80aabc9b7c5d2", "bag of vegetables-ada8975f7d9b7828", "fish cake packaging-19034de1b82cb506", "water bottle-353075a5dd1cb9fd", "fish cake3-4e7a66437dab735d", "fish cake2-e59c823a262bf319", "packaging-0c652fa7861c6d4e", "knife-1ea848f8ac021aab", "first single fish cake-eef19b89390c8d4f", "foil-232806f05ae312b8", "package of aluminum foil-aacf65d8c081894a", "baking tray-b3ad1cbdafc0c517", "plate-361fb404563eead5", "phone-42c87742240840b5", "package of garlic-7c818e1faadc4798" ], "reasoning": "1. I watched the video in order to find out how many times the user flipped the fish cakes. At 03:04, I noticed the user put the first fish cake (first single fish cake-eef19b89390c8d4f), the second fish cake (fish cake2-e59c823a262bf319), and the third fish cake (fish cake3-4e7a66437dab735d) onto a baking tray (baking tray-b3ad1cbdafc0c517) and into the oven .\n2. At 12:32, I noticed the user pulling the first fish cake (first single fish cake-eef19b89390c8d4f), the second fish cake (fish cake2-e59c823a262bf319), and the third fish cake (fish cake3-4e7a66437dab735d) out of the oven and then placing them down. From 12:39-12:46, the user flipped the first fish cake (first single fish cake-eef19b89390c8d4f), the second fish cake (fish cake2-e59c823a262bf319), and the third fish cake (fish cake3-4e7a66437dab735d) once. I counted this as 3 total flips.\n3. At 27:34, I noticed the user pulling the baking tray (baking tray-b3ad1cbdafc0c517) out of the oven again and placing them down. From 27:38-27:47, I noticed the user flipped the first fish cake (first single fish cake-eef19b89390c8d4f), the second fish cake (fish cake2-e59c823a262bf319), and the third fish cake (fish cake3-4e7a66437dab735d). I counted this as 3 additional flips.\n5. After this point, I did not notice any additional flips, leaving the final answer as 6 flips.", "referenced_object_masks": [ "baking tray-b3ad1cbdafc0c517", "fish cake2-e59c823a262bf319", "fish cake3-4e7a66437dab735d", "first single fish cake-eef19b89390c8d4f" ] }, { "key": "P08-20240617-184909:24d1f89c2a18fa4bd545861605862ca2a95945f1", "video_id": "P08-20240617-184909", "question": "How much time elapsed between when I first grabbed the bottle of oil and when I put it away?", "answer_choice_0": "2 minutes and 15 seconds", "answer_choice_1": "3 minutes and 2 seconds", "answer_choice_2": "2 minutes and 30 seconds", "answer_choice_3": "2 minutes and 51 seconds", "answer_choice_4": "3 minutes and 18 seconds", "answer_id": 3, "answer": "2 minutes and 51 seconds", "question_type": [ "Temporal Reasoning", "Action Recognition", "Object Recognition", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "knife4-e8308928a3c6425f", "knife3-e101e654d4a57fa7", "fork2-ae8c8e02238f62e6", "fork-c8c2853a78129edb", "dishrag-bbe2df7b38660613", "sponge2-4220d04e5ee109f0", "dish washer bottle-1451b29537682bd5", "plate3-fc2153c189f37dfe", "pasta spoon-34f4b8a5b86e05f8", "plastic container-95b0e25bed999af9", "plate2-be91db8ab97e6471", "bowl2-693355edb2ab5b55", "glass-58b2885c8bbc8c7b", "waste-844bdf381b55ede4", "paper towel2-a3db5be1a45512d2", "small glass tupperware-26cbef907d4e5064", "tea towel-4ed28f5e7d510235", "mug-d7a005ac3e09718b", "air fryer vents-41c76fe48d17f1b0", "bottle of dish soap-c24e2894bc584300", "sponge-f95a76572317480d", "sesame oil bottle cap-44ea37b8fb7230dd", "bottle of sesame oil-07137ca60d286e0f", "pepper grinder-b1476962739ce5bf", "salt grinder-ed7009b5c28257d2", "air fryer tray-f9f88e22a176ca58", "small bowl-c0ac0b5a68fdf5bb", "top of tupperware-0f3ffebcd8a1ecfc", "bottle-61367938b822e3fd", "small plate-724aef7ad6f5f2ec", "spoon4-976725ae159c5c1d", "tongs-f39cde91d6e92b9a", "peeler-7739174f9e0f6874", "spoon3-871a33554ef3cd62", "spatula2-cc1f179a93614be6", "knife2-b0e12860e818c31e", "small coffee pot-2300905f719a9571", "pot top2-1fa05adb6453c57e", "pot2-5ea1ae8cfc84ec9f", "small pan-b0cea205b713ca5f", "colander-ebb5cac0adcc4521", "spoon2-a73cfe81343f16a5", "bottle of olive oil-4f15f428e6c84f51", "spatula-de7371adc061725d", "oven mitt-44c62254b0bb306d", "garlic-c324b0433066a029", "pan-39323c0daccefa4f", "top of pot-06f8cfdf733b2b2a", "spoon-fe400a2753b8d3df", "couscous bag-53243da314f150d0", "measuring spoons-99f7c2e0b06b248e", "baking tray2-b9066a55cb65e9f6", "pot-5de34c4f5af16197", "kettle-fc60b79e3a1ade3f", "bag of couscous-20826eedd286affe", "chilli-cf9f47c8a2cacab7", "tomato bowl-ccb9d5303f95dbfc", "onion-d662a09583449534", "bell pepper-38a8fbd533f963ff", "paper towel-2a88dfe85b274d1f", "bowl-cbf69d89dc48a148", "kitchen scale-744ffc4507f71aad", "knife holder-60a00c7b18c37453", "clear bowl-e5d6058cbfc51868", "air fryer-8eb47c13480c4726", "chopping board-5038a9a5137f3473", "another bag of vegetables-c2b80aabc9b7c5d2", "bag of vegetables-ada8975f7d9b7828", "fish cake packaging-19034de1b82cb506", "water bottle-353075a5dd1cb9fd", "fish cake3-4e7a66437dab735d", "fish cake2-e59c823a262bf319", "packaging-0c652fa7861c6d4e", "knife-1ea848f8ac021aab", "first single fish cake-eef19b89390c8d4f", "foil-232806f05ae312b8", "package of aluminum foil-aacf65d8c081894a", "baking tray-b3ad1cbdafc0c517", "plate-361fb404563eead5", "phone-42c87742240840b5", "package of garlic-7c818e1faadc4798" ], "reasoning": "1. I watched the video in order to figure out when the user grabbed a bottle of oil. At 12:57, I noticed the user grabbing a tall, thin bottle of olive oil (bottle of olive oil-4f15f428e6c84f51).\n2. I noticed the user did not touch the bottle (bottle of olive oil-4f15f428e6c84f51), but at 13:12 the user grabbed it again and from 13:17-13:22, the user poured some into a spoon (spoon2-a73cfe81343f16a5) and then added it to the pan (pan-39323c0daccefa4f).\n3. At 13:23, I noticed the user put down the bottle (bottle of olive oil-4f15f428e6c84f51) on the countertop. I continued watching to see when they put it away.\n4. For a long period of time, the bottle (bottle of olive oil-4f15f428e6c84f51) is not touched. However, at 15:48, I noticed the user picked up the bottle (bottle of olive oil-4f15f428e6c84f51) and put it away to where they had originally retrieved it from.\n5. I subtracted 12:57 from 15:48 (15:48-12:57) in order to get a final answer of 2 minutes and 51 seconds.", "referenced_object_masks": [ "pan-39323c0daccefa4f", "spoon2-a73cfe81343f16a5", "bottle of olive oil-4f15f428e6c84f51" ] }, { "key": "P08-20240617-184909:2984ff94d918e1643d8d74b9cf3d0ebfbd16e54d", "video_id": "P08-20240617-184909", "question": "In what order do I wash the items in the sink?", "answer_choice_0": "Top of pot, cutting board, plate", "answer_choice_1": "Cutting board, top of pot, plate", "answer_choice_2": "Plate, top of pot, cutting board", "answer_choice_3": "Cutting board, plate, top of pot", "answer_choice_4": "Top of pot, cutting board, bowl", "answer_id": 1, "answer": "Cutting board, top of pot, plate", "question_type": [ "Action Recognition", "Object Recognition" ], "objects_of_interest": [ "phone-42c87742240840b5", "plate-361fb404563eead5", "baking tray-b3ad1cbdafc0c517", "package of aluminum foil-aacf65d8c081894a", "foil-232806f05ae312b8", "first single fish cake-eef19b89390c8d4f", "knife-1ea848f8ac021aab", "packaging-0c652fa7861c6d4e", "fish cake2-e59c823a262bf319", "fish cake3-4e7a66437dab735d", "water bottle-353075a5dd1cb9fd", "fish cake packaging-19034de1b82cb506", "bag of vegetables-ada8975f7d9b7828", "another bag of vegetables-c2b80aabc9b7c5d2", "chopping board-5038a9a5137f3473", "air fryer-8eb47c13480c4726", "clear bowl-e5d6058cbfc51868", "knife holder-60a00c7b18c37453", "kitchen scale-744ffc4507f71aad", "bowl-cbf69d89dc48a148", "paper towel-2a88dfe85b274d1f", "bell pepper-38a8fbd533f963ff", "onion-d662a09583449534", "tomato bowl-ccb9d5303f95dbfc", "chilli-cf9f47c8a2cacab7", "bag of couscous-20826eedd286affe", "kettle-fc60b79e3a1ade3f", "pot-5de34c4f5af16197", "baking tray2-b9066a55cb65e9f6", "measuring spoons-99f7c2e0b06b248e", "couscous bag-53243da314f150d0", "spoon-fe400a2753b8d3df", "pan-39323c0daccefa4f", "garlic-c324b0433066a029", "oven mitt-44c62254b0bb306d", "spatula2-cc1f179a93614be6", "package of garlic-7c818e1faadc4798", "bottle of olive oil-4f15f428e6c84f51", "spoon2-a73cfe81343f16a5", "top of pot-06f8cfdf733b2b2a", "colander-ebb5cac0adcc4521", "small pan-b0cea205b713ca5f", "pot2-5ea1ae8cfc84ec9f", "pot top2-1fa05adb6453c57e", "small coffee pot-2300905f719a9571", "knife2-b0e12860e818c31e", "spatula-de7371adc061725d", "spoon3-871a33554ef3cd62", "peeler-7739174f9e0f6874", "tongs-f39cde91d6e92b9a", "spoon4-976725ae159c5c1d", "small plate-724aef7ad6f5f2ec", "bottle-61367938b822e3fd", "top of tupperware-0f3ffebcd8a1ecfc", "small bowl-c0ac0b5a68fdf5bb", "air fryer tray-f9f88e22a176ca58", "salt grinder-ed7009b5c28257d2", "pepper grinder-b1476962739ce5bf", "bottle of sesame oil-07137ca60d286e0f", "sesame oil bottle cap-44ea37b8fb7230dd", "sponge-f95a76572317480d", "bottle of dish soap-c24e2894bc584300", "air fryer vents-41c76fe48d17f1b0", "mug-d7a005ac3e09718b", "tea towel-4ed28f5e7d510235", "small glass tupperware-26cbef907d4e5064", "paper towel2-a3db5be1a45512d2", "waste-844bdf381b55ede4", "glass-58b2885c8bbc8c7b", "bowl2-693355edb2ab5b55", "plate2-be91db8ab97e6471", "plastic container-95b0e25bed999af9", "pasta spoon-34f4b8a5b86e05f8", "plate3-fc2153c189f37dfe", "dish washer bottle-1451b29537682bd5", "sponge2-4220d04e5ee109f0", "dishrag-bbe2df7b38660613", "fork-c8c2853a78129edb", "fork2-ae8c8e02238f62e6", "knife3-e101e654d4a57fa7", "knife4-e8308928a3c6425f" ], "reasoning": "1. I noticed the user washing the cutitng board at 21:03. \n2. I noticed the user washing the top of the pot at 21:20. \n3. I noticed the user washing the bowl at 21:35.\n4. I noticed the user washing the plate at 21:38.", "referenced_object_masks": [] }, { "key": "P08-20240617-184909:4dd10ab95b74b33be16aaad6841edc1596af7511", "video_id": "P08-20240617-184909", "question": "What is the amount of time in mm:ss format if I add together the amount of time to put the items away from the drying rack near the sink and the amount of time elapsed from when I add the broccoli to the frying pan and when I sneeze?", "answer_choice_0": "01:50", "answer_choice_1": "02:05", "answer_choice_2": "02:00", "answer_choice_3": "01:56", "answer_choice_4": "02:03", "answer_id": 2, "answer": "02:00", "question_type": [ "Action Recognition", "Event Occurence", "Numerical Reasoning", "Counting", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "phone-42c87742240840b5", "plate-361fb404563eead5", "baking tray-b3ad1cbdafc0c517", "package of aluminum foil-aacf65d8c081894a", "foil-232806f05ae312b8", "first single fish cake-eef19b89390c8d4f", "knife-1ea848f8ac021aab", "packaging-0c652fa7861c6d4e", "fish cake2-e59c823a262bf319", "fish cake3-4e7a66437dab735d", "water bottle-353075a5dd1cb9fd", "fish cake packaging-19034de1b82cb506", "bag of vegetables-ada8975f7d9b7828", "another bag of vegetables-c2b80aabc9b7c5d2", "chopping board-5038a9a5137f3473", "air fryer-8eb47c13480c4726", "clear bowl-e5d6058cbfc51868", "knife holder-60a00c7b18c37453", "kitchen scale-744ffc4507f71aad", "bowl-cbf69d89dc48a148", "paper towel-2a88dfe85b274d1f", "bell pepper-38a8fbd533f963ff", "onion-d662a09583449534", "tomato bowl-ccb9d5303f95dbfc", "chilli-cf9f47c8a2cacab7", "bag of couscous-20826eedd286affe", "kettle-fc60b79e3a1ade3f", "pot-5de34c4f5af16197", "baking tray2-b9066a55cb65e9f6", "measuring spoons-99f7c2e0b06b248e", "couscous bag-53243da314f150d0", "spoon-fe400a2753b8d3df", "pan-39323c0daccefa4f", "garlic-c324b0433066a029", "oven mitt-44c62254b0bb306d", "spatula2-cc1f179a93614be6", "package of garlic-7c818e1faadc4798", "bottle of olive oil-4f15f428e6c84f51", "spoon2-a73cfe81343f16a5", "top of pot-06f8cfdf733b2b2a", "colander-ebb5cac0adcc4521", "small pan-b0cea205b713ca5f", "pot2-5ea1ae8cfc84ec9f", "pot top2-1fa05adb6453c57e", "small coffee pot-2300905f719a9571", "knife2-b0e12860e818c31e", "spatula-de7371adc061725d", "spoon3-871a33554ef3cd62", "peeler-7739174f9e0f6874", "tongs-f39cde91d6e92b9a", "spoon4-976725ae159c5c1d", "small plate-724aef7ad6f5f2ec", "bottle-61367938b822e3fd", "top of tupperware-0f3ffebcd8a1ecfc", "small bowl-c0ac0b5a68fdf5bb", "air fryer tray-f9f88e22a176ca58", "salt grinder-ed7009b5c28257d2", "pepper grinder-b1476962739ce5bf", "bottle of sesame oil-07137ca60d286e0f", "sesame oil bottle cap-44ea37b8fb7230dd", "sponge-f95a76572317480d", "bottle of dish soap-c24e2894bc584300", "air fryer vents-41c76fe48d17f1b0", "mug-d7a005ac3e09718b", "tea towel-4ed28f5e7d510235", "small glass tupperware-26cbef907d4e5064", "paper towel2-a3db5be1a45512d2", "waste-844bdf381b55ede4", "glass-58b2885c8bbc8c7b", "bowl2-693355edb2ab5b55", "plate2-be91db8ab97e6471", "plastic container-95b0e25bed999af9", "pasta spoon-34f4b8a5b86e05f8", "plate3-fc2153c189f37dfe", "dish washer bottle-1451b29537682bd5", "sponge2-4220d04e5ee109f0", "dishrag-bbe2df7b38660613", "fork-c8c2853a78129edb", "fork2-ae8c8e02238f62e6", "knife3-e101e654d4a57fa7", "knife4-e8308928a3c6425f" ], "reasoning": "1. I watched the whole video and saw the user put away items from the drying rack from 14:37 to 15:14.\n2. 14:37 to 15:14 is 37 seconds.\n3. At 18:22, the user added broccoli to the frying pan (pan-39323c0daccefa4f).\n4. At 19:45, the user sneezes.\n5. The time elapsed from putting the broccoli in the frying pan (pan-39323c0daccefa4f) to the sneeze is 1 minute and 23 seconds.\n6. Add together the 37 seconds and 1 minute and 23 seconds to get 01:60.\n7. 60 seconds is 1 minute, so they need to add 1 minute to the time to get 02:00 minutes.", "referenced_object_masks": [ "pan-39323c0daccefa4f" ] }, { "key": "P08-20240617-184909:5d1db76d7f4b4866df74dcb7d43930e754d26d94", "video_id": "P08-20240617-184909", "question": "In what order do I cut the ingredients?", "answer_choice_0": "Red bell pepper, onions, chili", "answer_choice_1": "Chili, onions, red bell pepper", "answer_choice_2": "Red bell pepper, chili, onions", "answer_choice_3": "Onions, red bell pepper, chili", "answer_choice_4": "Onions, chili, red bell pepper", "answer_id": 4, "answer": "Onions, chili, red bell pepper", "question_type": [ "Counting", "Numerical Reasoning", "Object Recognition", "Situational Awareness", "State Changes" ], "objects_of_interest": [ "phone-42c87742240840b5", "plate-361fb404563eead5", "baking tray-b3ad1cbdafc0c517", "package of aluminum foil-aacf65d8c081894a", "foil-232806f05ae312b8", "first single fish cake-eef19b89390c8d4f", "knife-1ea848f8ac021aab", "packaging-0c652fa7861c6d4e", "fish cake2-e59c823a262bf319", "fish cake3-4e7a66437dab735d", "water bottle-353075a5dd1cb9fd", "fish cake packaging-19034de1b82cb506", "bag of vegetables-ada8975f7d9b7828", "another bag of vegetables-c2b80aabc9b7c5d2", "chopping board-5038a9a5137f3473", "air fryer-8eb47c13480c4726", "clear bowl-e5d6058cbfc51868", "knife holder-60a00c7b18c37453", "kitchen scale-744ffc4507f71aad", "bowl-cbf69d89dc48a148", "paper towel-2a88dfe85b274d1f", "bell pepper-38a8fbd533f963ff", "onion-d662a09583449534", "tomato bowl-ccb9d5303f95dbfc", "chilli-cf9f47c8a2cacab7", "bag of couscous-20826eedd286affe", "kettle-fc60b79e3a1ade3f", "pot-5de34c4f5af16197", "baking tray2-b9066a55cb65e9f6", "measuring spoons-99f7c2e0b06b248e", "couscous bag-53243da314f150d0", "spoon-fe400a2753b8d3df", "pan-39323c0daccefa4f", "garlic-c324b0433066a029", "oven mitt-44c62254b0bb306d", "spatula2-cc1f179a93614be6", "package of garlic-7c818e1faadc4798", "bottle of olive oil-4f15f428e6c84f51", "spoon2-a73cfe81343f16a5", "top of pot-06f8cfdf733b2b2a", "colander-ebb5cac0adcc4521", "small pan-b0cea205b713ca5f", "pot2-5ea1ae8cfc84ec9f", "pot top2-1fa05adb6453c57e", "small coffee pot-2300905f719a9571", "knife2-b0e12860e818c31e", "spatula-de7371adc061725d", "spoon3-871a33554ef3cd62", "peeler-7739174f9e0f6874", "tongs-f39cde91d6e92b9a", "spoon4-976725ae159c5c1d", "small plate-724aef7ad6f5f2ec", "bottle-61367938b822e3fd", "top of tupperware-0f3ffebcd8a1ecfc", "small bowl-c0ac0b5a68fdf5bb", "air fryer tray-f9f88e22a176ca58", "salt grinder-ed7009b5c28257d2", "pepper grinder-b1476962739ce5bf", "bottle of sesame oil-07137ca60d286e0f", "sesame oil bottle cap-44ea37b8fb7230dd", "sponge-f95a76572317480d", "bottle of dish soap-c24e2894bc584300", "air fryer vents-41c76fe48d17f1b0", "mug-d7a005ac3e09718b", "tea towel-4ed28f5e7d510235", "small glass tupperware-26cbef907d4e5064", "paper towel2-a3db5be1a45512d2", "waste-844bdf381b55ede4", "glass-58b2885c8bbc8c7b", "bowl2-693355edb2ab5b55", "plate2-be91db8ab97e6471", "plastic container-95b0e25bed999af9", "pasta spoon-34f4b8a5b86e05f8", "plate3-fc2153c189f37dfe", "dish washer bottle-1451b29537682bd5", "sponge2-4220d04e5ee109f0", "dishrag-bbe2df7b38660613", "fork-c8c2853a78129edb", "fork2-ae8c8e02238f62e6", "knife3-e101e654d4a57fa7", "knife4-e8308928a3c6425f" ], "reasoning": "1. I noticed the use cut the onions (onion-d662a09583449534) at 7:54. \n2. I noticed the user cut chilli (chilli-cf9f47c8a2cacab7) at 8:22. \n3. I noticed the user cut red bell pepper (bell pepper-38a8fbd533f963ff) at 9:53.", "referenced_object_masks": [ "onion-d662a09583449534", "chilli-cf9f47c8a2cacab7", "bell pepper-38a8fbd533f963ff" ] }, { "key": "P08-20240617-184909:6aa9ae0108e34b3c41c1792a2005daae23b6ac2d", "video_id": "P08-20240617-184909", "question": "If I had scanned twice as many different item bar codes, how many different bar codes would I have scanned?", "answer_choice_0": "8", "answer_choice_1": "12", "answer_choice_2": "10", "answer_choice_3": "14", "answer_choice_4": "16", "answer_id": 2, "answer": "10", "question_type": [ "Counting", "Counterfactual", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "knife4-e8308928a3c6425f", "knife3-e101e654d4a57fa7", "fork2-ae8c8e02238f62e6", "fork-c8c2853a78129edb", "dishrag-bbe2df7b38660613", "sponge2-4220d04e5ee109f0", "dish washer bottle-1451b29537682bd5", "plate3-fc2153c189f37dfe", "pasta spoon-34f4b8a5b86e05f8", "plastic container-95b0e25bed999af9", "plate2-be91db8ab97e6471", "bowl2-693355edb2ab5b55", "glass-58b2885c8bbc8c7b", "waste-844bdf381b55ede4", "paper towel2-a3db5be1a45512d2", "small glass tupperware-26cbef907d4e5064", "tea towel-4ed28f5e7d510235", "mug-d7a005ac3e09718b", "air fryer vents-41c76fe48d17f1b0", "bottle of dish soap-c24e2894bc584300", "sponge-f95a76572317480d", "sesame oil bottle cap-44ea37b8fb7230dd", "bottle of sesame oil-07137ca60d286e0f", "pepper grinder-b1476962739ce5bf", "salt grinder-ed7009b5c28257d2", "air fryer tray-f9f88e22a176ca58", "small bowl-c0ac0b5a68fdf5bb", "top of tupperware-0f3ffebcd8a1ecfc", "bottle-61367938b822e3fd", "small plate-724aef7ad6f5f2ec", "spoon4-976725ae159c5c1d", "tongs-f39cde91d6e92b9a", "peeler-7739174f9e0f6874", "spoon3-871a33554ef3cd62", "spatula2-cc1f179a93614be6", "knife2-b0e12860e818c31e", "small coffee pot-2300905f719a9571", "pot top2-1fa05adb6453c57e", "pot2-5ea1ae8cfc84ec9f", "small pan-b0cea205b713ca5f", "colander-ebb5cac0adcc4521", "spoon2-a73cfe81343f16a5", "bottle of olive oil-4f15f428e6c84f51", "spatula-de7371adc061725d", "oven mitt-44c62254b0bb306d", "garlic-c324b0433066a029", "pan-39323c0daccefa4f", "top of pot-06f8cfdf733b2b2a", "spoon-fe400a2753b8d3df", "couscous bag-53243da314f150d0", "measuring spoons-99f7c2e0b06b248e", "baking tray2-b9066a55cb65e9f6", "pot-5de34c4f5af16197", "kettle-fc60b79e3a1ade3f", "bag of couscous-20826eedd286affe", "chilli-cf9f47c8a2cacab7", "tomato bowl-ccb9d5303f95dbfc", "onion-d662a09583449534", "bell pepper-38a8fbd533f963ff", "paper towel-2a88dfe85b274d1f", "bowl-cbf69d89dc48a148", "kitchen scale-744ffc4507f71aad", "knife holder-60a00c7b18c37453", "clear bowl-e5d6058cbfc51868", "air fryer-8eb47c13480c4726", "chopping board-5038a9a5137f3473", "another bag of vegetables-c2b80aabc9b7c5d2", "bag of vegetables-ada8975f7d9b7828", "fish cake packaging-19034de1b82cb506", "water bottle-353075a5dd1cb9fd", "fish cake3-4e7a66437dab735d", "fish cake2-e59c823a262bf319", "packaging-0c652fa7861c6d4e", "knife-1ea848f8ac021aab", "first single fish cake-eef19b89390c8d4f", "foil-232806f05ae312b8", "package of aluminum foil-aacf65d8c081894a", "baking tray-b3ad1cbdafc0c517", "plate-361fb404563eead5", "phone-42c87742240840b5", "package of garlic-7c818e1faadc4798" ], "reasoning": "1. I watched the video to find the times when the user scanned different bar codes using their phone (phone-42c87742240840b5). I noticed the first scan took place from 01:29-01:41, when the user scanned a package which contained a fish cake (first single fish cake-eef19b89390c8d4f).\n2. The 2nd different item scan took place from 02:00-02:07, when the user scanned a different package of fish cakes (packaging-0c652fa7861c6d4e).\n3. The 3rd different item scan scan took place from 03:42-03:48, when the user scanned a package of green beans (bag of vegetables-ada8975f7d9b7828).\n4. The 4th scan took place from 05:18-05:21, when the user scanned a package of broccoli (another bag of vegetables-c2b80aabc9b7c5d2).\n5. The 5th scan took place at 06:24-06:29, when the user scanned a package of couscous (bag of couscous-20826eedd286affe).\n6. I watched the rest of the video and did not notice any other item scans. Since there were 5 different item bar code scans, I multiplied that total by 2 (5x2) in order to get a final answer of 10.", "referenced_object_masks": [ "another bag of vegetables-c2b80aabc9b7c5d2", "phone-42c87742240840b5", "bag of vegetables-ada8975f7d9b7828", "packaging-0c652fa7861c6d4e", "first single fish cake-eef19b89390c8d4f", "bag of couscous-20826eedd286affe" ] }, { "key": "P08-20240617-184909:93ffcb72d2beba42c0c1c1d707f3709e48e337f9", "video_id": "P08-20240617-184909", "question": "After I grab the oil from the back and open the drawer, what object do I touch third?", "answer_choice_0": "Knife", "answer_choice_1": "Knob", "answer_choice_2": "Spatula", "answer_choice_3": "Pan", "answer_choice_4": "Spoon", "answer_id": 1, "answer": "Knob", "question_type": [ "Goal Reasoning", "Counting", "Action Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "phone-42c87742240840b5", "plate-361fb404563eead5", "baking tray-b3ad1cbdafc0c517", "package of aluminum foil-aacf65d8c081894a", "foil-232806f05ae312b8", "first single fish cake-eef19b89390c8d4f", "knife-1ea848f8ac021aab", "packaging-0c652fa7861c6d4e", "fish cake2-e59c823a262bf319", "fish cake3-4e7a66437dab735d", "water bottle-353075a5dd1cb9fd", "fish cake packaging-19034de1b82cb506", "bag of vegetables-ada8975f7d9b7828", "another bag of vegetables-c2b80aabc9b7c5d2", "chopping board-5038a9a5137f3473", "air fryer-8eb47c13480c4726", "clear bowl-e5d6058cbfc51868", "knife holder-60a00c7b18c37453", "kitchen scale-744ffc4507f71aad", "bowl-cbf69d89dc48a148", "paper towel-2a88dfe85b274d1f", "bell pepper-38a8fbd533f963ff", "onion-d662a09583449534", "tomato bowl-ccb9d5303f95dbfc", "chilli-cf9f47c8a2cacab7", "bag of couscous-20826eedd286affe", "kettle-fc60b79e3a1ade3f", "pot-5de34c4f5af16197", "baking tray2-b9066a55cb65e9f6", "measuring spoons-99f7c2e0b06b248e", "couscous bag-53243da314f150d0", "spoon-fe400a2753b8d3df", "pan-39323c0daccefa4f", "garlic-c324b0433066a029", "oven mitt-44c62254b0bb306d", "spatula2-cc1f179a93614be6", "package of garlic-7c818e1faadc4798", "bottle of olive oil-4f15f428e6c84f51", "spoon2-a73cfe81343f16a5", "top of pot-06f8cfdf733b2b2a", "colander-ebb5cac0adcc4521", "small pan-b0cea205b713ca5f", "pot2-5ea1ae8cfc84ec9f", "pot top2-1fa05adb6453c57e", "small coffee pot-2300905f719a9571", "knife2-b0e12860e818c31e", "spatula-de7371adc061725d", "spoon3-871a33554ef3cd62", "peeler-7739174f9e0f6874", "tongs-f39cde91d6e92b9a", "spoon4-976725ae159c5c1d", "small plate-724aef7ad6f5f2ec", "bottle-61367938b822e3fd", "top of tupperware-0f3ffebcd8a1ecfc", "small bowl-c0ac0b5a68fdf5bb", "air fryer tray-f9f88e22a176ca58", "salt grinder-ed7009b5c28257d2", "pepper grinder-b1476962739ce5bf", "bottle of sesame oil-07137ca60d286e0f", "sesame oil bottle cap-44ea37b8fb7230dd", "sponge-f95a76572317480d", "bottle of dish soap-c24e2894bc584300", "air fryer vents-41c76fe48d17f1b0", "mug-d7a005ac3e09718b", "tea towel-4ed28f5e7d510235", "small glass tupperware-26cbef907d4e5064", "paper towel2-a3db5be1a45512d2", "waste-844bdf381b55ede4", "glass-58b2885c8bbc8c7b", "bowl2-693355edb2ab5b55", "plate2-be91db8ab97e6471", "plastic container-95b0e25bed999af9", "pasta spoon-34f4b8a5b86e05f8", "plate3-fc2153c189f37dfe", "dish washer bottle-1451b29537682bd5", "sponge2-4220d04e5ee109f0", "dishrag-bbe2df7b38660613", "fork-c8c2853a78129edb", "fork2-ae8c8e02238f62e6", "knife3-e101e654d4a57fa7", "knife4-e8308928a3c6425f" ], "reasoning": "1. I saw the user grab the bottle of oil ( bottle of olive oil-4f15f428e6c84f51) at 12:56.\n2. I saw the user open the drawer at 12:58.\n3. I saw the user grab a spoon (spoon2-a73cfe81343f16a5) at 13:04. \n4. I saw the user grab a pan (pan-39323c0daccefa4f) at 13:05.\n5. I saw the user touch the knob at 13:06.", "referenced_object_masks": [ "bottle of olive oil-4f15f428e6c84f51", "pan-39323c0daccefa4f", "spoon2-a73cfe81343f16a5" ] }, { "key": "P08-20240617-184909:a26ea5305e46f082ce90791f161513178581e65e", "video_id": "P08-20240617-184909", "question": "After I place all the food I cooked in the video on the plates, how many more unique foods are on the leftmost plate compared to the rightmost plate on the counter?", "answer_choice_0": "4", "answer_choice_1": "1", "answer_choice_2": "2", "answer_choice_3": "3", "answer_choice_4": "5", "answer_id": 1, "answer": "1", "question_type": [ "Egocentric & Spatial Perception", "Action Recognition", "Counting", "Object Recognition" ], "objects_of_interest": [ "knife4-e8308928a3c6425f", "knife3-e101e654d4a57fa7", "fork2-ae8c8e02238f62e6", "fork-c8c2853a78129edb", "dishrag-bbe2df7b38660613", "sponge2-4220d04e5ee109f0", "dish washer bottle-1451b29537682bd5", "plate3-fc2153c189f37dfe", "pasta spoon-34f4b8a5b86e05f8", "plastic container-95b0e25bed999af9", "plate2-be91db8ab97e6471", "bowl2-693355edb2ab5b55", "glass-58b2885c8bbc8c7b", "waste-844bdf381b55ede4", "paper towel2-a3db5be1a45512d2", "small glass tupperware-26cbef907d4e5064", "tea towel-4ed28f5e7d510235", "mug-d7a005ac3e09718b", "air fryer vents-41c76fe48d17f1b0", "bottle of dish soap-c24e2894bc584300", "sponge-f95a76572317480d", "sesame oil bottle cap-44ea37b8fb7230dd", "bottle of sesame oil-07137ca60d286e0f", "pepper grinder-b1476962739ce5bf", "salt grinder-ed7009b5c28257d2", "air fryer tray-f9f88e22a176ca58", "small bowl-c0ac0b5a68fdf5bb", "top of tupperware-0f3ffebcd8a1ecfc", "bottle-61367938b822e3fd", "small plate-724aef7ad6f5f2ec", "spoon4-976725ae159c5c1d", "tongs-f39cde91d6e92b9a", "peeler-7739174f9e0f6874", "spoon3-871a33554ef3cd62", "spatula2-cc1f179a93614be6", "knife2-b0e12860e818c31e", "small coffee pot-2300905f719a9571", "pot top2-1fa05adb6453c57e", "pot2-5ea1ae8cfc84ec9f", "small pan-b0cea205b713ca5f", "colander-ebb5cac0adcc4521", "spoon2-a73cfe81343f16a5", "bottle of olive oil-4f15f428e6c84f51", "spatula-de7371adc061725d", "oven mitt-44c62254b0bb306d", "garlic-c324b0433066a029", "pan-39323c0daccefa4f", "top of pot-06f8cfdf733b2b2a", "spoon-fe400a2753b8d3df", "couscous bag-53243da314f150d0", "measuring spoons-99f7c2e0b06b248e", "baking tray2-b9066a55cb65e9f6", "pot-5de34c4f5af16197", "kettle-fc60b79e3a1ade3f", "bag of couscous-20826eedd286affe", "chilli-cf9f47c8a2cacab7", "tomato bowl-ccb9d5303f95dbfc", "onion-d662a09583449534", "bell pepper-38a8fbd533f963ff", "paper towel-2a88dfe85b274d1f", "bowl-cbf69d89dc48a148", "kitchen scale-744ffc4507f71aad", "knife holder-60a00c7b18c37453", "clear bowl-e5d6058cbfc51868", "air fryer-8eb47c13480c4726", "chopping board-5038a9a5137f3473", "another bag of vegetables-c2b80aabc9b7c5d2", "bag of vegetables-ada8975f7d9b7828", "fish cake packaging-19034de1b82cb506", "water bottle-353075a5dd1cb9fd", "fish cake3-4e7a66437dab735d", "fish cake2-e59c823a262bf319", "packaging-0c652fa7861c6d4e", "knife-1ea848f8ac021aab", "first single fish cake-eef19b89390c8d4f", "foil-232806f05ae312b8", "package of aluminum foil-aacf65d8c081894a", "baking tray-b3ad1cbdafc0c517", "plate-361fb404563eead5", "phone-42c87742240840b5", "package of garlic-7c818e1faadc4798" ], "reasoning": "1. First, I looked for the user to finish placing all the food on the plates (plate3-fc2153c189f37dfe, plate2-be91db8ab97e6471). I found the last food they placed on the plates (plate3-fc2153c189f37dfe, plate2-be91db8ab97e6471) to be the fish cakes (first single fish cake-eef19b89390c8d4f, fish cake2-e59c823a262bf319, fish cake3-4e7a66437dab735d) from 30:28-30:41.\n2. Next, I went back to find the additional food types the user added to each plate (plate3-fc2153c189f37dfe, plate2-be91db8ab97e6471). At 25:49, I saw the user add cooked couscous to the leftmost plate (plate2-be91db8ab97e6471) but not to the rightmost plate (plate3-fc2153c189f37dfe). \n3. Next, I found the moment where the user added cooked stir fry to both plates (plate3-fc2153c189f37dfe, plate2-be91db8ab97e6471) at 28:09-28:34. This is the last unique food item the user adds to either plate (plate3-fc2153c189f37dfe, plate2-be91db8ab97e6471) before the fishcakes (first single fish cake-eef19b89390c8d4f, fish cake2-e59c823a262bf319, fish cake3-4e7a66437dab735d), meaning that the leftmost plate (plate2-be91db8ab97e6471) has 1 more unique food item than the rightmost plate (plate3-fc2153c189f37dfe).", "referenced_object_masks": [ "plate3-fc2153c189f37dfe", "plate2-be91db8ab97e6471", "first single fish cake-eef19b89390c8d4f", "fish cake2-e59c823a262bf319", "fish cake3-4e7a66437dab735d" ] }, { "key": "P08-20240617-184909:b4f9054a14f5b59dca9ebb3371138686b12f65fc", "video_id": "P08-20240617-184909", "question": "What is the product of the number of times that I shake the pot of couscous to the right immediately after putting the pot lid in the sink and the number of minutes that pass on the oven clock from the first and third time that the oven clock time is visible?", "answer_choice_0": "0", "answer_choice_1": "2", "answer_choice_2": "3", "answer_choice_3": "4", "answer_choice_4": "6", "answer_id": 4, "answer": "6", "question_type": [ "Counting", "Situational Awareness" ], "objects_of_interest": [ "phone-42c87742240840b5", "plate-361fb404563eead5", "baking tray-b3ad1cbdafc0c517", "package of aluminum foil-aacf65d8c081894a", "foil-232806f05ae312b8", "first single fish cake-eef19b89390c8d4f", "knife-1ea848f8ac021aab", "packaging-0c652fa7861c6d4e", "fish cake2-e59c823a262bf319", "fish cake3-4e7a66437dab735d", "water bottle-353075a5dd1cb9fd", "fish cake packaging-19034de1b82cb506", "bag of vegetables-ada8975f7d9b7828", "another bag of vegetables-c2b80aabc9b7c5d2", "chopping board-5038a9a5137f3473", "air fryer-8eb47c13480c4726", "clear bowl-e5d6058cbfc51868", "knife holder-60a00c7b18c37453", "kitchen scale-744ffc4507f71aad", "bowl-cbf69d89dc48a148", "paper towel-2a88dfe85b274d1f", "bell pepper-38a8fbd533f963ff", "onion-d662a09583449534", "tomato bowl-ccb9d5303f95dbfc", "chilli-cf9f47c8a2cacab7", "bag of couscous-20826eedd286affe", "kettle-fc60b79e3a1ade3f", "pot-5de34c4f5af16197", "baking tray2-b9066a55cb65e9f6", "measuring spoons-99f7c2e0b06b248e", "couscous bag-53243da314f150d0", "spoon-fe400a2753b8d3df", "pan-39323c0daccefa4f", "garlic-c324b0433066a029", "oven mitt-44c62254b0bb306d", "spatula2-cc1f179a93614be6", "package of garlic-7c818e1faadc4798", "bottle of olive oil-4f15f428e6c84f51", "spoon2-a73cfe81343f16a5", "top of pot-06f8cfdf733b2b2a", "colander-ebb5cac0adcc4521", "small pan-b0cea205b713ca5f", "pot2-5ea1ae8cfc84ec9f", "pot top2-1fa05adb6453c57e", "small coffee pot-2300905f719a9571", "knife2-b0e12860e818c31e", "spatula-de7371adc061725d", "spoon3-871a33554ef3cd62", "peeler-7739174f9e0f6874", "tongs-f39cde91d6e92b9a", "spoon4-976725ae159c5c1d", "small plate-724aef7ad6f5f2ec", "bottle-61367938b822e3fd", "top of tupperware-0f3ffebcd8a1ecfc", "small bowl-c0ac0b5a68fdf5bb", "air fryer tray-f9f88e22a176ca58", "salt grinder-ed7009b5c28257d2", "pepper grinder-b1476962739ce5bf", "bottle of sesame oil-07137ca60d286e0f", "sesame oil bottle cap-44ea37b8fb7230dd", "sponge-f95a76572317480d", "bottle of dish soap-c24e2894bc584300", "air fryer vents-41c76fe48d17f1b0", "mug-d7a005ac3e09718b", "tea towel-4ed28f5e7d510235", "small glass tupperware-26cbef907d4e5064", "paper towel2-a3db5be1a45512d2", "waste-844bdf381b55ede4", "glass-58b2885c8bbc8c7b", "bowl2-693355edb2ab5b55", "plate2-be91db8ab97e6471", "plastic container-95b0e25bed999af9", "pasta spoon-34f4b8a5b86e05f8", "plate3-fc2153c189f37dfe", "dish washer bottle-1451b29537682bd5", "sponge2-4220d04e5ee109f0", "dishrag-bbe2df7b38660613", "fork-c8c2853a78129edb", "fork2-ae8c8e02238f62e6", "knife3-e101e654d4a57fa7", "knife4-e8308928a3c6425f" ], "reasoning": "1. I noticed the user take the pot lid (top of pot-06f8cfdf733b2b2a) and put it in the sink at 13:33.\n2. I then counted the number of times that the user shook the pot (pot-5de34c4f5af16197) with couscous to the right immediately thereafter: 3 times at 13:36.\n3. Next, I rewound the video to watch for the first time that the oven clock comes into focus. I noted that this happens at 00:25 shortly after the user turns on the oven, thereby setting the clock at 12:00.\n4. I then waited for the second time that the oven clock came into focus and noted that this happens at 02:55, where the oven clock is visible, albeit not very well focused.\n5. Lastly, I waited for the third time that the oven clock came into focus and noted that this happens at 03:04 when the user puts the fish cakes (first single fish cake-eef19b89390c8d4f, fish cake2-e59c823a262bf319, fish cake3-4e7a66437dab735d) into the oven, at which point the oven clock reads 12:02.\n6. From here I calculated the number of minutes that passed from the first and third time the oven clock came into focus: 12:02 - 12:00 = 2 minutes.\n6. Lastly, I calculated the product of shakes to the right (3) and minutes (2): 3*2=6.", "referenced_object_masks": [ "top of pot-06f8cfdf733b2b2a", "pot-5de34c4f5af16197", "first single fish cake-eef19b89390c8d4f", "fish cake2-e59c823a262bf319", "fish cake3-4e7a66437dab735d" ] }, { "key": "P08-20240617-184909:b5601908d2d98e35230485a90f268bb106332ae6", "video_id": "P08-20240617-184909", "question": "If I add the weight of the broccoli and the weight of the red onion and then take the square root of this number, what is it?", "answer_choice_0": "17.9", "answer_choice_1": "17.7", "answer_choice_2": "18", "answer_choice_3": "18.5", "answer_choice_4": "18.3", "answer_id": 2, "answer": "18", "question_type": [ "Action Recognition", "Event Occurence", "Numerical Reasoning", "Object Recognition", "Audio Reasoning" ], "objects_of_interest": [ "phone-42c87742240840b5", "plate-361fb404563eead5", "baking tray-b3ad1cbdafc0c517", "package of aluminum foil-aacf65d8c081894a", "foil-232806f05ae312b8", "first single fish cake-eef19b89390c8d4f", "knife-1ea848f8ac021aab", "packaging-0c652fa7861c6d4e", "fish cake2-e59c823a262bf319", "fish cake3-4e7a66437dab735d", "water bottle-353075a5dd1cb9fd", "fish cake packaging-19034de1b82cb506", "bag of vegetables-ada8975f7d9b7828", "another bag of vegetables-c2b80aabc9b7c5d2", "chopping board-5038a9a5137f3473", "air fryer-8eb47c13480c4726", "clear bowl-e5d6058cbfc51868", "knife holder-60a00c7b18c37453", "kitchen scale-744ffc4507f71aad", "bowl-cbf69d89dc48a148", "paper towel-2a88dfe85b274d1f", "bell pepper-38a8fbd533f963ff", "onion-d662a09583449534", "tomato bowl-ccb9d5303f95dbfc", "chilli-cf9f47c8a2cacab7", "bag of couscous-20826eedd286affe", "kettle-fc60b79e3a1ade3f", "pot-5de34c4f5af16197", "baking tray2-b9066a55cb65e9f6", "measuring spoons-99f7c2e0b06b248e", "couscous bag-53243da314f150d0", "spoon-fe400a2753b8d3df", "pan-39323c0daccefa4f", "garlic-c324b0433066a029", "oven mitt-44c62254b0bb306d", "spatula2-cc1f179a93614be6", "package of garlic-7c818e1faadc4798", "bottle of olive oil-4f15f428e6c84f51", "spoon2-a73cfe81343f16a5", "top of pot-06f8cfdf733b2b2a", "colander-ebb5cac0adcc4521", "small pan-b0cea205b713ca5f", "pot2-5ea1ae8cfc84ec9f", "pot top2-1fa05adb6453c57e", "small coffee pot-2300905f719a9571", "knife2-b0e12860e818c31e", "spatula-de7371adc061725d", "spoon3-871a33554ef3cd62", "peeler-7739174f9e0f6874", "tongs-f39cde91d6e92b9a", "spoon4-976725ae159c5c1d", "small plate-724aef7ad6f5f2ec", "bottle-61367938b822e3fd", "top of tupperware-0f3ffebcd8a1ecfc", "small bowl-c0ac0b5a68fdf5bb", "air fryer tray-f9f88e22a176ca58", "salt grinder-ed7009b5c28257d2", "pepper grinder-b1476962739ce5bf", "bottle of sesame oil-07137ca60d286e0f", "sesame oil bottle cap-44ea37b8fb7230dd", "sponge-f95a76572317480d", "bottle of dish soap-c24e2894bc584300", "air fryer vents-41c76fe48d17f1b0", "mug-d7a005ac3e09718b", "tea towel-4ed28f5e7d510235", "small glass tupperware-26cbef907d4e5064", "paper towel2-a3db5be1a45512d2", "waste-844bdf381b55ede4", "glass-58b2885c8bbc8c7b", "bowl2-693355edb2ab5b55", "plate2-be91db8ab97e6471", "plastic container-95b0e25bed999af9", "pasta spoon-34f4b8a5b86e05f8", "plate3-fc2153c189f37dfe", "dish washer bottle-1451b29537682bd5", "sponge2-4220d04e5ee109f0", "dishrag-bbe2df7b38660613", "fork-c8c2853a78129edb", "fork2-ae8c8e02238f62e6", "knife3-e101e654d4a57fa7", "knife4-e8308928a3c6425f" ], "reasoning": "1. I watched the whole video and saw the user weigh the broccoli on the kitchen scale kitchen scale-744ffc4507f71aad) at 05:40.\n2. The broccoli weighed 273 g.\n3. At 07:51, the user weighed the red onion 'onion-d662a09583449534' on the kitchen scale (kitchen scale-744ffc4507f71aad).\n4. The red onion 'onion-d662a09583449534' weighed 51 g.\n5. Adding together the weight of the broccoli and the weight of the red onion ('onion-d662a09583449534', they get 324.\n6. 324 is a perfect square whose square root is 18.", "referenced_object_masks": [ "kitchen scale-744ffc4507f71aad" ] }, { "key": "P08-20240617-184909:c894c9e5124a3ae26b1946f54a086572b4471d6e", "video_id": "P08-20240617-184909", "question": "Where is the bottle of sesame oil located in relation to me as I bent down the second time to pick up something from the floor?", "answer_choice_0": "In front of me and to my right", "answer_choice_1": "Behind me and to my right", "answer_choice_2": "Behind me and to my left", "answer_choice_3": "Above me", "answer_choice_4": "In front of me and to my left", "answer_id": 2, "answer": "Behind me and to my left", "question_type": [ "Action Recognition", "Counting", "Egocentric & Spatial Perception", "Situational Awareness" ], "objects_of_interest": [ "knife4-e8308928a3c6425f", "knife3-e101e654d4a57fa7", "fork2-ae8c8e02238f62e6", "fork-c8c2853a78129edb", "dishrag-bbe2df7b38660613", "sponge2-4220d04e5ee109f0", "dish washer bottle-1451b29537682bd5", "plate3-fc2153c189f37dfe", "pasta spoon-34f4b8a5b86e05f8", "plastic container-95b0e25bed999af9", "plate2-be91db8ab97e6471", "bowl2-693355edb2ab5b55", "glass-58b2885c8bbc8c7b", "waste-844bdf381b55ede4", "paper towel2-a3db5be1a45512d2", "small glass tupperware-26cbef907d4e5064", "tea towel-4ed28f5e7d510235", "mug-d7a005ac3e09718b", "air fryer vents-41c76fe48d17f1b0", "bottle of dish soap-c24e2894bc584300", "sponge-f95a76572317480d", "sesame oil bottle cap-44ea37b8fb7230dd", "bottle of sesame oil-07137ca60d286e0f", "pepper grinder-b1476962739ce5bf", "salt grinder-ed7009b5c28257d2", "air fryer tray-f9f88e22a176ca58", "small bowl-c0ac0b5a68fdf5bb", "top of tupperware-0f3ffebcd8a1ecfc", "bottle-61367938b822e3fd", "small plate-724aef7ad6f5f2ec", "spoon4-976725ae159c5c1d", "tongs-f39cde91d6e92b9a", "peeler-7739174f9e0f6874", "spoon3-871a33554ef3cd62", "spatula2-cc1f179a93614be6", "knife2-b0e12860e818c31e", "small coffee pot-2300905f719a9571", "pot top2-1fa05adb6453c57e", "pot2-5ea1ae8cfc84ec9f", "small pan-b0cea205b713ca5f", "colander-ebb5cac0adcc4521", "spoon2-a73cfe81343f16a5", "bottle of olive oil-4f15f428e6c84f51", "spatula-de7371adc061725d", "oven mitt-44c62254b0bb306d", "garlic-c324b0433066a029", "pan-39323c0daccefa4f", "top of pot-06f8cfdf733b2b2a", "spoon-fe400a2753b8d3df", "couscous bag-53243da314f150d0", "measuring spoons-99f7c2e0b06b248e", "baking tray2-b9066a55cb65e9f6", "pot-5de34c4f5af16197", "kettle-fc60b79e3a1ade3f", "bag of couscous-20826eedd286affe", "chilli-cf9f47c8a2cacab7", "tomato bowl-ccb9d5303f95dbfc", "onion-d662a09583449534", "bell pepper-38a8fbd533f963ff", "paper towel-2a88dfe85b274d1f", "bowl-cbf69d89dc48a148", "kitchen scale-744ffc4507f71aad", "knife holder-60a00c7b18c37453", "clear bowl-e5d6058cbfc51868", "air fryer-8eb47c13480c4726", "chopping board-5038a9a5137f3473", "another bag of vegetables-c2b80aabc9b7c5d2", "bag of vegetables-ada8975f7d9b7828", "fish cake packaging-19034de1b82cb506", "water bottle-353075a5dd1cb9fd", "fish cake3-4e7a66437dab735d", "fish cake2-e59c823a262bf319", "packaging-0c652fa7861c6d4e", "knife-1ea848f8ac021aab", "first single fish cake-eef19b89390c8d4f", "foil-232806f05ae312b8", "package of aluminum foil-aacf65d8c081894a", "baking tray-b3ad1cbdafc0c517", "plate-361fb404563eead5", "phone-42c87742240840b5", "package of garlic-7c818e1faadc4798" ], "reasoning": "1. I watched the video and looked for the times when the user bent down to pick up something from the floor. I noticed this happening for the first time at 20:06.\n2. I continued watching and noticed the 2nd occurrence happened very soon after, at 20:08. They picked this up after turning to the right and walking forward a couple of steps toward a trash can on the right side of the kitchen.\n3. Knowing 20:08 was the 2nd occurrence, I then looked back to see where the bottle of sesame oil (bottle of sesame oil-07137ca60d286e0f) was located as this happened. At 20:06, I noticed the bottle of sesame oil (bottle of sesame oil-07137ca60d286e0f) was on the countertop directly in front of the user when they bent down to pick something up.\n4. Since the user then turned to the right and walked forward, and then bent down the 2nd time, I can surmise that the bottle of sesame oil (bottle of sesame oil-07137ca60d286e0f) was behind the user and to their left at the time.", "referenced_object_masks": [ "bottle of sesame oil-07137ca60d286e0f" ] }, { "key": "P08-20240617-184909:cb3539be147c4cf515a1328aee4de0758056093f", "video_id": "P08-20240617-184909", "question": "When I pull out the air fryer drawer at 09:48 to check the broccoli and green beans, in what relative direction is the sink from the air fryer?", "answer_choice_0": "3 o'clock", "answer_choice_1": "5 o'clock", "answer_choice_2": "9 o'clock", "answer_choice_3": "1 o'clock", "answer_choice_4": "6 o'clock", "answer_id": 2, "answer": "9 o'clock", "question_type": [ "Egocentric & Spatial Perception", "Action Recognition", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "phone-42c87742240840b5", "plate-361fb404563eead5", "baking tray-b3ad1cbdafc0c517", "package of aluminum foil-aacf65d8c081894a", "foil-232806f05ae312b8", "first single fish cake-eef19b89390c8d4f", "knife-1ea848f8ac021aab", "packaging-0c652fa7861c6d4e", "fish cake2-e59c823a262bf319", "fish cake3-4e7a66437dab735d", "water bottle-353075a5dd1cb9fd", "fish cake packaging-19034de1b82cb506", "bag of vegetables-ada8975f7d9b7828", "another bag of vegetables-c2b80aabc9b7c5d2", "chopping board-5038a9a5137f3473", "air fryer-8eb47c13480c4726", "clear bowl-e5d6058cbfc51868", "knife holder-60a00c7b18c37453", "kitchen scale-744ffc4507f71aad", "bowl-cbf69d89dc48a148", "paper towel-2a88dfe85b274d1f", "bell pepper-38a8fbd533f963ff", "onion-d662a09583449534", "tomato bowl-ccb9d5303f95dbfc", "chilli-cf9f47c8a2cacab7", "bag of couscous-20826eedd286affe", "kettle-fc60b79e3a1ade3f", "pot-5de34c4f5af16197", "baking tray2-b9066a55cb65e9f6", "measuring spoons-99f7c2e0b06b248e", "couscous bag-53243da314f150d0", "spoon-fe400a2753b8d3df", "pan-39323c0daccefa4f", "garlic-c324b0433066a029", "oven mitt-44c62254b0bb306d", "spatula2-cc1f179a93614be6", "package of garlic-7c818e1faadc4798", "bottle of olive oil-4f15f428e6c84f51", "spoon2-a73cfe81343f16a5", "top of pot-06f8cfdf733b2b2a", "colander-ebb5cac0adcc4521", "small pan-b0cea205b713ca5f", "pot2-5ea1ae8cfc84ec9f", "pot top2-1fa05adb6453c57e", "small coffee pot-2300905f719a9571", "knife2-b0e12860e818c31e", "spatula-de7371adc061725d", "spoon3-871a33554ef3cd62", "peeler-7739174f9e0f6874", "tongs-f39cde91d6e92b9a", "spoon4-976725ae159c5c1d", "small plate-724aef7ad6f5f2ec", "bottle-61367938b822e3fd", "top of tupperware-0f3ffebcd8a1ecfc", "small bowl-c0ac0b5a68fdf5bb", "air fryer tray-f9f88e22a176ca58", "salt grinder-ed7009b5c28257d2", "pepper grinder-b1476962739ce5bf", "bottle of sesame oil-07137ca60d286e0f", "sesame oil bottle cap-44ea37b8fb7230dd", "sponge-f95a76572317480d", "bottle of dish soap-c24e2894bc584300", "air fryer vents-41c76fe48d17f1b0", "mug-d7a005ac3e09718b", "tea towel-4ed28f5e7d510235", "small glass tupperware-26cbef907d4e5064", "paper towel2-a3db5be1a45512d2", "waste-844bdf381b55ede4", "glass-58b2885c8bbc8c7b", "bowl2-693355edb2ab5b55", "plate2-be91db8ab97e6471", "plastic container-95b0e25bed999af9", "pasta spoon-34f4b8a5b86e05f8", "plate3-fc2153c189f37dfe", "dish washer bottle-1451b29537682bd5", "sponge2-4220d04e5ee109f0", "dishrag-bbe2df7b38660613", "fork-c8c2853a78129edb", "fork2-ae8c8e02238f62e6", "knife3-e101e654d4a57fa7", "knife4-e8308928a3c6425f" ], "reasoning": "1. I watched the whole video and saw the user pull out the air fryer drawer (air fryer tray-f9f88e22a176ca58) at 09:48 to check the broccoli and green beans .\n2. I can't see the sink at this angle, so to get a better sense of my bearings I turn to another timecode that offers a better angle.\n3. I find such an angle at 03:57, where I can see the air fryer (air fryer-8eb47c13480c4726) in the distance, directly across from the sink -- to the left of the sink if I were facing the air fryer (air fryer-8eb47c13480c4726).\n4. Given this information, I can conclude that the relative direction of the air fryer (air fryer-8eb47c13480c4726) to the sink would be 9 o'clock at 09:48.", "referenced_object_masks": [ "air fryer-8eb47c13480c4726", "air fryer tray-f9f88e22a176ca58" ] }, { "key": "P08-20240617-184909:d91c6bc075820ee2ccb2411739ff9ca9043557d2", "video_id": "P08-20240617-184909", "question": "If I multiply the number of items I wash at the sink from 20:55 to 21:54 by the number of garlic cloves I use in this dish and then raise it to the 4th power, what number do I get?", "answer_choice_0": "100", "answer_choice_1": "10,000", "answer_choice_2": "256", "answer_choice_3": "38, 416", "answer_choice_4": "4,096", "answer_id": 1, "answer": "10,000", "question_type": [ "Action Recognition", "Counting", "Event Occurence", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "phone-42c87742240840b5", "plate-361fb404563eead5", "baking tray-b3ad1cbdafc0c517", "package of aluminum foil-aacf65d8c081894a", "foil-232806f05ae312b8", "first single fish cake-eef19b89390c8d4f", "knife-1ea848f8ac021aab", "packaging-0c652fa7861c6d4e", "fish cake2-e59c823a262bf319", "fish cake3-4e7a66437dab735d", "water bottle-353075a5dd1cb9fd", "fish cake packaging-19034de1b82cb506", "bag of vegetables-ada8975f7d9b7828", "another bag of vegetables-c2b80aabc9b7c5d2", "chopping board-5038a9a5137f3473", "air fryer-8eb47c13480c4726", "clear bowl-e5d6058cbfc51868", "knife holder-60a00c7b18c37453", "kitchen scale-744ffc4507f71aad", "bowl-cbf69d89dc48a148", "paper towel-2a88dfe85b274d1f", "bell pepper-38a8fbd533f963ff", "onion-d662a09583449534", "tomato bowl-ccb9d5303f95dbfc", "chilli-cf9f47c8a2cacab7", "bag of couscous-20826eedd286affe", "kettle-fc60b79e3a1ade3f", "pot-5de34c4f5af16197", "baking tray2-b9066a55cb65e9f6", "measuring spoons-99f7c2e0b06b248e", "couscous bag-53243da314f150d0", "spoon-fe400a2753b8d3df", "pan-39323c0daccefa4f", "garlic-c324b0433066a029", "oven mitt-44c62254b0bb306d", "spatula2-cc1f179a93614be6", "package of garlic-7c818e1faadc4798", "bottle of olive oil-4f15f428e6c84f51", "spoon2-a73cfe81343f16a5", "top of pot-06f8cfdf733b2b2a", "colander-ebb5cac0adcc4521", "small pan-b0cea205b713ca5f", "pot2-5ea1ae8cfc84ec9f", "pot top2-1fa05adb6453c57e", "small coffee pot-2300905f719a9571", "knife2-b0e12860e818c31e", "spatula-de7371adc061725d", "spoon3-871a33554ef3cd62", "peeler-7739174f9e0f6874", "tongs-f39cde91d6e92b9a", "spoon4-976725ae159c5c1d", "small plate-724aef7ad6f5f2ec", "bottle-61367938b822e3fd", "top of tupperware-0f3ffebcd8a1ecfc", "small bowl-c0ac0b5a68fdf5bb", "air fryer tray-f9f88e22a176ca58", "salt grinder-ed7009b5c28257d2", "pepper grinder-b1476962739ce5bf", "bottle of sesame oil-07137ca60d286e0f", "sesame oil bottle cap-44ea37b8fb7230dd", "sponge-f95a76572317480d", "bottle of dish soap-c24e2894bc584300", "air fryer vents-41c76fe48d17f1b0", "mug-d7a005ac3e09718b", "tea towel-4ed28f5e7d510235", "small glass tupperware-26cbef907d4e5064", "paper towel2-a3db5be1a45512d2", "waste-844bdf381b55ede4", "glass-58b2885c8bbc8c7b", "bowl2-693355edb2ab5b55", "plate2-be91db8ab97e6471", "plastic container-95b0e25bed999af9", "pasta spoon-34f4b8a5b86e05f8", "plate3-fc2153c189f37dfe", "dish washer bottle-1451b29537682bd5", "sponge2-4220d04e5ee109f0", "dishrag-bbe2df7b38660613", "fork-c8c2853a78129edb", "fork2-ae8c8e02238f62e6", "knife3-e101e654d4a57fa7", "knife4-e8308928a3c6425f" ], "reasoning": "1. I watched the whole video and saw the user crush and cut up 2 cloves of garlic (garlic-c324b0433066a029) starting at 11:41.\n2. Starting at 20:55 and ending at 21:54, the user washed 5 items at the kitchen sink .\n3. Multiply 2 and 5 to get 10.\n4. Raise 10 to the 4th power to get 10,000.", "referenced_object_masks": [ "garlic-c324b0433066a029" ] }, { "key": "P08-20240617-184909:f388d06f1d6763116c7f92cdf82122bde6651025", "video_id": "P08-20240617-184909", "question": "How many broccoli pieces did I pull out with the tongs before I stir the vegetables in the pan?", "answer_choice_0": "6", "answer_choice_1": "8", "answer_choice_2": "7", "answer_choice_3": "5", "answer_choice_4": "9", "answer_id": 4, "answer": "9", "question_type": [ "Action Recognition", "Cause and Effect", "Counting", "Event Occurence" ], "objects_of_interest": [ "phone-42c87742240840b5", "plate-361fb404563eead5", "baking tray-b3ad1cbdafc0c517", "package of aluminum foil-aacf65d8c081894a", "foil-232806f05ae312b8", "first single fish cake-eef19b89390c8d4f", "knife-1ea848f8ac021aab", "packaging-0c652fa7861c6d4e", "fish cake2-e59c823a262bf319", "fish cake3-4e7a66437dab735d", "water bottle-353075a5dd1cb9fd", "fish cake packaging-19034de1b82cb506", "bag of vegetables-ada8975f7d9b7828", "another bag of vegetables-c2b80aabc9b7c5d2", "chopping board-5038a9a5137f3473", "air fryer-8eb47c13480c4726", "clear bowl-e5d6058cbfc51868", "knife holder-60a00c7b18c37453", "kitchen scale-744ffc4507f71aad", "bowl-cbf69d89dc48a148", "paper towel-2a88dfe85b274d1f", "bell pepper-38a8fbd533f963ff", "onion-d662a09583449534", "tomato bowl-ccb9d5303f95dbfc", "chilli-cf9f47c8a2cacab7", "bag of couscous-20826eedd286affe", "kettle-fc60b79e3a1ade3f", "pot-5de34c4f5af16197", "baking tray2-b9066a55cb65e9f6", "measuring spoons-99f7c2e0b06b248e", "couscous bag-53243da314f150d0", "spoon-fe400a2753b8d3df", "pan-39323c0daccefa4f", "garlic-c324b0433066a029", "oven mitt-44c62254b0bb306d", "spatula2-cc1f179a93614be6", "package of garlic-7c818e1faadc4798", "bottle of olive oil-4f15f428e6c84f51", "spoon2-a73cfe81343f16a5", "top of pot-06f8cfdf733b2b2a", "colander-ebb5cac0adcc4521", "small pan-b0cea205b713ca5f", "pot2-5ea1ae8cfc84ec9f", "pot top2-1fa05adb6453c57e", "small coffee pot-2300905f719a9571", "knife2-b0e12860e818c31e", "spatula-de7371adc061725d", "spoon3-871a33554ef3cd62", "peeler-7739174f9e0f6874", "tongs-f39cde91d6e92b9a", "spoon4-976725ae159c5c1d", "small plate-724aef7ad6f5f2ec", "bottle-61367938b822e3fd", "top of tupperware-0f3ffebcd8a1ecfc", "small bowl-c0ac0b5a68fdf5bb", "air fryer tray-f9f88e22a176ca58", "salt grinder-ed7009b5c28257d2", "pepper grinder-b1476962739ce5bf", "bottle of sesame oil-07137ca60d286e0f", "sesame oil bottle cap-44ea37b8fb7230dd", "sponge-f95a76572317480d", "bottle of dish soap-c24e2894bc584300", "air fryer vents-41c76fe48d17f1b0", "mug-d7a005ac3e09718b", "tea towel-4ed28f5e7d510235", "small glass tupperware-26cbef907d4e5064", "paper towel2-a3db5be1a45512d2", "waste-844bdf381b55ede4", "glass-58b2885c8bbc8c7b", "bowl2-693355edb2ab5b55", "plate2-be91db8ab97e6471", "plastic container-95b0e25bed999af9", "pasta spoon-34f4b8a5b86e05f8", "plate3-fc2153c189f37dfe", "dish washer bottle-1451b29537682bd5", "sponge2-4220d04e5ee109f0", "dishrag-bbe2df7b38660613", "fork-c8c2853a78129edb", "fork2-ae8c8e02238f62e6", "knife3-e101e654d4a57fa7", "knife4-e8308928a3c6425f" ], "reasoning": "1. I noticed the user pull out the first broccoli at 15:54. \n2. I noticed the user pull out the 9th broccoli at 16:36.\n3. I noticed the user begin to stir the vegetables at 16:46.", "referenced_object_masks": [] }, { "key": "P08-20240618-163547:1d65278888df056308a7054452cdbadd887105bb", "video_id": "P08-20240618-163547", "question": "Where is the yellow pepper and the lemon half in relation to my hand when I first reach into the fridge?", "answer_choice_0": "Below it", "answer_choice_1": "On the right of it", "answer_choice_2": "On each side of it", "answer_choice_3": "On the left of it", "answer_choice_4": "Above it", "answer_id": 0, "answer": "Below it", "question_type": [ "Object Recognition", "Egocentric & Spatial Perception", "Action Recognition" ], "objects_of_interest": [ "phone-9cdc08bf04556c76", "kettle-c93846da3e16a309", "kitchen utensils-e3cec8dbbda54a1f", "weighing scale-fc130cb7b7c69614", "food-ea78597e1899c9f1", "mushroom-bc984594bb46967e", "carrots-e01bc0303c758441", "bowl-e183f8ba79c23e01", "tupperware of food-bb982233cfea4a18", "kiwi-c000cd056d72295a", "lemon-c52334f9f5cf9be1", "tupperware of food2-a0c838aa23ff496e", "tupperware of food3-4e1390a2b998a25a", "tupperware of food4-be577585fb42aed8", "celery stalks-400b53896c7b2bca", "silver bowl-da8bfe9936f0a849", "pot-71499c16630fa4ba", "second pot-c88a3c9a0951c3df", "lid-d6417fa8d1d85cf5", "colander-65445ce02b5dce74", "lid2-2ddfeb942b09aeec", "small pan-47845b2263b0ae02", "wok-ad2a5eb074a12b67", "cutting board-c22038cb46c6c41c", "plate-a916e0de60db34b1", "spoon holder-d2b4b0e44bac78ca", "measuring cups-ead1849b7fac0a82", "cheese grater-c04a000e7f8d5394", "tupperware top-29d7f2dec98a7396", "knife1-0aac862e38a23a73", "knife2-86f319cb6fa801d8", "can of tomatoes-1a0cd303b655bff6", "jar of tomato sauce-b034c657b5c4c4b4", "cloth-67b8b927f44f4657", "red onion-6ef9fc3979e886e2", "onion2-db5b9f413356eeb2", "bowl2-40c9c67cbffe562e", "tea towel-e529fd9cb8a3920a", "one half of onion-c0215b1c960d17c9", "second onion half-a2fcfcb8230cd301", "bag of garlic-112f6a72a370ef3b", "scraps of garlic cloves-f6f191b4f5c3273e", "smaller blue plate-f74e3a3a17a146f9", "cherry tomatoes-6e430d964f21b2c7", "cherry tomato1-fe6dbc1fd720e7c5", "cherry tomato2-e1db0d76d96e1ea4", "cherry tomato3-d1312e9d288610a6", "cherry tomato4-f683cf85eea55557", "cherry tomato5-4e9576d3b34750dd", "single stalk-b4caf8eb772abd7b", "piece of paper towel-4cf067c5e3ecf79d", "carrot1-e12ac7517bda2270", "peeler-d187fa429d20f414", "green pepper-34d649f28dafa3c0", "pieces of pepper-a5d0fe445ac665af", "scraps of pepper head-a636309488bf3fe7", "mug-57d3f85d89dde3dc", "piece of paper towel2-a9efbfe379c1eda6", "pieces of paper towel3-f88d27d5173cc696", "single mushroom-fb441ba818471372", "plastic casing of mushrooms-d5038e1bd595c139", "bottle of olive oil-7dd447c74efe02b6", "spoon-8223d69e1cc1faa5", "spatula-b7f47c3883af9098", "chilli pepper-282456d213843ff2", "paper container of mushrooms-a38a017a4bc3f46a", "glass-6e0e61cd490497f8", "sponge-229f3a7e23312f21", "dish soap-8c8973299d8802b9", "glass bowl-b90104c5a059e6db" ], "reasoning": "1. I determine when the user first reaches into the fridge. This occurs at 00:18 when the user grabs a package of ground meat from the second shelf on the right side of the fridge. \n2. Next, I locate the yellow pepper situated in the container (tupperware of food4-be577585fb42aed8) on the bottom shelf on the left side of the fridge and the lemon half (lemon-c52334f9f5cf9be1) situated on the bottom shelf on the right side of the fridge. \n3. Since the user is reaching up to retrieve the package of ground meat from the second shelf, and the yellow pepper and the lemon half (lemon-c52334f9f5cf9be1) are on the bottom shelf, those 2 items are situated below the user's hand.", "referenced_object_masks": [ "tupperware of food4-be577585fb42aed8", "lemon-c52334f9f5cf9be1" ] }, { "key": "P08-20240618-163547:54e65da688ab0781a217543b7f62b8b682e52c92", "video_id": "P08-20240618-163547", "question": "If I took the number of times I tapped the screen, and put it to the power of the number of times I successfully swiped and moved my phone screen from 16:05 to 16:34, what would the answer be?", "answer_choice_0": "84", "answer_choice_1": "42034", "answer_choice_2": "21952", "answer_choice_3": "69", "answer_choice_4": "31", "answer_id": 2, "answer": "21952", "question_type": [ "Action Recognition", "Counting", "Numerical Reasoning", "Situational Awareness", "Temporal Reasoning", "Object Recognition" ], "objects_of_interest": [ "phone-9cdc08bf04556c76", "kettle-c93846da3e16a309", "kitchen utensils-e3cec8dbbda54a1f", "weighing scale-fc130cb7b7c69614", "food-ea78597e1899c9f1", "mushroom-bc984594bb46967e", "carrots-e01bc0303c758441", "bowl-e183f8ba79c23e01", "tupperware of food-bb982233cfea4a18", "kiwi-c000cd056d72295a", "lemon-c52334f9f5cf9be1", "tupperware of food2-a0c838aa23ff496e", "tupperware of food3-4e1390a2b998a25a", "tupperware of food4-be577585fb42aed8", "celery stalks-400b53896c7b2bca", "silver bowl-da8bfe9936f0a849", "pot-71499c16630fa4ba", "second pot-c88a3c9a0951c3df", "lid-d6417fa8d1d85cf5", "colander-65445ce02b5dce74", "lid2-2ddfeb942b09aeec", "small pan-47845b2263b0ae02", "wok-ad2a5eb074a12b67", "cutting board-c22038cb46c6c41c", "plate-a916e0de60db34b1", "spoon holder-d2b4b0e44bac78ca", "measuring cups-ead1849b7fac0a82", "cheese grater-c04a000e7f8d5394", "tupperware top-29d7f2dec98a7396", "knife1-0aac862e38a23a73", "knife2-86f319cb6fa801d8", "can of tomatoes-1a0cd303b655bff6", "jar of tomato sauce-b034c657b5c4c4b4", "cloth-67b8b927f44f4657", "red onion-6ef9fc3979e886e2", "onion2-db5b9f413356eeb2", "bowl2-40c9c67cbffe562e", "tea towel-e529fd9cb8a3920a", "one half of onion-c0215b1c960d17c9", "second onion half-a2fcfcb8230cd301", "bag of garlic-112f6a72a370ef3b", "scraps of garlic cloves-f6f191b4f5c3273e", "smaller blue plate-f74e3a3a17a146f9", "cherry tomatoes-6e430d964f21b2c7", "cherry tomato1-fe6dbc1fd720e7c5", "cherry tomato2-e1db0d76d96e1ea4", "cherry tomato3-d1312e9d288610a6", "cherry tomato4-f683cf85eea55557", "cherry tomato5-4e9576d3b34750dd", "single stalk-b4caf8eb772abd7b", "piece of paper towel-4cf067c5e3ecf79d", "carrot1-e12ac7517bda2270", "peeler-d187fa429d20f414", "green pepper-34d649f28dafa3c0", "pieces of pepper-a5d0fe445ac665af", "scraps of pepper head-a636309488bf3fe7", "mug-57d3f85d89dde3dc", "piece of paper towel2-a9efbfe379c1eda6", "pieces of paper towel3-f88d27d5173cc696", "single mushroom-fb441ba818471372", "plastic casing of mushrooms-d5038e1bd595c139", "bottle of olive oil-7dd447c74efe02b6", "spoon-8223d69e1cc1faa5", "spatula-b7f47c3883af9098", "chilli pepper-282456d213843ff2", "paper container of mushrooms-a38a017a4bc3f46a", "glass-6e0e61cd490497f8", "sponge-229f3a7e23312f21", "dish soap-8c8973299d8802b9", "glass bowl-b90104c5a059e6db" ], "reasoning": "1. I identify the phone (phone-9cdc08bf04556c76) at 16:05.\n2. I then counted the number of times the user tapped the phone (phone-9cdc08bf04556c76) screen from 16:05 to 16:34, and it resulted in 28.\n3. I then went back to count the number of times the user swiped the phone (phone-9cdc08bf04556c76) screen, successfully moving the screen within the same timestamp, and found that it resulted in 3.\n4. 28 to the power of 3 is 21952.", "referenced_object_masks": [ "phone-9cdc08bf04556c76" ] }, { "key": "P08-20240618-163547:66de03bd9000512618dc0079da152ccdec2f9a6b", "video_id": "P08-20240618-163547", "question": "From which cardinal directions do I begin each swipe on my phone between 00:08 to 00:10?", "answer_choice_0": "South north south", "answer_choice_1": "West west west", "answer_choice_2": "North north north", "answer_choice_3": "North north south", "answer_choice_4": "East south west", "answer_id": 0, "answer": "South north south", "question_type": [ "Action Recognition", "Counting", "Object Recognition", "Situational Awareness", "Event Occurence", "Temporal Reasoning" ], "objects_of_interest": [ "phone-9cdc08bf04556c76", "kettle-c93846da3e16a309", "kitchen utensils-e3cec8dbbda54a1f", "weighing scale-fc130cb7b7c69614", "food-ea78597e1899c9f1", "mushroom-bc984594bb46967e", "carrots-e01bc0303c758441", "bowl-e183f8ba79c23e01", "tupperware of food-bb982233cfea4a18", "kiwi-c000cd056d72295a", "lemon-c52334f9f5cf9be1", "tupperware of food2-a0c838aa23ff496e", "tupperware of food3-4e1390a2b998a25a", "tupperware of food4-be577585fb42aed8", "celery stalks-400b53896c7b2bca", "silver bowl-da8bfe9936f0a849", "pot-71499c16630fa4ba", "second pot-c88a3c9a0951c3df", "lid-d6417fa8d1d85cf5", "colander-65445ce02b5dce74", "lid2-2ddfeb942b09aeec", "small pan-47845b2263b0ae02", "wok-ad2a5eb074a12b67", "cutting board-c22038cb46c6c41c", "plate-a916e0de60db34b1", "spoon holder-d2b4b0e44bac78ca", "measuring cups-ead1849b7fac0a82", "cheese grater-c04a000e7f8d5394", "tupperware top-29d7f2dec98a7396", "knife1-0aac862e38a23a73", "knife2-86f319cb6fa801d8", "can of tomatoes-1a0cd303b655bff6", "jar of tomato sauce-b034c657b5c4c4b4", "cloth-67b8b927f44f4657", "red onion-6ef9fc3979e886e2", "onion2-db5b9f413356eeb2", "bowl2-40c9c67cbffe562e", "tea towel-e529fd9cb8a3920a", "one half of onion-c0215b1c960d17c9", "second onion half-a2fcfcb8230cd301", "bag of garlic-112f6a72a370ef3b", "scraps of garlic cloves-f6f191b4f5c3273e", "smaller blue plate-f74e3a3a17a146f9", "cherry tomatoes-6e430d964f21b2c7", "cherry tomato1-fe6dbc1fd720e7c5", "cherry tomato2-e1db0d76d96e1ea4", "cherry tomato3-d1312e9d288610a6", "cherry tomato4-f683cf85eea55557", "cherry tomato5-4e9576d3b34750dd", "single stalk-b4caf8eb772abd7b", "piece of paper towel-4cf067c5e3ecf79d", "carrot1-e12ac7517bda2270", "peeler-d187fa429d20f414", "green pepper-34d649f28dafa3c0", "pieces of pepper-a5d0fe445ac665af", "scraps of pepper head-a636309488bf3fe7", "mug-57d3f85d89dde3dc", "piece of paper towel2-a9efbfe379c1eda6", "pieces of paper towel3-f88d27d5173cc696", "single mushroom-fb441ba818471372", "plastic casing of mushrooms-d5038e1bd595c139", "bottle of olive oil-7dd447c74efe02b6", "spoon-8223d69e1cc1faa5", "spatula-b7f47c3883af9098", "chilli pepper-282456d213843ff2", "paper container of mushrooms-a38a017a4bc3f46a", "glass-6e0e61cd490497f8", "sponge-229f3a7e23312f21", "dish soap-8c8973299d8802b9", "glass bowl-b90104c5a059e6db" ], "reasoning": "1. I observe the user walk toward the counter near the tea kettle (kettle-c93846da3e16a309) while holding their phone (phone-9cdc08bf04556c76)) from 00:05 to 00:08\n2. From 00:08 to 00:10, I observe the user swipe the screen on their phone (phone-9cdc08bf04556c76) 3 times: starting from the bottom to swipe upwards, starting from the top to swipe downwards, and starting from the bottom to swipe upwards.\n3. I determine that the starting position for each swipe would correlate to the following cardinal directions: south, north, south.", "referenced_object_masks": [ "phone-9cdc08bf04556c76", "kettle-c93846da3e16a309" ] }, { "key": "P08-20240618-163547:78a0a994f67f85ba5cb923359266b313462f3159", "video_id": "P08-20240618-163547", "question": "What number would I get if I subtracted the number of glass cups to the left of the sink from the number of knives in the knife block from 06:36 to 06:54?", "answer_choice_0": "6", "answer_choice_1": "3", "answer_choice_2": "2", "answer_choice_3": "5", "answer_choice_4": "4", "answer_id": 3, "answer": "5", "question_type": [ "Object Recognition", "Temporal Reasoning", "Numerical Reasoning", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "phone-9cdc08bf04556c76", "kettle-c93846da3e16a309", "kitchen utensils-e3cec8dbbda54a1f", "weighing scale-fc130cb7b7c69614", "food-ea78597e1899c9f1", "mushroom-bc984594bb46967e", "carrots-e01bc0303c758441", "bowl-e183f8ba79c23e01", "tupperware of food-bb982233cfea4a18", "kiwi-c000cd056d72295a", "lemon-c52334f9f5cf9be1", "tupperware of food2-a0c838aa23ff496e", "tupperware of food3-4e1390a2b998a25a", "tupperware of food4-be577585fb42aed8", "celery stalks-400b53896c7b2bca", "silver bowl-da8bfe9936f0a849", "pot-71499c16630fa4ba", "second pot-c88a3c9a0951c3df", "lid-d6417fa8d1d85cf5", "colander-65445ce02b5dce74", "lid2-2ddfeb942b09aeec", "small pan-47845b2263b0ae02", "wok-ad2a5eb074a12b67", "cutting board-c22038cb46c6c41c", "plate-a916e0de60db34b1", "spoon holder-d2b4b0e44bac78ca", "measuring cups-ead1849b7fac0a82", "cheese grater-c04a000e7f8d5394", "tupperware top-29d7f2dec98a7396", "knife1-0aac862e38a23a73", "knife2-86f319cb6fa801d8", "can of tomatoes-1a0cd303b655bff6", "jar of tomato sauce-b034c657b5c4c4b4", "cloth-67b8b927f44f4657", "red onion-6ef9fc3979e886e2", "onion2-db5b9f413356eeb2", "bowl2-40c9c67cbffe562e", "tea towel-e529fd9cb8a3920a", "one half of onion-c0215b1c960d17c9", "second onion half-a2fcfcb8230cd301", "bag of garlic-112f6a72a370ef3b", "scraps of garlic cloves-f6f191b4f5c3273e", "smaller blue plate-f74e3a3a17a146f9", "cherry tomatoes-6e430d964f21b2c7", "cherry tomato1-fe6dbc1fd720e7c5", "cherry tomato2-e1db0d76d96e1ea4", "cherry tomato3-d1312e9d288610a6", "cherry tomato4-f683cf85eea55557", "cherry tomato5-4e9576d3b34750dd", "single stalk-b4caf8eb772abd7b", "piece of paper towel-4cf067c5e3ecf79d", "carrot1-e12ac7517bda2270", "peeler-d187fa429d20f414", "green pepper-34d649f28dafa3c0", "pieces of pepper-a5d0fe445ac665af", "scraps of pepper head-a636309488bf3fe7", "mug-57d3f85d89dde3dc", "piece of paper towel2-a9efbfe379c1eda6", "pieces of paper towel3-f88d27d5173cc696", "single mushroom-fb441ba818471372", "plastic casing of mushrooms-d5038e1bd595c139", "bottle of olive oil-7dd447c74efe02b6", "spoon-8223d69e1cc1faa5", "spatula-b7f47c3883af9098", "chilli pepper-282456d213843ff2", "paper container of mushrooms-a38a017a4bc3f46a", "glass-6e0e61cd490497f8", "sponge-229f3a7e23312f21", "dish soap-8c8973299d8802b9", "glass bowl-b90104c5a059e6db" ], "reasoning": "1. I identify the knife block at 06:36 towards the back of the counter in front of the user.\n2. I count 7 knives in the knife block at 06:42.\n3. At 06:53, I observe the user turn left and walk towards the sink .\n4. At 06:54, I count 1 glass cup to the right of the towel (tea towel-e529fd9cb8a3920a) and 1 glass cup (glass-6e0e61cd490497f8) to its left.\n5. I subtract 2 from 7 to get 5.", "referenced_object_masks": [ "glass-6e0e61cd490497f8", "tea towel-e529fd9cb8a3920a" ] }, { "key": "P08-20240618-163547:83fd380d7cfa66e520b77dd70dda3297da61ff1d", "video_id": "P08-20240618-163547", "question": "Where are the scissors in relation to me when I put away the grater?", "answer_choice_0": "Below; facing towards me", "answer_choice_1": "Below; facing away from me", "answer_choice_2": "Eye level; on my right", "answer_choice_3": "Above; facing away from me", "answer_choice_4": "Eye level; facing towards me", "answer_id": 1, "answer": "Below; facing away from me", "question_type": [ "Object Recognition", "Egocentric & Spatial Perception", "Action Recognition" ], "objects_of_interest": [ "phone-9cdc08bf04556c76", "kettle-c93846da3e16a309", "kitchen utensils-e3cec8dbbda54a1f", "weighing scale-fc130cb7b7c69614", "food-ea78597e1899c9f1", "mushroom-bc984594bb46967e", "carrots-e01bc0303c758441", "bowl-e183f8ba79c23e01", "tupperware of food-bb982233cfea4a18", "kiwi-c000cd056d72295a", "lemon-c52334f9f5cf9be1", "tupperware of food2-a0c838aa23ff496e", "tupperware of food3-4e1390a2b998a25a", "tupperware of food4-be577585fb42aed8", "celery stalks-400b53896c7b2bca", "silver bowl-da8bfe9936f0a849", "pot-71499c16630fa4ba", "second pot-c88a3c9a0951c3df", "lid-d6417fa8d1d85cf5", "colander-65445ce02b5dce74", "lid2-2ddfeb942b09aeec", "small pan-47845b2263b0ae02", "wok-ad2a5eb074a12b67", "cutting board-c22038cb46c6c41c", "plate-a916e0de60db34b1", "spoon holder-d2b4b0e44bac78ca", "measuring cups-ead1849b7fac0a82", "cheese grater-c04a000e7f8d5394", "tupperware top-29d7f2dec98a7396", "knife1-0aac862e38a23a73", "knife2-86f319cb6fa801d8", "can of tomatoes-1a0cd303b655bff6", "jar of tomato sauce-b034c657b5c4c4b4", "cloth-67b8b927f44f4657", "red onion-6ef9fc3979e886e2", "onion2-db5b9f413356eeb2", "bowl2-40c9c67cbffe562e", "tea towel-e529fd9cb8a3920a", "one half of onion-c0215b1c960d17c9", "second onion half-a2fcfcb8230cd301", "bag of garlic-112f6a72a370ef3b", "scraps of garlic cloves-f6f191b4f5c3273e", "smaller blue plate-f74e3a3a17a146f9", "cherry tomatoes-6e430d964f21b2c7", "cherry tomato1-fe6dbc1fd720e7c5", "cherry tomato2-e1db0d76d96e1ea4", "cherry tomato3-d1312e9d288610a6", "cherry tomato4-f683cf85eea55557", "cherry tomato5-4e9576d3b34750dd", "single stalk-b4caf8eb772abd7b", "piece of paper towel-4cf067c5e3ecf79d", "carrot1-e12ac7517bda2270", "peeler-d187fa429d20f414", "green pepper-34d649f28dafa3c0", "pieces of pepper-a5d0fe445ac665af", "scraps of pepper head-a636309488bf3fe7", "mug-57d3f85d89dde3dc", "piece of paper towel2-a9efbfe379c1eda6", "pieces of paper towel3-f88d27d5173cc696", "single mushroom-fb441ba818471372", "plastic casing of mushrooms-d5038e1bd595c139", "bottle of olive oil-7dd447c74efe02b6", "spoon-8223d69e1cc1faa5", "spatula-b7f47c3883af9098", "chilli pepper-282456d213843ff2", "paper container of mushrooms-a38a017a4bc3f46a", "glass-6e0e61cd490497f8", "sponge-229f3a7e23312f21", "dish soap-8c8973299d8802b9", "glass bowl-b90104c5a059e6db" ], "reasoning": "1. I determine when the user retrieves the grater (cheese grater-c04a000e7f8d5394). This occurs at 01:49 when the user grabs it from the dish rack . \n2. Next, the user puts the grater (cheese grater-c04a000e7f8d5394) into a pull-out drawer at 01:50, positioned underneath the stove top. The user looks down to put the grater (cheese grater-c04a000e7f8d5394) away. \n3. When the user opens the drawer at 01:50, the light blue handle from a pair of scissors is visible in the top left, amidst an array of other kitchen tools . The handle is facing away from the user.\n4. Since the pull-out drawer is positioned about waist length to the user, while the scissor handle is facing away from the user, the answer is C.", "referenced_object_masks": [ "cheese grater-c04a000e7f8d5394" ] }, { "key": "P08-20240618-163547:b394b4c6482c4192d0b46b51836df59432255d19", "video_id": "P08-20240618-163547", "question": "Where is the matching green top in relation to the bottle of siracha?", "answer_choice_0": "Behind and to the right", "answer_choice_1": "In front and to the left", "answer_choice_2": "To the right", "answer_choice_3": "Behind and to the left", "answer_choice_4": "To the left", "answer_id": 0, "answer": "Behind and to the right", "question_type": [ "Object Recognition", "Egocentric & Spatial Perception", "Counting" ], "objects_of_interest": [ "phone-9cdc08bf04556c76", "kettle-c93846da3e16a309", "kitchen utensils-e3cec8dbbda54a1f", "weighing scale-fc130cb7b7c69614", "food-ea78597e1899c9f1", "mushroom-bc984594bb46967e", "carrots-e01bc0303c758441", "bowl-e183f8ba79c23e01", "tupperware of food-bb982233cfea4a18", "kiwi-c000cd056d72295a", "lemon-c52334f9f5cf9be1", "tupperware of food2-a0c838aa23ff496e", "tupperware of food3-4e1390a2b998a25a", "tupperware of food4-be577585fb42aed8", "celery stalks-400b53896c7b2bca", "silver bowl-da8bfe9936f0a849", "pot-71499c16630fa4ba", "second pot-c88a3c9a0951c3df", "lid-d6417fa8d1d85cf5", "colander-65445ce02b5dce74", "lid2-2ddfeb942b09aeec", "small pan-47845b2263b0ae02", "wok-ad2a5eb074a12b67", "cutting board-c22038cb46c6c41c", "plate-a916e0de60db34b1", "spoon holder-d2b4b0e44bac78ca", "measuring cups-ead1849b7fac0a82", "cheese grater-c04a000e7f8d5394", "tupperware top-29d7f2dec98a7396", "knife1-0aac862e38a23a73", "knife2-86f319cb6fa801d8", "can of tomatoes-1a0cd303b655bff6", "jar of tomato sauce-b034c657b5c4c4b4", "cloth-67b8b927f44f4657", "red onion-6ef9fc3979e886e2", "onion2-db5b9f413356eeb2", "bowl2-40c9c67cbffe562e", "tea towel-e529fd9cb8a3920a", "one half of onion-c0215b1c960d17c9", "second onion half-a2fcfcb8230cd301", "bag of garlic-112f6a72a370ef3b", "scraps of garlic cloves-f6f191b4f5c3273e", "smaller blue plate-f74e3a3a17a146f9", "cherry tomatoes-6e430d964f21b2c7", "cherry tomato1-fe6dbc1fd720e7c5", "cherry tomato2-e1db0d76d96e1ea4", "cherry tomato3-d1312e9d288610a6", "cherry tomato4-f683cf85eea55557", "cherry tomato5-4e9576d3b34750dd", "single stalk-b4caf8eb772abd7b", "piece of paper towel-4cf067c5e3ecf79d", "carrot1-e12ac7517bda2270", "peeler-d187fa429d20f414", "green pepper-34d649f28dafa3c0", "pieces of pepper-a5d0fe445ac665af", "scraps of pepper head-a636309488bf3fe7", "mug-57d3f85d89dde3dc", "piece of paper towel2-a9efbfe379c1eda6", "pieces of paper towel3-f88d27d5173cc696", "single mushroom-fb441ba818471372", "plastic casing of mushrooms-d5038e1bd595c139", "bottle of olive oil-7dd447c74efe02b6", "spoon-8223d69e1cc1faa5", "spatula-b7f47c3883af9098", "chilli pepper-282456d213843ff2", "paper container of mushrooms-a38a017a4bc3f46a", "glass-6e0e61cd490497f8", "sponge-229f3a7e23312f21", "dish soap-8c8973299d8802b9", "glass bowl-b90104c5a059e6db" ], "reasoning": "1. First, I locate the siracha . I spot it when the user first opens the fridge at 00:19. It is visible on the second shelf, on the left, in a clear bottle with white text and a green top. \n2. Next, I search for the matching green top . I look to the right of the siracha and notice a plastic container with a white label . Behind that container is another green top that matches the one on the siracha bottle. \n3. Therefore, the matching green top is positioned behind and to the right of the siracha bottle .", "referenced_object_masks": [] }, { "key": "P08-20240618-163547:b88bd9ebd054ab0973defdde796746b3c911860e", "video_id": "P08-20240618-163547", "question": "If the calendar on the fridge faces north, which direction would lead me to the candelabra from the third position of the carrots?", "answer_choice_0": "Northeast", "answer_choice_1": "Northwest", "answer_choice_2": "East", "answer_choice_3": "Southeast", "answer_choice_4": "South", "answer_id": 3, "answer": "Southeast", "question_type": [ "Object Recognition", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "phone-9cdc08bf04556c76", "kettle-c93846da3e16a309", "kitchen utensils-e3cec8dbbda54a1f", "weighing scale-fc130cb7b7c69614", "food-ea78597e1899c9f1", "mushroom-bc984594bb46967e", "carrots-e01bc0303c758441", "bowl-e183f8ba79c23e01", "tupperware of food-bb982233cfea4a18", "kiwi-c000cd056d72295a", "lemon-c52334f9f5cf9be1", "tupperware of food2-a0c838aa23ff496e", "tupperware of food3-4e1390a2b998a25a", "tupperware of food4-be577585fb42aed8", "celery stalks-400b53896c7b2bca", "silver bowl-da8bfe9936f0a849", "pot-71499c16630fa4ba", "second pot-c88a3c9a0951c3df", "lid-d6417fa8d1d85cf5", "colander-65445ce02b5dce74", "lid2-2ddfeb942b09aeec", "small pan-47845b2263b0ae02", "wok-ad2a5eb074a12b67", "cutting board-c22038cb46c6c41c", "plate-a916e0de60db34b1", "spoon holder-d2b4b0e44bac78ca", "measuring cups-ead1849b7fac0a82", "cheese grater-c04a000e7f8d5394", "tupperware top-29d7f2dec98a7396", "knife1-0aac862e38a23a73", "knife2-86f319cb6fa801d8", "can of tomatoes-1a0cd303b655bff6", "jar of tomato sauce-b034c657b5c4c4b4", "cloth-67b8b927f44f4657", "red onion-6ef9fc3979e886e2", "onion2-db5b9f413356eeb2", "bowl2-40c9c67cbffe562e", "tea towel-e529fd9cb8a3920a", "one half of onion-c0215b1c960d17c9", "second onion half-a2fcfcb8230cd301", "bag of garlic-112f6a72a370ef3b", "scraps of garlic cloves-f6f191b4f5c3273e", "smaller blue plate-f74e3a3a17a146f9", "cherry tomatoes-6e430d964f21b2c7", "cherry tomato1-fe6dbc1fd720e7c5", "cherry tomato2-e1db0d76d96e1ea4", "cherry tomato3-d1312e9d288610a6", "cherry tomato4-f683cf85eea55557", "cherry tomato5-4e9576d3b34750dd", "single stalk-b4caf8eb772abd7b", "piece of paper towel-4cf067c5e3ecf79d", "carrot1-e12ac7517bda2270", "peeler-d187fa429d20f414", "green pepper-34d649f28dafa3c0", "pieces of pepper-a5d0fe445ac665af", "scraps of pepper head-a636309488bf3fe7", "mug-57d3f85d89dde3dc", "piece of paper towel2-a9efbfe379c1eda6", "pieces of paper towel3-f88d27d5173cc696", "single mushroom-fb441ba818471372", "plastic casing of mushrooms-d5038e1bd595c139", "bottle of olive oil-7dd447c74efe02b6", "spoon-8223d69e1cc1faa5", "spatula-b7f47c3883af9098", "chilli pepper-282456d213843ff2", "paper container of mushrooms-a38a017a4bc3f46a", "glass-6e0e61cd490497f8", "sponge-229f3a7e23312f21", "dish soap-8c8973299d8802b9", "glass bowl-b90104c5a059e6db" ], "reasoning": "1. I first identified the calendar on the refrigerator at 00:16. The calendar directly faces the left side of the sink and the corner of the kitchen counter on the opposite side of the room.\n2. I identified the first position of the bag of carrots (carrots-e01bc0303c758441) in the refrigerator drawer at 00:22 when the user picks it up.\n3. The user places the carrots (carrots-e01bc0303c758441) down on the counter near the corner at 00:26. This is the second position of the carrots (carrots-e01bc0303c758441).\n4. The user places the carrots (carrots-e01bc0303c758441) down on the counter to the immediate right of the sink at 02:07. This is the third position of the carrots (carrots-e01bc0303c758441).\n5. I then identified the candelabra on the dining table at 24:11 when the user walked toward the dining table from the corner of the kitchen at 24:08.\n6. According to the orientation of north provided by the calendar, which faces the sink perpendicularly, to get to the candelabra from the immediate right of the sink, where the third position of the carrots (carrots-e01bc0303c758441) is located, the user would walk in the southeast direction.", "referenced_object_masks": [ "carrots-e01bc0303c758441" ] }, { "key": "P08-20240618-163547:bf0876a9781dabe12cbdcc54be98993be9439cb6", "video_id": "P08-20240618-163547", "question": "What number would I get if I added the number of mushrooms on the cutting board at 21:01 and the number of times I touch my phone's screen the next time I use it?", "answer_choice_0": "11", "answer_choice_1": "14", "answer_choice_2": "13", "answer_choice_3": "12", "answer_choice_4": "15", "answer_id": 2, "answer": "13", "question_type": [ "Counting", "Egocentric & Spatial Perception", "Numerical Reasoning", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "phone-9cdc08bf04556c76", "kettle-c93846da3e16a309", "kitchen utensils-e3cec8dbbda54a1f", "weighing scale-fc130cb7b7c69614", "food-ea78597e1899c9f1", "mushroom-bc984594bb46967e", "carrots-e01bc0303c758441", "bowl-e183f8ba79c23e01", "tupperware of food-bb982233cfea4a18", "kiwi-c000cd056d72295a", "lemon-c52334f9f5cf9be1", "tupperware of food2-a0c838aa23ff496e", "tupperware of food3-4e1390a2b998a25a", "tupperware of food4-be577585fb42aed8", "celery stalks-400b53896c7b2bca", "silver bowl-da8bfe9936f0a849", "pot-71499c16630fa4ba", "second pot-c88a3c9a0951c3df", "lid-d6417fa8d1d85cf5", "colander-65445ce02b5dce74", "lid2-2ddfeb942b09aeec", "small pan-47845b2263b0ae02", "wok-ad2a5eb074a12b67", "cutting board-c22038cb46c6c41c", "plate-a916e0de60db34b1", "spoon holder-d2b4b0e44bac78ca", "measuring cups-ead1849b7fac0a82", "cheese grater-c04a000e7f8d5394", "tupperware top-29d7f2dec98a7396", "knife1-0aac862e38a23a73", "knife2-86f319cb6fa801d8", "can of tomatoes-1a0cd303b655bff6", "jar of tomato sauce-b034c657b5c4c4b4", "cloth-67b8b927f44f4657", "red onion-6ef9fc3979e886e2", "onion2-db5b9f413356eeb2", "bowl2-40c9c67cbffe562e", "tea towel-e529fd9cb8a3920a", "one half of onion-c0215b1c960d17c9", "second onion half-a2fcfcb8230cd301", "bag of garlic-112f6a72a370ef3b", "scraps of garlic cloves-f6f191b4f5c3273e", "smaller blue plate-f74e3a3a17a146f9", "cherry tomatoes-6e430d964f21b2c7", "cherry tomato1-fe6dbc1fd720e7c5", "cherry tomato2-e1db0d76d96e1ea4", "cherry tomato3-d1312e9d288610a6", "cherry tomato4-f683cf85eea55557", "cherry tomato5-4e9576d3b34750dd", "single stalk-b4caf8eb772abd7b", "piece of paper towel-4cf067c5e3ecf79d", "carrot1-e12ac7517bda2270", "peeler-d187fa429d20f414", "green pepper-34d649f28dafa3c0", "pieces of pepper-a5d0fe445ac665af", "scraps of pepper head-a636309488bf3fe7", "mug-57d3f85d89dde3dc", "piece of paper towel2-a9efbfe379c1eda6", "pieces of paper towel3-f88d27d5173cc696", "single mushroom-fb441ba818471372", "plastic casing of mushrooms-d5038e1bd595c139", "bottle of olive oil-7dd447c74efe02b6", "spoon-8223d69e1cc1faa5", "spatula-b7f47c3883af9098", "chilli pepper-282456d213843ff2", "paper container of mushrooms-a38a017a4bc3f46a", "glass-6e0e61cd490497f8", "sponge-229f3a7e23312f21", "dish soap-8c8973299d8802b9", "glass bowl-b90104c5a059e6db" ], "reasoning": "1. I go to the timestamp at 21:01 and look for the cutting board (cutting board-c22038cb46c6c41c).\n2. On the cutting board (cutting board-c22038cb46c6c41c), I count 7 mushrooms (mushroom-bc984594bb46967e).\n3. After 21:03 I observe that the user begins using their phone (phone-9cdc08bf04556c76) at 21:04 and puts it back on the counter at 21:10.\n4. Between 21:04 and 21:10, I observe the user touch the screen on their phone (phone-9cdc08bf04556c76) 6 times.\n5. I add 6 and 7 to get 13.", "referenced_object_masks": [ "mushroom-bc984594bb46967e", "phone-9cdc08bf04556c76", "cutting board-c22038cb46c6c41c" ] }, { "key": "P08-20240618-163547:d7561e7ea1232882c5817f24eb52756e5182e492", "video_id": "P08-20240618-163547", "question": "What image is visible on my phone's homescreen?", "answer_choice_0": "Sunny sky over beach", "answer_choice_1": "Cloudy sky over water", "answer_choice_2": "Plain white background", "answer_choice_3": "Plain blue background", "answer_choice_4": "Abstract colorful image", "answer_id": 1, "answer": "Cloudy sky over water", "question_type": [ "Object Recognition", "State Changes" ], "objects_of_interest": [ "phone-9cdc08bf04556c76", "kettle-c93846da3e16a309", "kitchen utensils-e3cec8dbbda54a1f", "weighing scale-fc130cb7b7c69614", "food-ea78597e1899c9f1", "mushroom-bc984594bb46967e", "carrots-e01bc0303c758441", "bowl-e183f8ba79c23e01", "tupperware of food-bb982233cfea4a18", "kiwi-c000cd056d72295a", "lemon-c52334f9f5cf9be1", "tupperware of food2-a0c838aa23ff496e", "tupperware of food3-4e1390a2b998a25a", "tupperware of food4-be577585fb42aed8", "celery stalks-400b53896c7b2bca", "silver bowl-da8bfe9936f0a849", "pot-71499c16630fa4ba", "second pot-c88a3c9a0951c3df", "lid-d6417fa8d1d85cf5", "colander-65445ce02b5dce74", "lid2-2ddfeb942b09aeec", "small pan-47845b2263b0ae02", "wok-ad2a5eb074a12b67", "cutting board-c22038cb46c6c41c", "plate-a916e0de60db34b1", "spoon holder-d2b4b0e44bac78ca", "measuring cups-ead1849b7fac0a82", "cheese grater-c04a000e7f8d5394", "tupperware top-29d7f2dec98a7396", "knife1-0aac862e38a23a73", "knife2-86f319cb6fa801d8", "can of tomatoes-1a0cd303b655bff6", "jar of tomato sauce-b034c657b5c4c4b4", "cloth-67b8b927f44f4657", "red onion-6ef9fc3979e886e2", "onion2-db5b9f413356eeb2", "bowl2-40c9c67cbffe562e", "tea towel-e529fd9cb8a3920a", "one half of onion-c0215b1c960d17c9", "second onion half-a2fcfcb8230cd301", "bag of garlic-112f6a72a370ef3b", "scraps of garlic cloves-f6f191b4f5c3273e", "smaller blue plate-f74e3a3a17a146f9", "cherry tomatoes-6e430d964f21b2c7", "cherry tomato1-fe6dbc1fd720e7c5", "cherry tomato2-e1db0d76d96e1ea4", "cherry tomato3-d1312e9d288610a6", "cherry tomato4-f683cf85eea55557", "cherry tomato5-4e9576d3b34750dd", "single stalk-b4caf8eb772abd7b", "piece of paper towel-4cf067c5e3ecf79d", "carrot1-e12ac7517bda2270", "peeler-d187fa429d20f414", "green pepper-34d649f28dafa3c0", "pieces of pepper-a5d0fe445ac665af", "scraps of pepper head-a636309488bf3fe7", "mug-57d3f85d89dde3dc", "piece of paper towel2-a9efbfe379c1eda6", "pieces of paper towel3-f88d27d5173cc696", "single mushroom-fb441ba818471372", "plastic casing of mushrooms-d5038e1bd595c139", "bottle of olive oil-7dd447c74efe02b6", "spoon-8223d69e1cc1faa5", "spatula-b7f47c3883af9098", "chilli pepper-282456d213843ff2", "paper container of mushrooms-a38a017a4bc3f46a", "glass-6e0e61cd490497f8", "sponge-229f3a7e23312f21", "dish soap-8c8973299d8802b9", "glass bowl-b90104c5a059e6db" ], "reasoning": "1. I locate timestamps where I can see the user's homescreen. I found these on their phone (phone-9cdc08bf04556c76) at 00:09, 28:56, and 29:02.\n2. Next, I tried to distinguish the image on the phone's (phone-9cdc08bf04556c76) homescreen. \n3. Upon looking closely at each of the timestamps, I determined the homescreen displays a static image of a dark, cloudy sky above a mass of lapping waves.", "referenced_object_masks": [ "phone-9cdc08bf04556c76" ] }, { "key": "P08-20240618-163547:f13b296e9f44404514ac5112c12e9dbd7edafe3c", "video_id": "P08-20240618-163547", "question": "What number would I get if I added the number of times I hit the side of the wok from 28:45 to 28:49 and the number of times I swipe on my phone the next time I use it before picking it up off of the counter?", "answer_choice_0": "4", "answer_choice_1": "7", "answer_choice_2": "8", "answer_choice_3": "6", "answer_choice_4": "5", "answer_id": 3, "answer": "6", "question_type": [ "Counting", "Egocentric & Spatial Perception", "Numerical Reasoning", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "phone-9cdc08bf04556c76", "kettle-c93846da3e16a309", "kitchen utensils-e3cec8dbbda54a1f", "weighing scale-fc130cb7b7c69614", "food-ea78597e1899c9f1", "mushroom-bc984594bb46967e", "carrots-e01bc0303c758441", "bowl-e183f8ba79c23e01", "tupperware of food-bb982233cfea4a18", "kiwi-c000cd056d72295a", "lemon-c52334f9f5cf9be1", "tupperware of food2-a0c838aa23ff496e", "tupperware of food3-4e1390a2b998a25a", "tupperware of food4-be577585fb42aed8", "celery stalks-400b53896c7b2bca", "silver bowl-da8bfe9936f0a849", "pot-71499c16630fa4ba", "second pot-c88a3c9a0951c3df", "lid-d6417fa8d1d85cf5", "colander-65445ce02b5dce74", "lid2-2ddfeb942b09aeec", "small pan-47845b2263b0ae02", "wok-ad2a5eb074a12b67", "cutting board-c22038cb46c6c41c", "plate-a916e0de60db34b1", "spoon holder-d2b4b0e44bac78ca", "measuring cups-ead1849b7fac0a82", "cheese grater-c04a000e7f8d5394", "tupperware top-29d7f2dec98a7396", "knife1-0aac862e38a23a73", "knife2-86f319cb6fa801d8", "can of tomatoes-1a0cd303b655bff6", "jar of tomato sauce-b034c657b5c4c4b4", "cloth-67b8b927f44f4657", "red onion-6ef9fc3979e886e2", "onion2-db5b9f413356eeb2", "bowl2-40c9c67cbffe562e", "tea towel-e529fd9cb8a3920a", "one half of onion-c0215b1c960d17c9", "second onion half-a2fcfcb8230cd301", "bag of garlic-112f6a72a370ef3b", "scraps of garlic cloves-f6f191b4f5c3273e", "smaller blue plate-f74e3a3a17a146f9", "cherry tomatoes-6e430d964f21b2c7", "cherry tomato1-fe6dbc1fd720e7c5", "cherry tomato2-e1db0d76d96e1ea4", "cherry tomato3-d1312e9d288610a6", "cherry tomato4-f683cf85eea55557", "cherry tomato5-4e9576d3b34750dd", "single stalk-b4caf8eb772abd7b", "piece of paper towel-4cf067c5e3ecf79d", "carrot1-e12ac7517bda2270", "peeler-d187fa429d20f414", "green pepper-34d649f28dafa3c0", "pieces of pepper-a5d0fe445ac665af", "scraps of pepper head-a636309488bf3fe7", "mug-57d3f85d89dde3dc", "piece of paper towel2-a9efbfe379c1eda6", "pieces of paper towel3-f88d27d5173cc696", "single mushroom-fb441ba818471372", "plastic casing of mushrooms-d5038e1bd595c139", "bottle of olive oil-7dd447c74efe02b6", "spoon-8223d69e1cc1faa5", "spatula-b7f47c3883af9098", "chilli pepper-282456d213843ff2", "paper container of mushrooms-a38a017a4bc3f46a", "glass-6e0e61cd490497f8", "sponge-229f3a7e23312f21", "dish soap-8c8973299d8802b9", "glass bowl-b90104c5a059e6db" ], "reasoning": "1. I identify the wok at the timestamp at 28:45 to 28:49.\n2. I observe the user hit the side of the wok 4 times.\n3. Directly after, I observe the user on their phone (phone-9cdc08bf04556c76) from 28:54 to 29:03.\n4. From 28:54 to 29:09, I identify the user swipe their phone (phone-9cdc08bf04556c76) 2 times.\n5. I add 4 and 2 to get 6.", "referenced_object_masks": [ "phone-9cdc08bf04556c76" ] }, { "key": "P08-20240618-171546:16b97e97eac7001298ed2446b7068103cde72025", "video_id": "P08-20240618-171546", "question": "What direction would I have turned to look at the smartphone and mixing bowl of mushrooms and other vegetables if they were in the opposite direction while I was at the stove?", "answer_choice_0": "180 degrees behind", "answer_choice_1": "135 degrees to the right", "answer_choice_2": "135 degrees to the left", "answer_choice_3": "90 degrees to the left", "answer_choice_4": "90 degrees to the right", "answer_id": 4, "answer": "90 degrees to the right", "question_type": [ "Situational Awareness", "Object Recognition", "Action Recognition", "Egocentric & Spatial Perception", "Temporal Reasoning", "Counterfactual" ], "objects_of_interest": [ "wok-9e1cf84269b6186a", "spatula-c63b57339b7d9c48", "jar-a20f81c12f00cd5e", "can of tomato sauce-b66cd8d180fa4b8f", "beef mince-5fdf23e1941e9c7f", "scissors-95741841fc545be2", "bottle of olive oil-e0b595e03e0b19d2", "salt shaker-1b786d29af96e9db", "pepper shaker-76e4b064dbd9fb46", "bottle cap-a489634d271d8d59", "phone-29c473e138a33e05", "mixing bowl-4c993a72b68c5108", "can opener-5d8bc575627017b7", "frying pan-e005204aa3f833a1", "frying pan2-609d7da1454cea57", "peeler-71e624fe22d2bc33", "sponge-ea0ab2781cc19d6b", "mug-25b9d5083485a62e", "piece of paper towel-462f32b420f2aedc", "weighing scales-88a9a7bf423ddb49", "mug holder-ec12d118dcb03860", "spice jar-5d60c007db7aa1f1", "lid of spice jar-518fb2059fb40af9", "spoon-f560e9e6a8bc038d", "another jar-b7666e418b5f2bd3", "lid of jar-16bc701950b4653d", "piece of paper towel2-9b837d51f2c4c407", "spoon2-582b24de2d13f792", "flask-cac3c3ae306368a2", "lid of flask-c67c8d7fdd9f8fcc", "spray-17b55bbaf1ac91c5", "tea towel-b6698e7a1e158d69", "measuring cups-f2f3c038f53ca901", "cheese grater-f44c7756d54da468", "coffee mixer-b5f7e0abcd8fbdf8", "coffee machine-138152c0c0e31c51", "kettle-f9e2e04932a1ef26", "packet of pasta-08ec74bb13028081", "clip-57d52c6d658fd4b8", "spaghetti-0febc2a934afe2cf", "bowl-62526f084883b12d" ], "reasoning": "1. I watched the video and found the stove at 00:02.\n2. At 00:14, I saw the smartphone (phone-29c473e138a33e05) and mixing bowl (mixing bowl-4c993a72b68c5108) of mushrooms and other vegetables when the user turned 90 degrees to the left to look from the stove and towards the smartphone (phone-29c473e138a33e05) and mixing bowl (mixing bowl-4c993a72b68c5108).\n3. Since the user turned 90 degrees to the left to view the smartphone (phone-29c473e138a33e05) and mixing bowl (mixing bowl-4c993a72b68c5108) from the stove, the smartphone (phone-29c473e138a33e05) and mixing bowl (mixing bowl-4c993a72b68c5108) were consequently to the left of the user while they were at the stove.\n4. Since the opposite of \"to the left\" is \"to the right,\" the opposite of \"90 degrees to the left\" is consequently \"90 degrees to the right.\"\n5. Therefore, the user would have turned 90 degrees to the right to look at the smartphone (phone-29c473e138a33e05) and mixing bowl (mixing bowl-4c993a72b68c5108) of mushrooms and other vegetables if they were in the opposite direction while the user was at the stove.", "referenced_object_masks": [ "phone-29c473e138a33e05", "mixing bowl-4c993a72b68c5108" ] }, { "key": "P08-20240618-171546:21c1d8c0eac69a091ced5ac7f4c0b444ba8e8f3a", "video_id": "P08-20240618-171546", "question": "What is the second item I touch with my right hand after dumping the meat into the pan and placing its packaging onto the counter?", "answer_choice_0": "Scissors", "answer_choice_1": "Can", "answer_choice_2": "Spatula", "answer_choice_3": "Wok handle", "answer_choice_4": "The meat packaging", "answer_id": 4, "answer": "The meat packaging", "question_type": [ "Egocentric & Spatial Perception", "Action Recognition", "Object Recognition", "Situational Awareness", "Temporal Reasoning" ], "objects_of_interest": [ "wok-9e1cf84269b6186a", "spatula-c63b57339b7d9c48", "jar-a20f81c12f00cd5e", "can of tomato sauce-b66cd8d180fa4b8f", "beef mince-5fdf23e1941e9c7f", "scissors-95741841fc545be2", "bottle of olive oil-e0b595e03e0b19d2", "salt shaker-1b786d29af96e9db", "pepper shaker-76e4b064dbd9fb46", "bottle cap-a489634d271d8d59", "phone-29c473e138a33e05", "mixing bowl-4c993a72b68c5108", "can opener-5d8bc575627017b7", "frying pan-e005204aa3f833a1", "frying pan2-609d7da1454cea57", "peeler-71e624fe22d2bc33", "sponge-ea0ab2781cc19d6b", "mug-25b9d5083485a62e", "piece of paper towel-462f32b420f2aedc", "weighing scales-88a9a7bf423ddb49", "mug holder-ec12d118dcb03860", "spice jar-5d60c007db7aa1f1", "lid of spice jar-518fb2059fb40af9", "spoon-f560e9e6a8bc038d", "another jar-b7666e418b5f2bd3", "lid of jar-16bc701950b4653d", "piece of paper towel2-9b837d51f2c4c407", "spoon2-582b24de2d13f792", "flask-cac3c3ae306368a2", "lid of flask-c67c8d7fdd9f8fcc", "spray-17b55bbaf1ac91c5", "tea towel-b6698e7a1e158d69", "measuring cups-f2f3c038f53ca901", "cheese grater-f44c7756d54da468", "coffee mixer-b5f7e0abcd8fbdf8", "coffee machine-138152c0c0e31c51", "kettle-f9e2e04932a1ef26", "packet of pasta-08ec74bb13028081", "clip-57d52c6d658fd4b8", "spaghetti-0febc2a934afe2cf", "bowl-62526f084883b12d" ], "reasoning": "1. I watched the video to observe when the user squeezes meat into a wok and discard the packaging.\n2. I saw the user squeeze meat (beef mince-5fdf23e1941e9c7f) into a wok (wok-9e1cf84269b6186a) from 00:35-00:39, and at 00:40, the user places the discarded packaging to their left onto the counter. After this event, I began to watch for items picked up in the user's right hand.\n3. Once the meat (beef mince-5fdf23e1941e9c7f) was in the wok (wok-9e1cf84269b6186a), I saw the user grab the spatula (spatula-c63b57339b7d9c48) with their right hand at 00:41. This is the first item grabbed with their right hand after dumping the meat into the wok (wok-9e1cf84269b6186a) and discarding the packaging .\n4. I continued watching until I observed the user turn to grab the empty meat packaging from 01:22-01:26, grabbing it in their right hand at 01:25, making the meat packaging the second item they grabbed with their right hand.", "referenced_object_masks": [ "wok-9e1cf84269b6186a", "spatula-c63b57339b7d9c48", "beef mince-5fdf23e1941e9c7f" ] }, { "key": "P08-20240618-171546:38760532bfc98d5f500803b095b518be7e85b78e", "video_id": "P08-20240618-171546", "question": "What item did I touch with my right hand immediately after using the spatula for the third time?", "answer_choice_0": "A salt shaker", "answer_choice_1": "A bottle of olive oil", "answer_choice_2": "The wok", "answer_choice_3": "Scissors", "answer_choice_4": "A pepper shaker", "answer_id": 1, "answer": "A bottle of olive oil", "question_type": [ "Object Recognition", "Counting", "Action Recognition", "Situational Awareness", "Temporal Reasoning", "State Changes", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "wok-9e1cf84269b6186a", "spatula-c63b57339b7d9c48", "jar-a20f81c12f00cd5e", "can of tomato sauce-b66cd8d180fa4b8f", "beef mince-5fdf23e1941e9c7f", "scissors-95741841fc545be2", "bottle of olive oil-e0b595e03e0b19d2", "salt shaker-1b786d29af96e9db", "pepper shaker-76e4b064dbd9fb46", "bottle cap-a489634d271d8d59", "phone-29c473e138a33e05", "mixing bowl-4c993a72b68c5108", "can opener-5d8bc575627017b7", "frying pan-e005204aa3f833a1", "frying pan2-609d7da1454cea57", "peeler-71e624fe22d2bc33", "sponge-ea0ab2781cc19d6b", "mug-25b9d5083485a62e", "piece of paper towel-462f32b420f2aedc", "weighing scales-88a9a7bf423ddb49", "mug holder-ec12d118dcb03860", "spice jar-5d60c007db7aa1f1", "lid of spice jar-518fb2059fb40af9", "spoon-f560e9e6a8bc038d", "another jar-b7666e418b5f2bd3", "lid of jar-16bc701950b4653d", "piece of paper towel2-9b837d51f2c4c407", "spoon2-582b24de2d13f792", "flask-cac3c3ae306368a2", "lid of flask-c67c8d7fdd9f8fcc", "spray-17b55bbaf1ac91c5", "tea towel-b6698e7a1e158d69", "measuring cups-f2f3c038f53ca901", "cheese grater-f44c7756d54da468", "coffee mixer-b5f7e0abcd8fbdf8", "coffee machine-138152c0c0e31c51", "kettle-f9e2e04932a1ef26", "packet of pasta-08ec74bb13028081", "clip-57d52c6d658fd4b8", "spaghetti-0febc2a934afe2cf", "bowl-62526f084883b12d" ], "reasoning": "1. I watched the video and found the spatula (spatula-c63b57339b7d9c48) at 00:03.\n2. I counted each time the user used the spatula (spatula-c63b57339b7d9c48).\n3. I saw the user use the spatula (spatula-c63b57339b7d9c48) for the first time at 00:03-00:13.\n4. I saw the user use the spatula (spatula-c63b57339b7d9c48) for the second time at 00:41-01:23.\n5. I saw the user use the spatula (spatula-c63b57339b7d9c48) for the third time at 1:32-1:39.\n6. Immediately after 01:39, I saw the user touch a bottle of olive oil (bottle of olive oil-e0b595e03e0b19d2) with their right hand at 01:40.\n7. Therefore, the user touched a bottle of olive oil (bottle of olive oil-e0b595e03e0b19d2) with their right hand immediately after using the spatula (spatula-c63b57339b7d9c48) for the third time.", "referenced_object_masks": [ "bottle of olive oil-e0b595e03e0b19d2", "spatula-c63b57339b7d9c48" ] }, { "key": "P08-20240618-171546:44b7532e088ccbefbd458ab280111587d6e889aa", "video_id": "P08-20240618-171546", "question": "In what order would I have added the first 3 ingredients to the wok of vegetables if I had done so in reverse?", "answer_choice_0": "Salt, ground beef, pepper", "answer_choice_1": "Salt, pepper, ground beef", "answer_choice_2": "Ground beef, pepper, salt", "answer_choice_3": "Pepper, salt, ground beef", "answer_choice_4": "Ground beef, salt, pepper", "answer_id": 3, "answer": "Pepper, salt, ground beef", "question_type": [ "Object Recognition", "Counting", "Action Recognition", "State Changes", "Temporal Reasoning", "Counterfactual" ], "objects_of_interest": [ "wok-9e1cf84269b6186a", "spatula-c63b57339b7d9c48", "jar-a20f81c12f00cd5e", "can of tomato sauce-b66cd8d180fa4b8f", "beef mince-5fdf23e1941e9c7f", "scissors-95741841fc545be2", "bottle of olive oil-e0b595e03e0b19d2", "salt shaker-1b786d29af96e9db", "pepper shaker-76e4b064dbd9fb46", "bottle cap-a489634d271d8d59", "phone-29c473e138a33e05", "mixing bowl-4c993a72b68c5108", "can opener-5d8bc575627017b7", "frying pan-e005204aa3f833a1", "frying pan2-609d7da1454cea57", "peeler-71e624fe22d2bc33", "sponge-ea0ab2781cc19d6b", "mug-25b9d5083485a62e", "piece of paper towel-462f32b420f2aedc", "weighing scales-88a9a7bf423ddb49", "mug holder-ec12d118dcb03860", "spice jar-5d60c007db7aa1f1", "lid of spice jar-518fb2059fb40af9", "spoon-f560e9e6a8bc038d", "another jar-b7666e418b5f2bd3", "lid of jar-16bc701950b4653d", "piece of paper towel2-9b837d51f2c4c407", "spoon2-582b24de2d13f792", "flask-cac3c3ae306368a2", "lid of flask-c67c8d7fdd9f8fcc", "spray-17b55bbaf1ac91c5", "tea towel-b6698e7a1e158d69", "measuring cups-f2f3c038f53ca901", "cheese grater-f44c7756d54da468", "coffee mixer-b5f7e0abcd8fbdf8", "coffee machine-138152c0c0e31c51", "kettle-f9e2e04932a1ef26", "packet of pasta-08ec74bb13028081", "clip-57d52c6d658fd4b8", "spaghetti-0febc2a934afe2cf", "bowl-62526f084883b12d" ], "reasoning": "1. I watched the video and found the wok (wok-9e1cf84269b6186a) of vegetables at 00:02.\n2. I counted the ingredients the user added to the wok (wok-9e1cf84269b6186a) of vegetables.\n3. At 00:39, I saw the user add the first ingredient to the wok (wok-9e1cf84269b6186a) of vegetables, which was ground beef (beef mince-5fdf23e1941e9c7f).\n4. At 01:42, I saw the user add the second ingredient to the wok (wok-9e1cf84269b6186a) of vegetables, which was salt from a salt shaker (salt shaker-1b786d29af96e9db).\n5. At 01:46, I saw the user add the third ingredient to the wok (wok-9e1cf84269b6186a) of vegetables, which was pepper from a pepper shaker (pepper shaker-76e4b064dbd9fb46).\n6. Since the user added ground beef (beef mince-5fdf23e1941e9c7f), then salt from a salt shaker (salt shaker-1b786d29af96e9db), and then pepper from a pepper shaker (pepper shaker-76e4b064dbd9fb46) to the wok (wok-9e1cf84269b6186a) of vegetables, the user would have therefore added the first 3 ingredients to the wok (wok-9e1cf84269b6186a) of vegetables in this order if they had done so in reverse: Pepper, salt, ground beef.", "referenced_object_masks": [ "wok-9e1cf84269b6186a", "pepper shaker-76e4b064dbd9fb46", "salt shaker-1b786d29af96e9db", "beef mince-5fdf23e1941e9c7f" ] }, { "key": "P08-20240618-171546:831e7fd4b80490ef4e61f39d744f42b04947f0ab", "video_id": "P08-20240618-171546", "question": "If the handle of the wok represented the minute hand of a clock and the handle of the black skillet represented the hour hand of a clock, approximately what time is represented when I put the black pan face down on the top of the wok?", "answer_choice_0": "7:05", "answer_choice_1": "2:20", "answer_choice_2": "4:30", "answer_choice_3": "3:45", "answer_choice_4": "6:15", "answer_id": 3, "answer": "3:45", "question_type": [ "Temporal Reasoning", "Object Recognition", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "wok-9e1cf84269b6186a", "spatula-c63b57339b7d9c48", "jar-a20f81c12f00cd5e", "can of tomato sauce-b66cd8d180fa4b8f", "beef mince-5fdf23e1941e9c7f", "scissors-95741841fc545be2", "bottle of olive oil-e0b595e03e0b19d2", "salt shaker-1b786d29af96e9db", "pepper shaker-76e4b064dbd9fb46", "bottle cap-a489634d271d8d59", "phone-29c473e138a33e05", "mixing bowl-4c993a72b68c5108", "can opener-5d8bc575627017b7", "frying pan-e005204aa3f833a1", "frying pan2-609d7da1454cea57", "peeler-71e624fe22d2bc33", "sponge-ea0ab2781cc19d6b", "mug-25b9d5083485a62e", "piece of paper towel-462f32b420f2aedc", "weighing scales-88a9a7bf423ddb49", "mug holder-ec12d118dcb03860", "spice jar-5d60c007db7aa1f1", "lid of spice jar-518fb2059fb40af9", "spoon-f560e9e6a8bc038d", "another jar-b7666e418b5f2bd3", "lid of jar-16bc701950b4653d", "piece of paper towel2-9b837d51f2c4c407", "spoon2-582b24de2d13f792", "flask-cac3c3ae306368a2", "lid of flask-c67c8d7fdd9f8fcc", "spray-17b55bbaf1ac91c5", "tea towel-b6698e7a1e158d69", "measuring cups-f2f3c038f53ca901", "cheese grater-f44c7756d54da468", "coffee mixer-b5f7e0abcd8fbdf8", "coffee machine-138152c0c0e31c51", "kettle-f9e2e04932a1ef26", "packet of pasta-08ec74bb13028081", "clip-57d52c6d658fd4b8", "spaghetti-0febc2a934afe2cf", "bowl-62526f084883b12d" ], "reasoning": "1. I observed the wok (wok-9e1cf84269b6186a) on the stove at 00:02.\n2. I continued to watch the video until I observed from 08:39-08:41, when the user lifts the black skillet (frying pan-e005204aa3f833a1) and places it upside down on top of the wok (wok-9e1cf84269b6186a).\n3. At this time, specifically using the frame at 08:40, I noticed the skillet handle (frying pan-e005204aa3f833a1) facing in the direction of 3 o'clock, and the wok handle (wok-9e1cf84269b6186a) facing in the direction of 45 minutes.\n4. Thus, these pans represent 3:45.", "referenced_object_masks": [ "wok-9e1cf84269b6186a", "frying pan-e005204aa3f833a1" ] }, { "key": "P08-20240618-171546:864cdd3513ecdb73b397efbe7cc25b417857789c", "video_id": "P08-20240618-171546", "question": "If I had mixed the wok of vegetables and ground beef twice as fast before adding any other ingredients, approximately how much time would I have spent doing so?", "answer_choice_0": "30 seconds", "answer_choice_1": "20 seconds", "answer_choice_2": "40 seconds", "answer_choice_3": "10 seconds", "answer_choice_4": "50 seconds", "answer_id": 1, "answer": "20 seconds", "question_type": [ "Object Recognition", "Action Recognition", "State Changes", "Temporal Reasoning", "Numerical Reasoning", "Counterfactual" ], "objects_of_interest": [ "wok-9e1cf84269b6186a", "spatula-c63b57339b7d9c48", "jar-a20f81c12f00cd5e", "can of tomato sauce-b66cd8d180fa4b8f", "beef mince-5fdf23e1941e9c7f", "scissors-95741841fc545be2", "bottle of olive oil-e0b595e03e0b19d2", "salt shaker-1b786d29af96e9db", "pepper shaker-76e4b064dbd9fb46", "bottle cap-a489634d271d8d59", "phone-29c473e138a33e05", "mixing bowl-4c993a72b68c5108", "can opener-5d8bc575627017b7", "frying pan-e005204aa3f833a1", "frying pan2-609d7da1454cea57", "peeler-71e624fe22d2bc33", "sponge-ea0ab2781cc19d6b", "mug-25b9d5083485a62e", "piece of paper towel-462f32b420f2aedc", "weighing scales-88a9a7bf423ddb49", "mug holder-ec12d118dcb03860", "spice jar-5d60c007db7aa1f1", "lid of spice jar-518fb2059fb40af9", "spoon-f560e9e6a8bc038d", "another jar-b7666e418b5f2bd3", "lid of jar-16bc701950b4653d", "piece of paper towel2-9b837d51f2c4c407", "spoon2-582b24de2d13f792", "flask-cac3c3ae306368a2", "lid of flask-c67c8d7fdd9f8fcc", "spray-17b55bbaf1ac91c5", "tea towel-b6698e7a1e158d69", "measuring cups-f2f3c038f53ca901", "cheese grater-f44c7756d54da468", "coffee mixer-b5f7e0abcd8fbdf8", "coffee machine-138152c0c0e31c51", "kettle-f9e2e04932a1ef26", "packet of pasta-08ec74bb13028081", "clip-57d52c6d658fd4b8", "spaghetti-0febc2a934afe2cf", "bowl-62526f084883b12d" ], "reasoning": "1. I watched the video and found the wok (wok-9e1cf84269b6186a) of vegetables at 00:02.\n2. At 00:39, I saw the user add ground beef (beef mince-5fdf23e1941e9c7f) to the wok (wok-9e1cf84269b6186a) of vegetables.\n3. I saw the user mix the wok (wok-9e1cf84269b6186a) of vegetables and ground beef (beef mince-5fdf23e1941e9c7f) at 00:42-01:22, calculating a total of 40 seconds.\n4. I saw the user mix the wok (wok-9e1cf84269b6186a) of vegetables and ground beef (beef mince-5fdf23e1941e9c7f) again at 01:33-01:39, calculating a total of 6 seconds.\n5. At 01:42, I saw the user add another ingredient to the wok (wok-9e1cf84269b6186a) of vegetables and ground beef (beef mince-5fdf23e1941e9c7f), which was salt from a salt shaker (salt shaker-1b786d29af96e9db).\n6. Since the user did not mix the wok (wok-9e1cf84269b6186a) of vegetables and ground beef (beef mince-5fdf23e1941e9c7f) again after 01:39 and before adding salt at 01:42, I calculated 40+6=46.\n7. I calculated 46/2=23, which approximates to 20.\n8. Therefore, if the user had mixed the wok (wok-9e1cf84269b6186a) of vegetables and ground beef (beef mince-5fdf23e1941e9c7f) twice as fast before adding any other ingredients, they would have spent approximately 20 seconds doing so.", "referenced_object_masks": [ "wok-9e1cf84269b6186a", "salt shaker-1b786d29af96e9db", "beef mince-5fdf23e1941e9c7f" ] }, { "key": "P08-20240618-171546:a510070e1bcfe77a8ebf1f835bcca37ac044b984", "video_id": "P08-20240618-171546", "question": "How many turns do I perform with the can opener?", "answer_choice_0": "17", "answer_choice_1": "20", "answer_choice_2": "12", "answer_choice_3": "13", "answer_choice_4": "15", "answer_id": 1, "answer": "20", "question_type": [ "Counting", "Egocentric & Spatial Perception", "Numerical Reasoning", "Action Recognition" ], "objects_of_interest": [ "wok-9e1cf84269b6186a", "spatula-c63b57339b7d9c48", "jar-a20f81c12f00cd5e", "can of tomato sauce-b66cd8d180fa4b8f", "beef mince-5fdf23e1941e9c7f", "scissors-95741841fc545be2", "bottle of olive oil-e0b595e03e0b19d2", "salt shaker-1b786d29af96e9db", "pepper shaker-76e4b064dbd9fb46", "bottle cap-a489634d271d8d59", "phone-29c473e138a33e05", "mixing bowl-4c993a72b68c5108", "can opener-5d8bc575627017b7", "frying pan-e005204aa3f833a1", "frying pan2-609d7da1454cea57", "peeler-71e624fe22d2bc33", "sponge-ea0ab2781cc19d6b", "mug-25b9d5083485a62e", "piece of paper towel-462f32b420f2aedc", "weighing scales-88a9a7bf423ddb49", "mug holder-ec12d118dcb03860", "spice jar-5d60c007db7aa1f1", "lid of spice jar-518fb2059fb40af9", "spoon-f560e9e6a8bc038d", "another jar-b7666e418b5f2bd3", "lid of jar-16bc701950b4653d", "piece of paper towel2-9b837d51f2c4c407", "spoon2-582b24de2d13f792", "flask-cac3c3ae306368a2", "lid of flask-c67c8d7fdd9f8fcc", "spray-17b55bbaf1ac91c5", "tea towel-b6698e7a1e158d69", "measuring cups-f2f3c038f53ca901", "cheese grater-f44c7756d54da468", "coffee mixer-b5f7e0abcd8fbdf8", "coffee machine-138152c0c0e31c51", "kettle-f9e2e04932a1ef26", "packet of pasta-08ec74bb13028081", "clip-57d52c6d658fd4b8", "spaghetti-0febc2a934afe2cf", "bowl-62526f084883b12d" ], "reasoning": "1. I watched the video to identify when the user grabs the can opener (can opener-5d8bc575627017b7), which occurs from 05:22-05:23.\n2. Then, I observed the user pick up the can opener (can opener-5d8bc575627017b7) and place it on top of the can (can of tomato sauce-b66cd8d180fa4b8f) at 05:25.\n3. Next, I watched the user use the right hand to turn it at the following times: 05:26, 05:27, 05:27, 05:28, 05:29, and 05:30. This is a total of six times so far.\n4. The user then briefly looks left, away from the can (can of tomato sauce-b66cd8d180fa4b8f), before turning back to look at the can.\n5. They then continued cranking the can opener (can opener-5d8bc575627017b7) at: 05:31, 05:31, 05:32, 05:33, 05:33, 05:34, totaling 6 more times.\n6. The user then lifts the can opener (can opener-5d8bc575627017b7) from 05:36-05:38.\n7. The user returns the can opener (can opener-5d8bc575627017b7) to the can from 05:40-05:47, before turning it again at 05:48, 05:49, 05:50, 05:50, 05:51, 05:52, 05:53, 05:54, totaling 8 additional turns.\n8. After this span, the user clearly has the lid of the can open as seen at 06:00.\n9. Final math here is 6 + 6 + 8 = 20 turns.", "referenced_object_masks": [ "can opener-5d8bc575627017b7", "can of tomato sauce-b66cd8d180fa4b8f" ] }, { "key": "P08-20240618-171546:ba1b6278f16ef9e23aef7e6f416ec42291751e4c", "video_id": "P08-20240618-171546", "question": "If I would have grabbed an item from the bottom shelf of the cabinet at 14:44, what material would it have been made of?", "answer_choice_0": "Ceramic", "answer_choice_1": "Wood", "answer_choice_2": "Steel", "answer_choice_3": "Glass", "answer_choice_4": "Plastic", "answer_id": 3, "answer": "Glass", "question_type": [ "Egocentric & Spatial Perception", "Counterfactual", "Object Recognition", "State Changes" ], "objects_of_interest": [ "wok-9e1cf84269b6186a", "spatula-c63b57339b7d9c48", "jar-a20f81c12f00cd5e", "can of tomato sauce-b66cd8d180fa4b8f", "beef mince-5fdf23e1941e9c7f", "scissors-95741841fc545be2", "bottle of olive oil-e0b595e03e0b19d2", "salt shaker-1b786d29af96e9db", "pepper shaker-76e4b064dbd9fb46", "bottle cap-a489634d271d8d59", "phone-29c473e138a33e05", "mixing bowl-4c993a72b68c5108", "can opener-5d8bc575627017b7", "frying pan-e005204aa3f833a1", "frying pan2-609d7da1454cea57", "peeler-71e624fe22d2bc33", "sponge-ea0ab2781cc19d6b", "mug-25b9d5083485a62e", "piece of paper towel-462f32b420f2aedc", "weighing scales-88a9a7bf423ddb49", "mug holder-ec12d118dcb03860", "spice jar-5d60c007db7aa1f1", "lid of spice jar-518fb2059fb40af9", "spoon-f560e9e6a8bc038d", "another jar-b7666e418b5f2bd3", "lid of jar-16bc701950b4653d", "piece of paper towel2-9b837d51f2c4c407", "spoon2-582b24de2d13f792", "flask-cac3c3ae306368a2", "lid of flask-c67c8d7fdd9f8fcc", "spray-17b55bbaf1ac91c5", "tea towel-b6698e7a1e158d69", "measuring cups-f2f3c038f53ca901", "cheese grater-f44c7756d54da468", "coffee mixer-b5f7e0abcd8fbdf8", "coffee machine-138152c0c0e31c51", "kettle-f9e2e04932a1ef26", "packet of pasta-08ec74bb13028081", "clip-57d52c6d658fd4b8", "spaghetti-0febc2a934afe2cf", "bowl-62526f084883b12d" ], "reasoning": "1. I fast-forwarded the video to 14:44 to observe the user opening a cabinet door .\n2. In the video, the user grabbed a mug from the top shelf.\n3. When the cabinet door is opened, I noted that the bottom shelf is full of glass cups , meaning the item would have been glass if the user would have grabbed something from the bottom shelf.", "referenced_object_masks": [] }, { "key": "P08-20240618-171546:ee5bb15117fe7a9de4256fbbc6ecb1be9deb0c43", "video_id": "P08-20240618-171546", "question": "What item was closest to the mixing bowl when I put it down after adding the second half of mushrooms and other vegetables to the wok of vegetables and ground beef?", "answer_choice_0": "A smartphone", "answer_choice_1": "A gray pair of scissors", "answer_choice_2": "A can opener", "answer_choice_3": "A jar of red sauce", "answer_choice_4": "A can of tomato sauce", "answer_id": 3, "answer": "A jar of red sauce", "question_type": [ "Object Recognition", "Action Recognition", "State Changes", "Counting", "Temporal Reasoning", "Egocentric & Spatial Perception", "Situational Awareness" ], "objects_of_interest": [ "wok-9e1cf84269b6186a", "spatula-c63b57339b7d9c48", "jar-a20f81c12f00cd5e", "can of tomato sauce-b66cd8d180fa4b8f", "beef mince-5fdf23e1941e9c7f", "scissors-95741841fc545be2", "bottle of olive oil-e0b595e03e0b19d2", "salt shaker-1b786d29af96e9db", "pepper shaker-76e4b064dbd9fb46", "bottle cap-a489634d271d8d59", "phone-29c473e138a33e05", "mixing bowl-4c993a72b68c5108", "can opener-5d8bc575627017b7", "frying pan-e005204aa3f833a1", "frying pan2-609d7da1454cea57", "peeler-71e624fe22d2bc33", "sponge-ea0ab2781cc19d6b", "mug-25b9d5083485a62e", "piece of paper towel-462f32b420f2aedc", "weighing scales-88a9a7bf423ddb49", "mug holder-ec12d118dcb03860", "spice jar-5d60c007db7aa1f1", "lid of spice jar-518fb2059fb40af9", "spoon-f560e9e6a8bc038d", "another jar-b7666e418b5f2bd3", "lid of jar-16bc701950b4653d", "piece of paper towel2-9b837d51f2c4c407", "spoon2-582b24de2d13f792", "flask-cac3c3ae306368a2", "lid of flask-c67c8d7fdd9f8fcc", "spray-17b55bbaf1ac91c5", "tea towel-b6698e7a1e158d69", "measuring cups-f2f3c038f53ca901", "cheese grater-f44c7756d54da468", "coffee mixer-b5f7e0abcd8fbdf8", "coffee machine-138152c0c0e31c51", "kettle-f9e2e04932a1ef26", "packet of pasta-08ec74bb13028081", "clip-57d52c6d658fd4b8", "spaghetti-0febc2a934afe2cf", "bowl-62526f084883b12d" ], "reasoning": "1. I watched the video and found the mixing bowl (mixing bowl-4c993a72b68c5108) of mushrooms and other vegetables at 00:00.\n2. I found the wok (wok-9e1cf84269b6186a) of vegetables at 00:02.\n3. At 00:39, I saw the user add ground beef (beef mince-5fdf23e1941e9c7f) to the wok (wok-9e1cf84269b6186a) of vegetables.\n4. At 07:39-07:42, I saw the user add the first half of the mushrooms and other vegetables from the mixing bowl (mixing bowl-4c993a72b68c5108) to the wok (wok-9e1cf84269b6186a) of vegetables and ground beef (beef mince-5fdf23e1941e9c7f).\n5. At 08:02-08:06, I saw the user add the second half of the mushrooms and other vegetables from the mixing bowl (mixing bowl-4c993a72b68c5108) to the wok (wok-9e1cf84269b6186a) of vegetables and ground beef (beef mince-5fdf23e1941e9c7f).\n6. After 08:06, I saw the user put the mixing bowl (mixing bowl-4c993a72b68c5108) down on the counter at 08:07.\n7. At 08:07, I saw the closest item to the mixing bowl (mixing bowl-4c993a72b68c5108) on the counter was a jar of red sauce (jar-a20f81c12f00cd5e).\n8. Therefore, a jar of red sauce (jar-a20f81c12f00cd5e) was closest to the mixing bowl (mixing bowl-4c993a72b68c5108) when the user put it down after adding the second half of mushrooms and other vegetables to the wok (wok-9e1cf84269b6186a) of vegetables and ground beef (beef mince-5fdf23e1941e9c7f).", "referenced_object_masks": [ "jar-a20f81c12f00cd5e", "wok-9e1cf84269b6186a", "mixing bowl-4c993a72b68c5108", "beef mince-5fdf23e1941e9c7f" ] }, { "key": "P08-20240618-171546:f2daf491a979f49166db5be5b55f68cde8b24e17", "video_id": "P08-20240618-171546", "question": "How many items are in the holder behind which I place the scale?", "answer_choice_0": "9", "answer_choice_1": "7", "answer_choice_2": "10", "answer_choice_3": "5", "answer_choice_4": "4", "answer_id": 1, "answer": "7", "question_type": [ "Counting", "Object Recognition", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "wok-9e1cf84269b6186a", "spatula-c63b57339b7d9c48", "jar-a20f81c12f00cd5e", "can of tomato sauce-b66cd8d180fa4b8f", "beef mince-5fdf23e1941e9c7f", "scissors-95741841fc545be2", "bottle of olive oil-e0b595e03e0b19d2", "salt shaker-1b786d29af96e9db", "pepper shaker-76e4b064dbd9fb46", "bottle cap-a489634d271d8d59", "phone-29c473e138a33e05", "mixing bowl-4c993a72b68c5108", "can opener-5d8bc575627017b7", "frying pan-e005204aa3f833a1", "frying pan2-609d7da1454cea57", "peeler-71e624fe22d2bc33", "sponge-ea0ab2781cc19d6b", "mug-25b9d5083485a62e", "piece of paper towel-462f32b420f2aedc", "weighing scales-88a9a7bf423ddb49", "mug holder-ec12d118dcb03860", "spice jar-5d60c007db7aa1f1", "lid of spice jar-518fb2059fb40af9", "spoon-f560e9e6a8bc038d", "another jar-b7666e418b5f2bd3", "lid of jar-16bc701950b4653d", "piece of paper towel2-9b837d51f2c4c407", "spoon2-582b24de2d13f792", "flask-cac3c3ae306368a2", "lid of flask-c67c8d7fdd9f8fcc", "spray-17b55bbaf1ac91c5", "tea towel-b6698e7a1e158d69", "measuring cups-f2f3c038f53ca901", "cheese grater-f44c7756d54da468", "coffee mixer-b5f7e0abcd8fbdf8", "coffee machine-138152c0c0e31c51", "kettle-f9e2e04932a1ef26", "packet of pasta-08ec74bb13028081", "clip-57d52c6d658fd4b8", "spaghetti-0febc2a934afe2cf", "bowl-62526f084883b12d" ], "reasoning": "1. I went through the video to find when the scale (weighing scales-88a9a7bf423ddb49) is seen, which first occurs at 09:14.\n2. From 09:14-09:24, I watched the user clean and dry the scale (weighing scales-88a9a7bf423ddb49).\n3. Next, the user picks up the scale (weighing scales-88a9a7bf423ddb49) and carries it to place it behind a utensil holder from 09:25-09:28.\n4. At 09:28, when the user places the scale (weighing scales-88a9a7bf423ddb49) behind the utensil holder , I counted 7 items in the holder.", "referenced_object_masks": [ "weighing scales-88a9a7bf423ddb49" ] }, { "key": "P08-20240620-180825:0649d35112882bb4753fc7de7c2428b55c507069", "video_id": "P08-20240620-180825", "question": "Which of the following objects do I not touch in the last minute of the video: Oven mitt, sink faucet, cutting board, scissors, tongs?", "answer_choice_0": "Scissors", "answer_choice_1": "Tongs", "answer_choice_2": "Sink Faucet", "answer_choice_3": "Cutting board", "answer_choice_4": "Oven mitt", "answer_id": 3, "answer": "Cutting board", "question_type": [ "Action Recognition", "Event Occurence", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "yellow pepper-babae09606ab4edc", "drawer of refrigerator-17cf55903f4849fb", "tupperware-0a4b3f7f0fae76a5", "cucumber-0973c52bca2fef6e", "baking tray-5e5421fe80e19cb1", "air fryer-b60a6ab79acb3e0d", "plug of air fryer-b5e2882ea27725f8", "bag of spinach-ff6081050ae00655", "plate-257627f16a89b2cb", "kitchen scale-6c2924912db32cf8", "bag of frozen vegetables-8ae5f6f9cceaf41a", "pot-561e9854f36e9f7e", "lid of pot-0817af02743a0337", "bag of rice-0456bd286bc00134", "phone-1dba9f164ad57f40", "measuring cups-6005da177eafde1e", "bowl-77ed94918a5599be", "scissors-3065ff742d9267c1", "tea towel-0f28a0dce6d162c5", "stove dial-a90a8ac2f0bc4804", "salt shaker-6bc8ca30a1847f65", "pepper shaker-7431f67f76b7d559", "garlic seasoning-8a6ab4f37d0e2775", "jar of seasoning-03463ea90de2ac45", "cap of garlic seasoning-1c36de87dc2ffcd6", "piece of broccoli-d033c7bab390c1f3", "second piece of broccoli-fad6d4f3e6e41101", "third piece of broccoli-d7a9f53993881341", "fourth piece of broccoli-41ff48c80f5c92db", "fifth piece of broccoli-04bb1360d3815577", "sixth piece of broccoli-14c5da1ed3142e37", "drawer of air fryer-a613897e83a3c877", "tea towel2-2c96921d1cd75d30", "bag of chicken-4abd2b86cd3a1ef6", "packaging of tinfoil-32380bbed975ec15", "piece of foil-15ae1d8f7a8f831d", "piece of chicken-fa548325107b7866", "second piece of chicken-bf92426e9e2cedfe", "spoon-25276264100e174e", "oil spray bottle-65d6cb29f54312d5", "top of pepper shaker-72961afc5f5acb6d", "brush-f58cf76d0ad57a92", "baking tray2-68728c3b25e70f39", "cutting board-1a23d017a8c8d36f", "clip-3a6f7c51cf92a519", "spoon holder-22b5c8e6d26022fe", "bowl2-e556f8124c4a408a", "spinach-064f07846b929b92", "spinach2-682184714cc7ed07", "spinach3-36ffd7dd4dee82c6", "phone charging wire-853bacc7a502d6ce", "phone charger-da52695873e86e3d", "spinach pile-e16976a388a79952", "sleeve-723c9fd65c448082", "scissors2-b4228fe77d62957b", "knife holder-3b775df4b11cbfcd", "knife-94df1ae1bab0a0b4", "slice of pepper-8b4e223d6c7aa3b7", "cherry tomato-37519f26ef2765ec", "cherry tomato2-b3b4073d6a5985d8", "cherry tomato3-627b9ccb2c8e9084", "cucumber piece-5f0decd07bab5d7b", "pieces of cucumber-617a4a2abf6c97ce", "cucumber piece2-e12e1f98962e468b", "new piece of cucumber-a8bfba66a295fec3", "saucepan-6bf9d129dafe826e", "water bottle1-7ab8b701cc3a0f13", "water bottle2-5f07caa9b5a1d83f", "top of water bottle2-7d2db336c5ce1806", "plastic packaging-eafff0d979d39a54", "washing up liquid bottle-f612ae3976312575", "recycling bin-020d43c49fd953e9", "small plate-e72f14c675fdf9a0", "tupperware2-ebb84424cbe23181", "top of water bottle1-9cba2b0f9bb07788", "water bottle3-edf424ba7f54b9c7", "peeler-e1e20e75e64c4a36", "food scraps-d285ba8901800756", "trash can-513ab3aa61889210", "oven mitt-d5b830a450d18126", "tongs-2780bc44c478f786", "lemon-17218cd2009a4c14", "new half of lemon-30e49428d7ae3967", "spoon2-0d4ef6178025765f", "spoon3-b11bb7384fab3e26", "wooden spoon-27489baf0229d2a3", "sponge-2a21e098a4cca24e", "dish soap-1745f720e87221df", "fork-2b0b17b51b8c6f89", "fork2-68891a2cac1d102e", "new coffee filter-7ddfac24e378c3fb", "bowl3-4dbfa119a4920d2f", "spoon4-8fba0cb53557d9df", "disk of air fryer-5e4ce1fd127fcbcb", "knife2-aa04eeaffd80a25b", "fork3-86d5aaea5c6ebbb8", "spoon5-a457392eb9662626", "packaging of sponge-2fadc3e44770bbaa", "sponge2-e9d54376cbc0ce6b", "surface cleaner-070286e45645388f", "coriander packaging-61055eb1d1e2de8b", "coriander-c1a24fbb19a74dc2", "tea towel3-f9c4e66f35e38c15", "knife3-9626293a70777190" ], "reasoning": "1. I watched the whole video and the last minute is 27:11 to 28:11.\n2. Then, I looked for the user to touch items during that timespan. At 27:18, the user is washing the tongs (tongs-2780bc44c478f786) at the kitchen sink .\n3. At 27:27, the user washes the scissors (scissors-3065ff742d9267c1) at the kitchen sink .\n4. At 27:30, the user touches the faucet at the kitchen sink .\n5. At 27:43, the user picks up the oven mitt (oven mitt-d5b830a450d18126) to get the aluminum foil (piece of foil-15ae1d8f7a8f831d) out of the baking tray (baking tray-5e5421fe80e19cb1).\n6. Therefore, the only item the user doesn't touch in the last minute of the video is the cutting board (cutting board-1a23d017a8c8d36f).", "referenced_object_masks": [ "baking tray-5e5421fe80e19cb1", "piece of foil-15ae1d8f7a8f831d", "oven mitt-d5b830a450d18126", "cutting board-1a23d017a8c8d36f", "tongs-2780bc44c478f786", "scissors-3065ff742d9267c1" ] }, { "key": "P08-20240620-180825:3508438b9ed42a523e6baf90eddeb85ebd8e607d", "video_id": "P08-20240620-180825", "question": "What is the second-to-last action I performed with the spoon before using it to scoop broccoli from the air fryer?", "answer_choice_0": "Licking it", "answer_choice_1": "Stirring rice", "answer_choice_2": "Putting it in the dish drainer", "answer_choice_3": "Combining salad ingredients", "answer_choice_4": "Cleaning off another spoon", "answer_id": 3, "answer": "Combining salad ingredients", "question_type": [ "Temporal Reasoning", "Object Recognition", "Action Recognition", "Counting" ], "objects_of_interest": [ "yellow pepper-babae09606ab4edc", "drawer of refrigerator-17cf55903f4849fb", "tupperware-0a4b3f7f0fae76a5", "cucumber-0973c52bca2fef6e", "baking tray-5e5421fe80e19cb1", "air fryer-b60a6ab79acb3e0d", "plug of air fryer-b5e2882ea27725f8", "bag of spinach-ff6081050ae00655", "plate-257627f16a89b2cb", "kitchen scale-6c2924912db32cf8", "bag of frozen vegetables-8ae5f6f9cceaf41a", "pot-561e9854f36e9f7e", "lid of pot-0817af02743a0337", "bag of rice-0456bd286bc00134", "phone-1dba9f164ad57f40", "measuring cups-6005da177eafde1e", "bowl-77ed94918a5599be", "scissors-3065ff742d9267c1", "tea towel-0f28a0dce6d162c5", "stove dial-a90a8ac2f0bc4804", "salt shaker-6bc8ca30a1847f65", "pepper shaker-7431f67f76b7d559", "garlic seasoning-8a6ab4f37d0e2775", "jar of seasoning-03463ea90de2ac45", "cap of garlic seasoning-1c36de87dc2ffcd6", "piece of broccoli-d033c7bab390c1f3", "second piece of broccoli-fad6d4f3e6e41101", "third piece of broccoli-d7a9f53993881341", "fourth piece of broccoli-41ff48c80f5c92db", "fifth piece of broccoli-04bb1360d3815577", "sixth piece of broccoli-14c5da1ed3142e37", "drawer of air fryer-a613897e83a3c877", "tea towel2-2c96921d1cd75d30", "bag of chicken-4abd2b86cd3a1ef6", "packaging of tinfoil-32380bbed975ec15", "piece of foil-15ae1d8f7a8f831d", "piece of chicken-fa548325107b7866", "second piece of chicken-bf92426e9e2cedfe", "spoon-25276264100e174e", "oil spray bottle-65d6cb29f54312d5", "top of pepper shaker-72961afc5f5acb6d", "brush-f58cf76d0ad57a92", "baking tray2-68728c3b25e70f39", "cutting board-1a23d017a8c8d36f", "clip-3a6f7c51cf92a519", "spoon holder-22b5c8e6d26022fe", "bowl2-e556f8124c4a408a", "spinach-064f07846b929b92", "spinach2-682184714cc7ed07", "spinach3-36ffd7dd4dee82c6", "phone charging wire-853bacc7a502d6ce", "phone charger-da52695873e86e3d", "spinach pile-e16976a388a79952", "sleeve-723c9fd65c448082", "scissors2-b4228fe77d62957b", "knife holder-3b775df4b11cbfcd", "knife-94df1ae1bab0a0b4", "slice of pepper-8b4e223d6c7aa3b7", "cherry tomato-37519f26ef2765ec", "cherry tomato2-b3b4073d6a5985d8", "cherry tomato3-627b9ccb2c8e9084", "cucumber piece-5f0decd07bab5d7b", "pieces of cucumber-617a4a2abf6c97ce", "cucumber piece2-e12e1f98962e468b", "new piece of cucumber-a8bfba66a295fec3", "saucepan-6bf9d129dafe826e", "water bottle1-7ab8b701cc3a0f13", "water bottle2-5f07caa9b5a1d83f", "top of water bottle2-7d2db336c5ce1806", "plastic packaging-eafff0d979d39a54", "washing up liquid bottle-f612ae3976312575", "recycling bin-020d43c49fd953e9", "small plate-e72f14c675fdf9a0", "tupperware2-ebb84424cbe23181", "top of water bottle1-9cba2b0f9bb07788", "water bottle3-edf424ba7f54b9c7", "peeler-e1e20e75e64c4a36", "food scraps-d285ba8901800756", "trash can-513ab3aa61889210", "oven mitt-d5b830a450d18126", "tongs-2780bc44c478f786", "lemon-17218cd2009a4c14", "new half of lemon-30e49428d7ae3967", "spoon2-0d4ef6178025765f", "spoon3-b11bb7384fab3e26", "wooden spoon-27489baf0229d2a3", "sponge-2a21e098a4cca24e", "dish soap-1745f720e87221df", "fork-2b0b17b51b8c6f89", "fork2-68891a2cac1d102e", "new coffee filter-7ddfac24e378c3fb", "bowl3-4dbfa119a4920d2f", "spoon4-8fba0cb53557d9df", "disk of air fryer-5e4ce1fd127fcbcb", "knife2-aa04eeaffd80a25b", "fork3-86d5aaea5c6ebbb8", "spoon5-a457392eb9662626", "packaging of sponge-2fadc3e44770bbaa", "sponge2-e9d54376cbc0ce6b", "surface cleaner-070286e45645388f", "coriander packaging-61055eb1d1e2de8b", "coriander-c1a24fbb19a74dc2", "tea towel3-f9c4e66f35e38c15", "knife3-9626293a70777190" ], "reasoning": "1. I watched for the user to scoop broccoli from the air fryer (air fryer-b60a6ab79acb3e0d) with a spoon (spoon4-8fba0cb53557d9df) which occurs at 21:41.\n2. I scrolled earlier in the video to find the last thing the user did with the spoon (spoon4-8fba0cb53557d9df) before this, and at 21:06, the user uses it to scoop rice off of a wooden spoon (wooden spoon-27489baf0229d2a3).\n3. I scrolled back even further, to find the last thing the user did with the spoon (spoon4-8fba0cb53557d9df) before that, and from 18:16-18:26, the user uses it to combine salad ingredients. \n4. This makes the penultimate action that the user performs with the spoon (spoon4-8fba0cb53557d9df) combining salad ingredients.", "referenced_object_masks": [ "air fryer-b60a6ab79acb3e0d", "spoon4-8fba0cb53557d9df", "wooden spoon-27489baf0229d2a3" ] }, { "key": "P08-20240620-180825:66e8306adc8d0c20b8f2206144b6604be8c22919", "video_id": "P08-20240620-180825", "question": "What is the relative position of the first place I put the air fryer, if the stove was at my 12 o'clock?", "answer_choice_0": "9 o'clock", "answer_choice_1": "10 o'clock", "answer_choice_2": "6 o'clock", "answer_choice_3": "11 o'clock", "answer_choice_4": "8 o'clock", "answer_id": 3, "answer": "11 o'clock", "question_type": [ "Egocentric & Spatial Perception", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "yellow pepper-babae09606ab4edc", "drawer of refrigerator-17cf55903f4849fb", "tupperware-0a4b3f7f0fae76a5", "cucumber-0973c52bca2fef6e", "baking tray-5e5421fe80e19cb1", "air fryer-b60a6ab79acb3e0d", "plug of air fryer-b5e2882ea27725f8", "bag of spinach-ff6081050ae00655", "plate-257627f16a89b2cb", "kitchen scale-6c2924912db32cf8", "bag of frozen vegetables-8ae5f6f9cceaf41a", "pot-561e9854f36e9f7e", "lid of pot-0817af02743a0337", "bag of rice-0456bd286bc00134", "phone-1dba9f164ad57f40", "measuring cups-6005da177eafde1e", "bowl-77ed94918a5599be", "scissors-3065ff742d9267c1", "tea towel-0f28a0dce6d162c5", "stove dial-a90a8ac2f0bc4804", "salt shaker-6bc8ca30a1847f65", "pepper shaker-7431f67f76b7d559", "garlic seasoning-8a6ab4f37d0e2775", "jar of seasoning-03463ea90de2ac45", "cap of garlic seasoning-1c36de87dc2ffcd6", "piece of broccoli-d033c7bab390c1f3", "second piece of broccoli-fad6d4f3e6e41101", "third piece of broccoli-d7a9f53993881341", "fourth piece of broccoli-41ff48c80f5c92db", "fifth piece of broccoli-04bb1360d3815577", "sixth piece of broccoli-14c5da1ed3142e37", "drawer of air fryer-a613897e83a3c877", "tea towel2-2c96921d1cd75d30", "bag of chicken-4abd2b86cd3a1ef6", "packaging of tinfoil-32380bbed975ec15", "piece of foil-15ae1d8f7a8f831d", "piece of chicken-fa548325107b7866", "second piece of chicken-bf92426e9e2cedfe", "spoon-25276264100e174e", "oil spray bottle-65d6cb29f54312d5", "top of pepper shaker-72961afc5f5acb6d", "brush-f58cf76d0ad57a92", "baking tray2-68728c3b25e70f39", "cutting board-1a23d017a8c8d36f", "clip-3a6f7c51cf92a519", "spoon holder-22b5c8e6d26022fe", "bowl2-e556f8124c4a408a", "spinach-064f07846b929b92", "spinach2-682184714cc7ed07", "spinach3-36ffd7dd4dee82c6", "phone charging wire-853bacc7a502d6ce", "phone charger-da52695873e86e3d", "spinach pile-e16976a388a79952", "sleeve-723c9fd65c448082", "scissors2-b4228fe77d62957b", "knife holder-3b775df4b11cbfcd", "knife-94df1ae1bab0a0b4", "slice of pepper-8b4e223d6c7aa3b7", "cherry tomato-37519f26ef2765ec", "cherry tomato2-b3b4073d6a5985d8", "cherry tomato3-627b9ccb2c8e9084", "cucumber piece-5f0decd07bab5d7b", "pieces of cucumber-617a4a2abf6c97ce", "cucumber piece2-e12e1f98962e468b", "new piece of cucumber-a8bfba66a295fec3", "saucepan-6bf9d129dafe826e", "water bottle1-7ab8b701cc3a0f13", "water bottle2-5f07caa9b5a1d83f", "top of water bottle2-7d2db336c5ce1806", "plastic packaging-eafff0d979d39a54", "washing up liquid bottle-f612ae3976312575", "recycling bin-020d43c49fd953e9", "small plate-e72f14c675fdf9a0", "tupperware2-ebb84424cbe23181", "top of water bottle1-9cba2b0f9bb07788", "water bottle3-edf424ba7f54b9c7", "peeler-e1e20e75e64c4a36", "food scraps-d285ba8901800756", "trash can-513ab3aa61889210", "oven mitt-d5b830a450d18126", "tongs-2780bc44c478f786", "lemon-17218cd2009a4c14", "new half of lemon-30e49428d7ae3967", "spoon2-0d4ef6178025765f", "spoon3-b11bb7384fab3e26", "wooden spoon-27489baf0229d2a3", "sponge-2a21e098a4cca24e", "dish soap-1745f720e87221df", "fork-2b0b17b51b8c6f89", "fork2-68891a2cac1d102e", "new coffee filter-7ddfac24e378c3fb", "bowl3-4dbfa119a4920d2f", "spoon4-8fba0cb53557d9df", "disk of air fryer-5e4ce1fd127fcbcb", "knife2-aa04eeaffd80a25b", "fork3-86d5aaea5c6ebbb8", "spoon5-a457392eb9662626", "packaging of sponge-2fadc3e44770bbaa", "sponge2-e9d54376cbc0ce6b", "surface cleaner-070286e45645388f", "coriander packaging-61055eb1d1e2de8b", "coriander-c1a24fbb19a74dc2", "tea towel3-f9c4e66f35e38c15", "knife3-9626293a70777190" ], "reasoning": "1. At 0:29, I get a good glimpse of the stove. As the user is moving straight toward it, I realize this angle is a perfect demonstration of the stove being at the user's 12.\n2. I identify the location of the first place the user puts the air fryer (air fryer-b60a6ab79acb3e0d). I notice that the user takes it out of a high cupboard and places it to the left of the stove at 00:45.\n3. I realize that, if the user was facing the stove, they would only need to turn about 30 degrees to the left to see the air fryer (air fryer-b60a6ab79acb3e0d).\n4. Were I to convert this change in direction to clock bearing, I would find that the air fryer (air fryer-b60a6ab79acb3e0d) is at 11 o'clock.\n5. The answer, therefore, is 11 o'clock.", "referenced_object_masks": [ "air fryer-b60a6ab79acb3e0d" ] }, { "key": "P08-20240620-180825:70540502c68f9f32dc3ecbc6c8277dafac2654ff", "video_id": "P08-20240620-180825", "question": "If I take the number of pieces of broccoli I put in the air fryer, subtract the number of times I twist the the salt shaker over the broccoli in the air fryer drawer, and then add 7, what number do I get?", "answer_choice_0": "7", "answer_choice_1": "9", "answer_choice_2": "4", "answer_choice_3": "6", "answer_choice_4": "8", "answer_id": 1, "answer": "9", "question_type": [ "Action Recognition", "Counting", "Event Occurence", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "yellow pepper-babae09606ab4edc", "drawer of refrigerator-17cf55903f4849fb", "tupperware-0a4b3f7f0fae76a5", "cucumber-0973c52bca2fef6e", "baking tray-5e5421fe80e19cb1", "air fryer-b60a6ab79acb3e0d", "plug of air fryer-b5e2882ea27725f8", "bag of spinach-ff6081050ae00655", "plate-257627f16a89b2cb", "kitchen scale-6c2924912db32cf8", "bag of frozen vegetables-8ae5f6f9cceaf41a", "pot-561e9854f36e9f7e", "lid of pot-0817af02743a0337", "bag of rice-0456bd286bc00134", "phone-1dba9f164ad57f40", "measuring cups-6005da177eafde1e", "bowl-77ed94918a5599be", "scissors-3065ff742d9267c1", "tea towel-0f28a0dce6d162c5", "stove dial-a90a8ac2f0bc4804", "salt shaker-6bc8ca30a1847f65", "pepper shaker-7431f67f76b7d559", "garlic seasoning-8a6ab4f37d0e2775", "jar of seasoning-03463ea90de2ac45", "cap of garlic seasoning-1c36de87dc2ffcd6", "piece of broccoli-d033c7bab390c1f3", "second piece of broccoli-fad6d4f3e6e41101", "third piece of broccoli-d7a9f53993881341", "fourth piece of broccoli-41ff48c80f5c92db", "fifth piece of broccoli-04bb1360d3815577", "sixth piece of broccoli-14c5da1ed3142e37", "drawer of air fryer-a613897e83a3c877", "tea towel2-2c96921d1cd75d30", "bag of chicken-4abd2b86cd3a1ef6", "packaging of tinfoil-32380bbed975ec15", "piece of foil-15ae1d8f7a8f831d", "piece of chicken-fa548325107b7866", "second piece of chicken-bf92426e9e2cedfe", "spoon-25276264100e174e", "oil spray bottle-65d6cb29f54312d5", "top of pepper shaker-72961afc5f5acb6d", "brush-f58cf76d0ad57a92", "baking tray2-68728c3b25e70f39", "cutting board-1a23d017a8c8d36f", "clip-3a6f7c51cf92a519", "spoon holder-22b5c8e6d26022fe", "bowl2-e556f8124c4a408a", "spinach-064f07846b929b92", "spinach2-682184714cc7ed07", "spinach3-36ffd7dd4dee82c6", "phone charging wire-853bacc7a502d6ce", "phone charger-da52695873e86e3d", "spinach pile-e16976a388a79952", "sleeve-723c9fd65c448082", "scissors2-b4228fe77d62957b", "knife holder-3b775df4b11cbfcd", "knife-94df1ae1bab0a0b4", "slice of pepper-8b4e223d6c7aa3b7", "cherry tomato-37519f26ef2765ec", "cherry tomato2-b3b4073d6a5985d8", "cherry tomato3-627b9ccb2c8e9084", "cucumber piece-5f0decd07bab5d7b", "pieces of cucumber-617a4a2abf6c97ce", "cucumber piece2-e12e1f98962e468b", "new piece of cucumber-a8bfba66a295fec3", "saucepan-6bf9d129dafe826e", "water bottle1-7ab8b701cc3a0f13", "water bottle2-5f07caa9b5a1d83f", "top of water bottle2-7d2db336c5ce1806", "plastic packaging-eafff0d979d39a54", "washing up liquid bottle-f612ae3976312575", "recycling bin-020d43c49fd953e9", "small plate-e72f14c675fdf9a0", "tupperware2-ebb84424cbe23181", "top of water bottle1-9cba2b0f9bb07788", "water bottle3-edf424ba7f54b9c7", "peeler-e1e20e75e64c4a36", "food scraps-d285ba8901800756", "trash can-513ab3aa61889210", "oven mitt-d5b830a450d18126", "tongs-2780bc44c478f786", "lemon-17218cd2009a4c14", "new half of lemon-30e49428d7ae3967", "spoon2-0d4ef6178025765f", "spoon3-b11bb7384fab3e26", "wooden spoon-27489baf0229d2a3", "sponge-2a21e098a4cca24e", "dish soap-1745f720e87221df", "fork-2b0b17b51b8c6f89", "fork2-68891a2cac1d102e", "new coffee filter-7ddfac24e378c3fb", "bowl3-4dbfa119a4920d2f", "spoon4-8fba0cb53557d9df", "disk of air fryer-5e4ce1fd127fcbcb", "knife2-aa04eeaffd80a25b", "fork3-86d5aaea5c6ebbb8", "spoon5-a457392eb9662626", "packaging of sponge-2fadc3e44770bbaa", "sponge2-e9d54376cbc0ce6b", "surface cleaner-070286e45645388f", "coriander packaging-61055eb1d1e2de8b", "coriander-c1a24fbb19a74dc2", "tea towel3-f9c4e66f35e38c15", "knife3-9626293a70777190" ], "reasoning": "1. I saw the user weigh 6 pieces of broccoli (piece of broccoli-d033c7bab390c1f3, second piece of broccoli-fad6d4f3e6e41101, third piece of broccoli-d7a9f53993881341, fourth piece of broccoli-41ff48c80f5c92db, fifth piece of broccoli-04bb1360d3815577) from 05:02-05:23 on a kitchen scale (kitchen scale-6c2924912db32cf8). From 05:29-05:34, the user places these 6 pieces into the air fryer tray (drawer of air fryer-a613897e83a3c877), and loads them into the air fryer (air fryer-b60a6ab79acb3e0d).\n3. At 10:32, the user twisted the salt shaker (salt shaker-6bc8ca30a1847f65) 4 times over the the broccoli in the air fryer drawer (drawer of air fryer-a613897e83a3c877).\n4. I subtract the number of times the salt shaker is twisted over the broccoli in the air fryer drawer from the total number of broccoli pieces put in the air fryer drawer and get 6-4=2.\n5. I add 7 to 2, and find my answer is 9.", "referenced_object_masks": [ "salt shaker-6bc8ca30a1847f65", "piece of broccoli-d033c7bab390c1f3", "second piece of broccoli-fad6d4f3e6e41101", "third piece of broccoli-d7a9f53993881341", "fourth piece of broccoli-41ff48c80f5c92db", "fifth piece of broccoli-04bb1360d3815577", "kitchen scale-6c2924912db32cf8", "air fryer-b60a6ab79acb3e0d", "drawer of air fryer-a613897e83a3c877" ] }, { "key": "P08-20240620-180825:70fdf16f16e07f7b4ad9d8e68c897fc9840a8027", "video_id": "P08-20240620-180825", "question": "How would I need to move to get from the white cutting board while I'm chopping ingredients to the wooden cutting board?", "answer_choice_0": "Turn to the left, open a cabinet", "answer_choice_1": "Turn left and walk to the refrigerator, stand on a chair", "answer_choice_2": "Turn left, open the pantry", "answer_choice_3": "Turn to the right, lift up the dish drainer", "answer_choice_4": "Turn right and walk to the corner, walk right past the stove", "answer_id": 4, "answer": "Turn right and walk to the corner, walk right past the stove", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Situational Awareness" ], "objects_of_interest": [ "yellow pepper-babae09606ab4edc", "drawer of refrigerator-17cf55903f4849fb", "tupperware-0a4b3f7f0fae76a5", "cucumber-0973c52bca2fef6e", "baking tray-5e5421fe80e19cb1", "air fryer-b60a6ab79acb3e0d", "plug of air fryer-b5e2882ea27725f8", "bag of spinach-ff6081050ae00655", "plate-257627f16a89b2cb", "kitchen scale-6c2924912db32cf8", "bag of frozen vegetables-8ae5f6f9cceaf41a", "pot-561e9854f36e9f7e", "lid of pot-0817af02743a0337", "bag of rice-0456bd286bc00134", "phone-1dba9f164ad57f40", "measuring cups-6005da177eafde1e", "bowl-77ed94918a5599be", "scissors-3065ff742d9267c1", "tea towel-0f28a0dce6d162c5", "stove dial-a90a8ac2f0bc4804", "salt shaker-6bc8ca30a1847f65", "pepper shaker-7431f67f76b7d559", "garlic seasoning-8a6ab4f37d0e2775", "jar of seasoning-03463ea90de2ac45", "cap of garlic seasoning-1c36de87dc2ffcd6", "piece of broccoli-d033c7bab390c1f3", "second piece of broccoli-fad6d4f3e6e41101", "third piece of broccoli-d7a9f53993881341", "fourth piece of broccoli-41ff48c80f5c92db", "fifth piece of broccoli-04bb1360d3815577", "sixth piece of broccoli-14c5da1ed3142e37", "drawer of air fryer-a613897e83a3c877", "tea towel2-2c96921d1cd75d30", "bag of chicken-4abd2b86cd3a1ef6", "packaging of tinfoil-32380bbed975ec15", "piece of foil-15ae1d8f7a8f831d", "piece of chicken-fa548325107b7866", "second piece of chicken-bf92426e9e2cedfe", "spoon-25276264100e174e", "oil spray bottle-65d6cb29f54312d5", "top of pepper shaker-72961afc5f5acb6d", "brush-f58cf76d0ad57a92", "baking tray2-68728c3b25e70f39", "cutting board-1a23d017a8c8d36f", "clip-3a6f7c51cf92a519", "spoon holder-22b5c8e6d26022fe", "bowl2-e556f8124c4a408a", "spinach-064f07846b929b92", "spinach2-682184714cc7ed07", "spinach3-36ffd7dd4dee82c6", "phone charging wire-853bacc7a502d6ce", "phone charger-da52695873e86e3d", "spinach pile-e16976a388a79952", "sleeve-723c9fd65c448082", "scissors2-b4228fe77d62957b", "knife holder-3b775df4b11cbfcd", "knife-94df1ae1bab0a0b4", "slice of pepper-8b4e223d6c7aa3b7", "cherry tomato-37519f26ef2765ec", "cherry tomato2-b3b4073d6a5985d8", "cherry tomato3-627b9ccb2c8e9084", "cucumber piece-5f0decd07bab5d7b", "pieces of cucumber-617a4a2abf6c97ce", "cucumber piece2-e12e1f98962e468b", "new piece of cucumber-a8bfba66a295fec3", "saucepan-6bf9d129dafe826e", "water bottle1-7ab8b701cc3a0f13", "water bottle2-5f07caa9b5a1d83f", "top of water bottle2-7d2db336c5ce1806", "plastic packaging-eafff0d979d39a54", "washing up liquid bottle-f612ae3976312575", "recycling bin-020d43c49fd953e9", "small plate-e72f14c675fdf9a0", "tupperware2-ebb84424cbe23181", "top of water bottle1-9cba2b0f9bb07788", "water bottle3-edf424ba7f54b9c7", "peeler-e1e20e75e64c4a36", "food scraps-d285ba8901800756", "trash can-513ab3aa61889210", "oven mitt-d5b830a450d18126", "tongs-2780bc44c478f786", "lemon-17218cd2009a4c14", "new half of lemon-30e49428d7ae3967", "spoon2-0d4ef6178025765f", "spoon3-b11bb7384fab3e26", "wooden spoon-27489baf0229d2a3", "sponge-2a21e098a4cca24e", "dish soap-1745f720e87221df", "fork-2b0b17b51b8c6f89", "fork2-68891a2cac1d102e", "new coffee filter-7ddfac24e378c3fb", "bowl3-4dbfa119a4920d2f", "spoon4-8fba0cb53557d9df", "disk of air fryer-5e4ce1fd127fcbcb", "knife2-aa04eeaffd80a25b", "fork3-86d5aaea5c6ebbb8", "spoon5-a457392eb9662626", "packaging of sponge-2fadc3e44770bbaa", "sponge2-e9d54376cbc0ce6b", "surface cleaner-070286e45645388f", "coriander packaging-61055eb1d1e2de8b", "coriander-c1a24fbb19a74dc2", "tea towel3-f9c4e66f35e38c15", "knife3-9626293a70777190" ], "reasoning": "1. I first looked for the white cutting board (cutting board-1a23d017a8c8d36f) which is seen on the dish drainer at 09:42.\n2. At 09:46, the user moves it to the countertop, where it remains while they chop various ingredients. \n3. I then looked for the wooden cutting board , which can be seen multiple times from 00:10, leaning on the wall to the right of the stove, and never moving from there.\n4. To get from the stated position of the white cutting board (cutting board-1a23d017a8c8d36f) to the wooden cutting board , the user would need to turn right and walk to the corner of the counter, then walk right past the stove.", "referenced_object_masks": [ "cutting board-1a23d017a8c8d36f" ] }, { "key": "P08-20240620-180825:77c2fc54924265298dd9e6be1b30f2d44cdf721c", "video_id": "P08-20240620-180825", "question": "Starting at 15:42, if I take the number of items I put away from the drying rack, add it to the number of items I clean in the sink from 19:06 to 20:32 and divide this number by the number of seconds it took to clean those items at the sink, what approximate number do I get, rounded to the nearest hundredth if need be?", "answer_choice_0": "00.20", "answer_choice_1": "00.25", "answer_choice_2": "00.31", "answer_choice_3": "00.52", "answer_choice_4": "00.15", "answer_id": 0, "answer": "00.20", "question_type": [ "Action Recognition", "Counting", "Event Occurence", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "yellow pepper-babae09606ab4edc", "drawer of refrigerator-17cf55903f4849fb", "tupperware-0a4b3f7f0fae76a5", "cucumber-0973c52bca2fef6e", "baking tray-5e5421fe80e19cb1", "air fryer-b60a6ab79acb3e0d", "plug of air fryer-b5e2882ea27725f8", "bag of spinach-ff6081050ae00655", "plate-257627f16a89b2cb", "kitchen scale-6c2924912db32cf8", "bag of frozen vegetables-8ae5f6f9cceaf41a", "pot-561e9854f36e9f7e", "lid of pot-0817af02743a0337", "bag of rice-0456bd286bc00134", "phone-1dba9f164ad57f40", "measuring cups-6005da177eafde1e", "bowl-77ed94918a5599be", "scissors-3065ff742d9267c1", "tea towel-0f28a0dce6d162c5", "stove dial-a90a8ac2f0bc4804", "salt shaker-6bc8ca30a1847f65", "pepper shaker-7431f67f76b7d559", "garlic seasoning-8a6ab4f37d0e2775", "jar of seasoning-03463ea90de2ac45", "cap of garlic seasoning-1c36de87dc2ffcd6", "piece of broccoli-d033c7bab390c1f3", "second piece of broccoli-fad6d4f3e6e41101", "third piece of broccoli-d7a9f53993881341", "fourth piece of broccoli-41ff48c80f5c92db", "fifth piece of broccoli-04bb1360d3815577", "sixth piece of broccoli-14c5da1ed3142e37", "drawer of air fryer-a613897e83a3c877", "tea towel2-2c96921d1cd75d30", "bag of chicken-4abd2b86cd3a1ef6", "packaging of tinfoil-32380bbed975ec15", "piece of foil-15ae1d8f7a8f831d", "piece of chicken-fa548325107b7866", "second piece of chicken-bf92426e9e2cedfe", "spoon-25276264100e174e", "oil spray bottle-65d6cb29f54312d5", "top of pepper shaker-72961afc5f5acb6d", "brush-f58cf76d0ad57a92", "baking tray2-68728c3b25e70f39", "cutting board-1a23d017a8c8d36f", "clip-3a6f7c51cf92a519", "spoon holder-22b5c8e6d26022fe", "bowl2-e556f8124c4a408a", "spinach-064f07846b929b92", "spinach2-682184714cc7ed07", "spinach3-36ffd7dd4dee82c6", "phone charging wire-853bacc7a502d6ce", "phone charger-da52695873e86e3d", "spinach pile-e16976a388a79952", "sleeve-723c9fd65c448082", "scissors2-b4228fe77d62957b", "knife holder-3b775df4b11cbfcd", "knife-94df1ae1bab0a0b4", "slice of pepper-8b4e223d6c7aa3b7", "cherry tomato-37519f26ef2765ec", "cherry tomato2-b3b4073d6a5985d8", "cherry tomato3-627b9ccb2c8e9084", "cucumber piece-5f0decd07bab5d7b", "pieces of cucumber-617a4a2abf6c97ce", "cucumber piece2-e12e1f98962e468b", "new piece of cucumber-a8bfba66a295fec3", "saucepan-6bf9d129dafe826e", "water bottle1-7ab8b701cc3a0f13", "water bottle2-5f07caa9b5a1d83f", "top of water bottle2-7d2db336c5ce1806", "plastic packaging-eafff0d979d39a54", "washing up liquid bottle-f612ae3976312575", "recycling bin-020d43c49fd953e9", "small plate-e72f14c675fdf9a0", "tupperware2-ebb84424cbe23181", "top of water bottle1-9cba2b0f9bb07788", "water bottle3-edf424ba7f54b9c7", "peeler-e1e20e75e64c4a36", "food scraps-d285ba8901800756", "trash can-513ab3aa61889210", "oven mitt-d5b830a450d18126", "tongs-2780bc44c478f786", "lemon-17218cd2009a4c14", "new half of lemon-30e49428d7ae3967", "spoon2-0d4ef6178025765f", "spoon3-b11bb7384fab3e26", "wooden spoon-27489baf0229d2a3", "sponge-2a21e098a4cca24e", "dish soap-1745f720e87221df", "fork-2b0b17b51b8c6f89", "fork2-68891a2cac1d102e", "new coffee filter-7ddfac24e378c3fb", "bowl3-4dbfa119a4920d2f", "spoon4-8fba0cb53557d9df", "disk of air fryer-5e4ce1fd127fcbcb", "knife2-aa04eeaffd80a25b", "fork3-86d5aaea5c6ebbb8", "spoon5-a457392eb9662626", "packaging of sponge-2fadc3e44770bbaa", "sponge2-e9d54376cbc0ce6b", "surface cleaner-070286e45645388f", "coriander packaging-61055eb1d1e2de8b", "coriander-c1a24fbb19a74dc2", "tea towel3-f9c4e66f35e38c15", "knife3-9626293a70777190" ], "reasoning": "1. I watched the whole video and saw the user put 9 items away from the drying rack at 15:42. At 15:45, the user put away a frying pan (saucepan-6bf9d129dafe826e). At 15:54, the user moved a water bottle (water bottle1-7ab8b701cc3a0f13) to the counter. At 15:56, the user picked up another water bottle (water bottle2-5f07caa9b5a1d83f). At 15:56, the user picked up the top with a straw (top of water bottle2-7d2db336c5ce1806) to this water bottle and screwed it on top of water bottle (water bottle2-5f07caa9b5a1d83f) picked up at 15:56. At 16:01, the user put a plastic container (plastic packaging-eafff0d979d39a54) in the recycling bin (recycling bin-020d43c49fd953e9). At 16:08, the user put a green saucer (small plate-e72f14c675fdf9a0) in a drawer. At 16:15, the user puts a plastic container (tupperware2-ebb84424cbe23181) in a cabinet. At 16:18, the user picks up a black bottle cap (top of water bottle1-9cba2b0f9bb07788) and screws it on the water bottle taken out of the dryer rack at 15:54. At at 16:32, the user puts away a potato peeler (peeler-e1e20e75e64c4a36) in a drawer. \n2. The number of items cleaned from 19:06 to 20:32 is 8. At 19:06, the user cleaned a white bowl . At 19:18, the user cleaned a white plate . At 19:28, the user cleaned a fork (fork2-68891a2cac1d102e). At 19:33, the user cleaned a knife (knife-94df1ae1bab0a0b4). At 19:37, the user cleaned a pair of scissors (scissors-3065ff742d9267c1). At 19:46, the user cleaned a cutting board (cutting board-1a23d017a8c8d36f). At 20:07, the user cleaned a coffee filter (new coffee filter-7ddfac24e378c3fb). At 20:20, the user cleaned the kitchen scale (kitchen scale-6c2924912db32cf8).\n3. The number of seconds from 19:06 to 20:32 is 86.\n4. I add together the number of items the user puts away from the drying rack, 9, and the number of items cleaned, 8, and I get 17.\n5. 17 divided by 86 is 00.0.19767.\n6. Rounded to the hundredth place is 00.20.", "referenced_object_masks": [ "peeler-e1e20e75e64c4a36", "water bottle1-7ab8b701cc3a0f13", "top of water bottle2-7d2db336c5ce1806", "knife-94df1ae1bab0a0b4", "plastic packaging-eafff0d979d39a54", "water bottle2-5f07caa9b5a1d83f", "recycling bin-020d43c49fd953e9", "scissors-3065ff742d9267c1", "tupperware2-ebb84424cbe23181", "new coffee filter-7ddfac24e378c3fb", "kitchen scale-6c2924912db32cf8", "top of water bottle1-9cba2b0f9bb07788", "small plate-e72f14c675fdf9a0", "fork2-68891a2cac1d102e", "saucepan-6bf9d129dafe826e", "cutting board-1a23d017a8c8d36f" ] }, { "key": "P08-20240620-180825:9ec2ab1b3dc0d7b04b3eff0563b9bf7c9a87c3b2", "video_id": "P08-20240620-180825", "question": "Before I pick up the black scissors for the first time, where are they in relation to the gray scissors?", "answer_choice_0": "On the other side of the drawer", "answer_choice_1": "To the left", "answer_choice_2": "To the right", "answer_choice_3": "Underneath them", "answer_choice_4": "On top of them", "answer_id": 4, "answer": "On top of them", "question_type": [ "Action Recognition", "Object Recognition" ], "objects_of_interest": [ "yellow pepper-babae09606ab4edc", "drawer of refrigerator-17cf55903f4849fb", "tupperware-0a4b3f7f0fae76a5", "cucumber-0973c52bca2fef6e", "baking tray-5e5421fe80e19cb1", "air fryer-b60a6ab79acb3e0d", "plug of air fryer-b5e2882ea27725f8", "bag of spinach-ff6081050ae00655", "plate-257627f16a89b2cb", "kitchen scale-6c2924912db32cf8", "bag of frozen vegetables-8ae5f6f9cceaf41a", "pot-561e9854f36e9f7e", "lid of pot-0817af02743a0337", "bag of rice-0456bd286bc00134", "phone-1dba9f164ad57f40", "measuring cups-6005da177eafde1e", "bowl-77ed94918a5599be", "scissors-3065ff742d9267c1", "tea towel-0f28a0dce6d162c5", "stove dial-a90a8ac2f0bc4804", "salt shaker-6bc8ca30a1847f65", "pepper shaker-7431f67f76b7d559", "garlic seasoning-8a6ab4f37d0e2775", "jar of seasoning-03463ea90de2ac45", "cap of garlic seasoning-1c36de87dc2ffcd6", "piece of broccoli-d033c7bab390c1f3", "second piece of broccoli-fad6d4f3e6e41101", "third piece of broccoli-d7a9f53993881341", "fourth piece of broccoli-41ff48c80f5c92db", "fifth piece of broccoli-04bb1360d3815577", "sixth piece of broccoli-14c5da1ed3142e37", "drawer of air fryer-a613897e83a3c877", "tea towel2-2c96921d1cd75d30", "bag of chicken-4abd2b86cd3a1ef6", "packaging of tinfoil-32380bbed975ec15", "piece of foil-15ae1d8f7a8f831d", "piece of chicken-fa548325107b7866", "second piece of chicken-bf92426e9e2cedfe", "spoon-25276264100e174e", "oil spray bottle-65d6cb29f54312d5", "top of pepper shaker-72961afc5f5acb6d", "brush-f58cf76d0ad57a92", "baking tray2-68728c3b25e70f39", "cutting board-1a23d017a8c8d36f", "clip-3a6f7c51cf92a519", "spoon holder-22b5c8e6d26022fe", "bowl2-e556f8124c4a408a", "spinach-064f07846b929b92", "spinach2-682184714cc7ed07", "spinach3-36ffd7dd4dee82c6", "phone charging wire-853bacc7a502d6ce", "phone charger-da52695873e86e3d", "spinach pile-e16976a388a79952", "sleeve-723c9fd65c448082", "scissors2-b4228fe77d62957b", "knife holder-3b775df4b11cbfcd", "knife-94df1ae1bab0a0b4", "slice of pepper-8b4e223d6c7aa3b7", "cherry tomato-37519f26ef2765ec", "cherry tomato2-b3b4073d6a5985d8", "cherry tomato3-627b9ccb2c8e9084", "cucumber piece-5f0decd07bab5d7b", "pieces of cucumber-617a4a2abf6c97ce", "cucumber piece2-e12e1f98962e468b", "new piece of cucumber-a8bfba66a295fec3", "saucepan-6bf9d129dafe826e", "water bottle1-7ab8b701cc3a0f13", "water bottle2-5f07caa9b5a1d83f", "top of water bottle2-7d2db336c5ce1806", "plastic packaging-eafff0d979d39a54", "washing up liquid bottle-f612ae3976312575", "recycling bin-020d43c49fd953e9", "small plate-e72f14c675fdf9a0", "tupperware2-ebb84424cbe23181", "top of water bottle1-9cba2b0f9bb07788", "water bottle3-edf424ba7f54b9c7", "peeler-e1e20e75e64c4a36", "food scraps-d285ba8901800756", "trash can-513ab3aa61889210", "oven mitt-d5b830a450d18126", "tongs-2780bc44c478f786", "lemon-17218cd2009a4c14", "new half of lemon-30e49428d7ae3967", "spoon2-0d4ef6178025765f", "spoon3-b11bb7384fab3e26", "wooden spoon-27489baf0229d2a3", "sponge-2a21e098a4cca24e", "dish soap-1745f720e87221df", "fork-2b0b17b51b8c6f89", "fork2-68891a2cac1d102e", "new coffee filter-7ddfac24e378c3fb", "bowl3-4dbfa119a4920d2f", "spoon4-8fba0cb53557d9df", "disk of air fryer-5e4ce1fd127fcbcb", "knife2-aa04eeaffd80a25b", "fork3-86d5aaea5c6ebbb8", "spoon5-a457392eb9662626", "packaging of sponge-2fadc3e44770bbaa", "sponge2-e9d54376cbc0ce6b", "surface cleaner-070286e45645388f", "coriander packaging-61055eb1d1e2de8b", "coriander-c1a24fbb19a74dc2", "tea towel3-f9c4e66f35e38c15", "knife3-9626293a70777190" ], "reasoning": "1. I watch for the user to retrieve the black scissors (scissors-3065ff742d9267c1) for the first time.\n2. This occurs at 02:20, after the user opens a drawer to find them inside.\n3. Before they are picked up, the black scissors (scissors-3065ff742d9267c1) are on top of the gray scissors (scissors2-b4228fe77d62957b).", "referenced_object_masks": [ "scissors2-b4228fe77d62957b", "scissors-3065ff742d9267c1" ] }, { "key": "P08-20240620-180825:a9428e7db9d36201d40514ba6d317ee208280153", "video_id": "P08-20240620-180825", "question": "What would I encounter if I walked up to the recycling bin, turned left, and took a step towards the corner?", "answer_choice_0": "A trash can", "answer_choice_1": "A washing machine", "answer_choice_2": "A kitty litter box", "answer_choice_3": "The refrigerator", "answer_choice_4": "The back door", "answer_id": 2, "answer": "A kitty litter box", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Counterfactual" ], "objects_of_interest": [ "yellow pepper-babae09606ab4edc", "drawer of refrigerator-17cf55903f4849fb", "tupperware-0a4b3f7f0fae76a5", "cucumber-0973c52bca2fef6e", "baking tray-5e5421fe80e19cb1", "air fryer-b60a6ab79acb3e0d", "plug of air fryer-b5e2882ea27725f8", "bag of spinach-ff6081050ae00655", "plate-257627f16a89b2cb", "kitchen scale-6c2924912db32cf8", "bag of frozen vegetables-8ae5f6f9cceaf41a", "pot-561e9854f36e9f7e", "lid of pot-0817af02743a0337", "bag of rice-0456bd286bc00134", "phone-1dba9f164ad57f40", "measuring cups-6005da177eafde1e", "bowl-77ed94918a5599be", "scissors-3065ff742d9267c1", "tea towel-0f28a0dce6d162c5", "stove dial-a90a8ac2f0bc4804", "salt shaker-6bc8ca30a1847f65", "pepper shaker-7431f67f76b7d559", "garlic seasoning-8a6ab4f37d0e2775", "jar of seasoning-03463ea90de2ac45", "cap of garlic seasoning-1c36de87dc2ffcd6", "piece of broccoli-d033c7bab390c1f3", "second piece of broccoli-fad6d4f3e6e41101", "third piece of broccoli-d7a9f53993881341", "fourth piece of broccoli-41ff48c80f5c92db", "fifth piece of broccoli-04bb1360d3815577", "sixth piece of broccoli-14c5da1ed3142e37", "drawer of air fryer-a613897e83a3c877", "tea towel2-2c96921d1cd75d30", "bag of chicken-4abd2b86cd3a1ef6", "packaging of tinfoil-32380bbed975ec15", "piece of foil-15ae1d8f7a8f831d", "piece of chicken-fa548325107b7866", "second piece of chicken-bf92426e9e2cedfe", "spoon-25276264100e174e", "oil spray bottle-65d6cb29f54312d5", "top of pepper shaker-72961afc5f5acb6d", "brush-f58cf76d0ad57a92", "baking tray2-68728c3b25e70f39", "cutting board-1a23d017a8c8d36f", "clip-3a6f7c51cf92a519", "spoon holder-22b5c8e6d26022fe", "bowl2-e556f8124c4a408a", "spinach-064f07846b929b92", "spinach2-682184714cc7ed07", "spinach3-36ffd7dd4dee82c6", "phone charging wire-853bacc7a502d6ce", "phone charger-da52695873e86e3d", "spinach pile-e16976a388a79952", "sleeve-723c9fd65c448082", "scissors2-b4228fe77d62957b", "knife holder-3b775df4b11cbfcd", "knife-94df1ae1bab0a0b4", "slice of pepper-8b4e223d6c7aa3b7", "cherry tomato-37519f26ef2765ec", "cherry tomato2-b3b4073d6a5985d8", "cherry tomato3-627b9ccb2c8e9084", "cucumber piece-5f0decd07bab5d7b", "pieces of cucumber-617a4a2abf6c97ce", "cucumber piece2-e12e1f98962e468b", "new piece of cucumber-a8bfba66a295fec3", "saucepan-6bf9d129dafe826e", "water bottle1-7ab8b701cc3a0f13", "water bottle2-5f07caa9b5a1d83f", "top of water bottle2-7d2db336c5ce1806", "plastic packaging-eafff0d979d39a54", "washing up liquid bottle-f612ae3976312575", "recycling bin-020d43c49fd953e9", "small plate-e72f14c675fdf9a0", "tupperware2-ebb84424cbe23181", "top of water bottle1-9cba2b0f9bb07788", "water bottle3-edf424ba7f54b9c7", "peeler-e1e20e75e64c4a36", "food scraps-d285ba8901800756", "trash can-513ab3aa61889210", "oven mitt-d5b830a450d18126", "tongs-2780bc44c478f786", "lemon-17218cd2009a4c14", "new half of lemon-30e49428d7ae3967", "spoon2-0d4ef6178025765f", "spoon3-b11bb7384fab3e26", "wooden spoon-27489baf0229d2a3", "sponge-2a21e098a4cca24e", "dish soap-1745f720e87221df", "fork-2b0b17b51b8c6f89", "fork2-68891a2cac1d102e", "new coffee filter-7ddfac24e378c3fb", "bowl3-4dbfa119a4920d2f", "spoon4-8fba0cb53557d9df", "disk of air fryer-5e4ce1fd127fcbcb", "knife2-aa04eeaffd80a25b", "fork3-86d5aaea5c6ebbb8", "spoon5-a457392eb9662626", "packaging of sponge-2fadc3e44770bbaa", "sponge2-e9d54376cbc0ce6b", "surface cleaner-070286e45645388f", "coriander packaging-61055eb1d1e2de8b", "coriander-c1a24fbb19a74dc2", "tea towel3-f9c4e66f35e38c15", "knife3-9626293a70777190" ], "reasoning": "1. I watched the video looking for the recycling bin recycling (bin-020d43c49fd953e9), which the user deposits items into at 16:01.\n2. The user turns to the left at 16:02 to walk back to the ktichen, giving a view of what is to the left in the corner.\n3. If the user had turned to the left and taken a step toward the corner, they would have encountered a kitty litter box .", "referenced_object_masks": [ "bin-020d43c49fd953e9" ] }, { "key": "P08-20240620-180825:ae12567e572e53dbae71bbc57384a6eb3d50cdca", "video_id": "P08-20240620-180825", "question": "After removing an initial section, how many times do I slice the yellow pepper before adjusting the knob of the air fryer, minus the number of time I slice it after adjusting the knob?", "answer_choice_0": "5", "answer_choice_1": "11", "answer_choice_2": "3", "answer_choice_3": "9", "answer_choice_4": "7", "answer_id": 3, "answer": "9", "question_type": [ "Action Recognition", "Numerical Reasoning", "Counting", "Event Occurence" ], "objects_of_interest": [ "yellow pepper-babae09606ab4edc", "drawer of refrigerator-17cf55903f4849fb", "tupperware-0a4b3f7f0fae76a5", "cucumber-0973c52bca2fef6e", "baking tray-5e5421fe80e19cb1", "air fryer-b60a6ab79acb3e0d", "plug of air fryer-b5e2882ea27725f8", "bag of spinach-ff6081050ae00655", "plate-257627f16a89b2cb", "kitchen scale-6c2924912db32cf8", "bag of frozen vegetables-8ae5f6f9cceaf41a", "pot-561e9854f36e9f7e", "lid of pot-0817af02743a0337", "bag of rice-0456bd286bc00134", "phone-1dba9f164ad57f40", "measuring cups-6005da177eafde1e", "bowl-77ed94918a5599be", "scissors-3065ff742d9267c1", "tea towel-0f28a0dce6d162c5", "stove dial-a90a8ac2f0bc4804", "salt shaker-6bc8ca30a1847f65", "pepper shaker-7431f67f76b7d559", "garlic seasoning-8a6ab4f37d0e2775", "jar of seasoning-03463ea90de2ac45", "cap of garlic seasoning-1c36de87dc2ffcd6", "piece of broccoli-d033c7bab390c1f3", "second piece of broccoli-fad6d4f3e6e41101", "third piece of broccoli-d7a9f53993881341", "fourth piece of broccoli-41ff48c80f5c92db", "fifth piece of broccoli-04bb1360d3815577", "sixth piece of broccoli-14c5da1ed3142e37", "drawer of air fryer-a613897e83a3c877", "tea towel2-2c96921d1cd75d30", "bag of chicken-4abd2b86cd3a1ef6", "packaging of tinfoil-32380bbed975ec15", "piece of foil-15ae1d8f7a8f831d", "piece of chicken-fa548325107b7866", "second piece of chicken-bf92426e9e2cedfe", "spoon-25276264100e174e", "oil spray bottle-65d6cb29f54312d5", "top of pepper shaker-72961afc5f5acb6d", "brush-f58cf76d0ad57a92", "baking tray2-68728c3b25e70f39", "cutting board-1a23d017a8c8d36f", "clip-3a6f7c51cf92a519", "spoon holder-22b5c8e6d26022fe", "bowl2-e556f8124c4a408a", "spinach-064f07846b929b92", "spinach2-682184714cc7ed07", "spinach3-36ffd7dd4dee82c6", "phone charging wire-853bacc7a502d6ce", "phone charger-da52695873e86e3d", "spinach pile-e16976a388a79952", "sleeve-723c9fd65c448082", "scissors2-b4228fe77d62957b", "knife holder-3b775df4b11cbfcd", "knife-94df1ae1bab0a0b4", "slice of pepper-8b4e223d6c7aa3b7", "cherry tomato-37519f26ef2765ec", "cherry tomato2-b3b4073d6a5985d8", "cherry tomato3-627b9ccb2c8e9084", "cucumber piece-5f0decd07bab5d7b", "pieces of cucumber-617a4a2abf6c97ce", "cucumber piece2-e12e1f98962e468b", "new piece of cucumber-a8bfba66a295fec3", "saucepan-6bf9d129dafe826e", "water bottle1-7ab8b701cc3a0f13", "water bottle2-5f07caa9b5a1d83f", "top of water bottle2-7d2db336c5ce1806", "plastic packaging-eafff0d979d39a54", "washing up liquid bottle-f612ae3976312575", "recycling bin-020d43c49fd953e9", "small plate-e72f14c675fdf9a0", "tupperware2-ebb84424cbe23181", "top of water bottle1-9cba2b0f9bb07788", "water bottle3-edf424ba7f54b9c7", "peeler-e1e20e75e64c4a36", "food scraps-d285ba8901800756", "trash can-513ab3aa61889210", "oven mitt-d5b830a450d18126", "tongs-2780bc44c478f786", "lemon-17218cd2009a4c14", "new half of lemon-30e49428d7ae3967", "spoon2-0d4ef6178025765f", "spoon3-b11bb7384fab3e26", "wooden spoon-27489baf0229d2a3", "sponge-2a21e098a4cca24e", "dish soap-1745f720e87221df", "fork-2b0b17b51b8c6f89", "fork2-68891a2cac1d102e", "new coffee filter-7ddfac24e378c3fb", "bowl3-4dbfa119a4920d2f", "spoon4-8fba0cb53557d9df", "disk of air fryer-5e4ce1fd127fcbcb", "knife2-aa04eeaffd80a25b", "fork3-86d5aaea5c6ebbb8", "spoon5-a457392eb9662626", "packaging of sponge-2fadc3e44770bbaa", "sponge2-e9d54376cbc0ce6b", "surface cleaner-070286e45645388f", "coriander packaging-61055eb1d1e2de8b", "coriander-c1a24fbb19a74dc2", "tea towel3-f9c4e66f35e38c15", "knife3-9626293a70777190" ], "reasoning": "1. I watch for the user to retrieve the yellow pepper (yellow pepper-babae09606ab4edc), which they take from the refrigerator at 00:06.\n2. The user cuts off a section of the yellow pepper (yellow pepper-babae09606ab4edc) from 12:05-12:15, so I keep watching.\n3. The user begins to slice the yellow pepper (yellow pepper-babae09606ab4edc) at 12:53, slicing 11 times.\n4. At 13:10, the user reaches out with their right hand to adjust the knob of the air fryer (air fryer-b60a6ab79acb3e0d).\n5. At 13:11, the user returns to slicing the yellow pepper (yellow pepper-babae09606ab4edc), and slices twice.\n6. The difference between the two numbers is 9.", "referenced_object_masks": [ "air fryer-b60a6ab79acb3e0d", "yellow pepper-babae09606ab4edc" ] }, { "key": "P08-20240620-180825:e365186a7d637d995327a6f0470b56d04a9934f8", "video_id": "P08-20240620-180825", "question": "When I am standing at the first place I put the baking trays, how many degrees do I have to turn to get to the stove?", "answer_choice_0": "150 degrees", "answer_choice_1": "120 degrees", "answer_choice_2": "90 degrees", "answer_choice_3": "0 degrees", "answer_choice_4": "180 degrees", "answer_id": 3, "answer": "0 degrees", "question_type": [ "Action Recognition", "Event Occurence", "Object Recognition", "Egocentric & Spatial Perception", "Temporal Reasoning", "Situational Awareness" ], "objects_of_interest": [ "yellow pepper-babae09606ab4edc", "drawer of refrigerator-17cf55903f4849fb", "tupperware-0a4b3f7f0fae76a5", "cucumber-0973c52bca2fef6e", "baking tray-5e5421fe80e19cb1", "air fryer-b60a6ab79acb3e0d", "plug of air fryer-b5e2882ea27725f8", "bag of spinach-ff6081050ae00655", "plate-257627f16a89b2cb", "kitchen scale-6c2924912db32cf8", "bag of frozen vegetables-8ae5f6f9cceaf41a", "pot-561e9854f36e9f7e", "lid of pot-0817af02743a0337", "bag of rice-0456bd286bc00134", "phone-1dba9f164ad57f40", "measuring cups-6005da177eafde1e", "bowl-77ed94918a5599be", "scissors-3065ff742d9267c1", "tea towel-0f28a0dce6d162c5", "stove dial-a90a8ac2f0bc4804", "salt shaker-6bc8ca30a1847f65", "pepper shaker-7431f67f76b7d559", "garlic seasoning-8a6ab4f37d0e2775", "jar of seasoning-03463ea90de2ac45", "cap of garlic seasoning-1c36de87dc2ffcd6", "piece of broccoli-d033c7bab390c1f3", "second piece of broccoli-fad6d4f3e6e41101", "third piece of broccoli-d7a9f53993881341", "fourth piece of broccoli-41ff48c80f5c92db", "fifth piece of broccoli-04bb1360d3815577", "sixth piece of broccoli-14c5da1ed3142e37", "drawer of air fryer-a613897e83a3c877", "tea towel2-2c96921d1cd75d30", "bag of chicken-4abd2b86cd3a1ef6", "packaging of tinfoil-32380bbed975ec15", "piece of foil-15ae1d8f7a8f831d", "piece of chicken-fa548325107b7866", "second piece of chicken-bf92426e9e2cedfe", "spoon-25276264100e174e", "oil spray bottle-65d6cb29f54312d5", "top of pepper shaker-72961afc5f5acb6d", "brush-f58cf76d0ad57a92", "baking tray2-68728c3b25e70f39", "cutting board-1a23d017a8c8d36f", "clip-3a6f7c51cf92a519", "spoon holder-22b5c8e6d26022fe", "bowl2-e556f8124c4a408a", "spinach-064f07846b929b92", "spinach2-682184714cc7ed07", "spinach3-36ffd7dd4dee82c6", "phone charging wire-853bacc7a502d6ce", "phone charger-da52695873e86e3d", "spinach pile-e16976a388a79952", "sleeve-723c9fd65c448082", "scissors2-b4228fe77d62957b", "knife holder-3b775df4b11cbfcd", "knife-94df1ae1bab0a0b4", "slice of pepper-8b4e223d6c7aa3b7", "cherry tomato-37519f26ef2765ec", "cherry tomato2-b3b4073d6a5985d8", "cherry tomato3-627b9ccb2c8e9084", "cucumber piece-5f0decd07bab5d7b", "pieces of cucumber-617a4a2abf6c97ce", "cucumber piece2-e12e1f98962e468b", "new piece of cucumber-a8bfba66a295fec3", "saucepan-6bf9d129dafe826e", "water bottle1-7ab8b701cc3a0f13", "water bottle2-5f07caa9b5a1d83f", "top of water bottle2-7d2db336c5ce1806", "plastic packaging-eafff0d979d39a54", "washing up liquid bottle-f612ae3976312575", "recycling bin-020d43c49fd953e9", "small plate-e72f14c675fdf9a0", "tupperware2-ebb84424cbe23181", "top of water bottle1-9cba2b0f9bb07788", "water bottle3-edf424ba7f54b9c7", "peeler-e1e20e75e64c4a36", "food scraps-d285ba8901800756", "trash can-513ab3aa61889210", "oven mitt-d5b830a450d18126", "tongs-2780bc44c478f786", "lemon-17218cd2009a4c14", "new half of lemon-30e49428d7ae3967", "spoon2-0d4ef6178025765f", "spoon3-b11bb7384fab3e26", "wooden spoon-27489baf0229d2a3", "sponge-2a21e098a4cca24e", "dish soap-1745f720e87221df", "fork-2b0b17b51b8c6f89", "fork2-68891a2cac1d102e", "new coffee filter-7ddfac24e378c3fb", "bowl3-4dbfa119a4920d2f", "spoon4-8fba0cb53557d9df", "disk of air fryer-5e4ce1fd127fcbcb", "knife2-aa04eeaffd80a25b", "fork3-86d5aaea5c6ebbb8", "spoon5-a457392eb9662626", "packaging of sponge-2fadc3e44770bbaa", "sponge2-e9d54376cbc0ce6b", "surface cleaner-070286e45645388f", "coriander packaging-61055eb1d1e2de8b", "coriander-c1a24fbb19a74dc2", "tea towel3-f9c4e66f35e38c15", "knife3-9626293a70777190" ], "reasoning": "1. I looked for the first place the user placed the baking trays (baking tray-5e5421fe80e19cb1, baking tray2-68728c3b25e70f39) and saw the user take them out of the oven and place them on top of the stovetop at 00:37.\n2. I realize that since the user has placed the trays (baking tray-5e5421fe80e19cb1, baking tray2-68728c3b25e70f39) at the stovetop, the user wouldn't have to turn at all to find the stove.\n3. Therefore, the answer is 0 degrees.", "referenced_object_masks": [ "baking tray-5e5421fe80e19cb1", "baking tray2-68728c3b25e70f39" ] }, { "key": "P08-20240622-141902:0114ae33b96283adf2f22bdd28952cbbacbaee49", "video_id": "P08-20240622-141902", "question": "What color is complementary to the of combination the color of the dish soap and the color of the leftmost candle of the candelebra?", "answer_choice_0": "purple", "answer_choice_1": "red", "answer_choice_2": "blue", "answer_choice_3": "green", "answer_choice_4": "orange", "answer_id": 1, "answer": "red", "question_type": [ "Counterfactual", "Situational Awareness", "Object Recognition", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "phone-ab28b182626e68fc", "green cover-9fab18d8fcc7cf13", "cucumber-6080177236aa1213", "red pot-001da57a3786cb57", "metal bowl-e664e31cdc52bf82", "espresso cup-7bf1a8d6684c6e98", "scale-b3a11e635a4b187c", "white chopping board-d47e2ed507befc8c", "knife-5263ddd1d58c7852", "entire piece of plastic casing-81cc398cc40127a2", "small piece of cucumber-6d446938758521f1", "small grey plate-963dfa32179d7607", "kitchen towel roll with stand-b85b7935b92267ed", "piece of paper towel-5ea215ea0d4999a1", "spoon-1e4efa53a8e1ccb9", "garlic bulb-d5d2d28fbced78d6", "piece of ginger-476c8deda11714a4", "garlic peel-36a8c06bece0b4c4", "garlic peel2-a2f01559e187c20a", "pieces of garlic-3f10d47ed489caed", "another piece of garlic-81cc05779900e214", "third piece of garlic-6b228109d85e9f82", "knife2-4230254947b1be7e", "larger knife-9f83679954e29dd0", "food scraps-17608dfd1ff9e4e9", "food recycle bin-773712746d9dd34f", "bottle of sauce-cd13ee7327c0b69e", "second bottle of sauce-82989a36a7fceddd", "seasoning jar-899df6197a8f14dc", "another bottle-c70544a0d240341c", "jar1-01b2e3533216d03e", "jar2-a95cca50d33f18fc", "large spoon-ed5d33601bd20c92", "spoon2-9ffea9d56e529f5f", "cap of sesame oil-1631c2df5aa94466", "lid of jar 2-96d852b968e5be51", "lid of jar1-757dd44a1e3ce4dc", "small piece of ginger-ff98dfea8cbdc77d", "grater-417f6ed8ab7a4741", "tea towel-400c5e66724aa07f", "seasoning jar2-2f6cf3209ff2fbd2", "salt seasoning-51c19f10a0bf6ace", "spatula-6f7cae0db67c6d63", "bag of garlic bulbs-404d97491af1fd87", "knife holder-a1500e2cc7d008dc", "sponge-119dd24110ca05b2", "bottle of dish soap-f5391f858262f25a", "piece of towel paper2-7b3b6938769bce94" ], "reasoning": "1. I identified the dish soap (bottle of dish soap-f5391f858262f25a) at 16:45 when the user picks it up. The liquid inside is yellow.\n2. I then identified the candelebra at 05:35 when the user walked to the dining table. The leftmost candle is blue.\n3. The combination of yellow and blue is green, and red is complementary to green.", "referenced_object_masks": [ "bottle of dish soap-f5391f858262f25a" ] }, { "key": "P08-20240622-141902:1299b129a063629d1f6b8739fbb7176790d896d2", "video_id": "P08-20240622-141902", "question": "If I doubled the number of garlic cloves I peel, how many would I have peeled?", "answer_choice_0": "4", "answer_choice_1": "8", "answer_choice_2": "6", "answer_choice_3": "0", "answer_choice_4": "2", "answer_id": 1, "answer": "8", "question_type": [ "Object Recognition", "Counting", "Action Recognition", "Numerical Reasoning", "Counterfactual", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "phone-ab28b182626e68fc", "green cover-9fab18d8fcc7cf13", "cucumber-6080177236aa1213", "red pot-001da57a3786cb57", "metal bowl-e664e31cdc52bf82", "espresso cup-7bf1a8d6684c6e98", "scale-b3a11e635a4b187c", "white chopping board-d47e2ed507befc8c", "knife-5263ddd1d58c7852", "entire piece of plastic casing-81cc398cc40127a2", "small piece of cucumber-6d446938758521f1", "small grey plate-963dfa32179d7607", "kitchen towel roll with stand-b85b7935b92267ed", "piece of paper towel-5ea215ea0d4999a1", "spoon-1e4efa53a8e1ccb9", "garlic bulb-d5d2d28fbced78d6", "piece of ginger-476c8deda11714a4", "garlic peel-36a8c06bece0b4c4", "garlic peel2-a2f01559e187c20a", "pieces of garlic-3f10d47ed489caed", "another piece of garlic-81cc05779900e214", "third piece of garlic-6b228109d85e9f82", "knife2-4230254947b1be7e", "larger knife-9f83679954e29dd0", "food scraps-17608dfd1ff9e4e9", "food recycle bin-773712746d9dd34f", "bottle of sauce-cd13ee7327c0b69e", "second bottle of sauce-82989a36a7fceddd", "seasoning jar-899df6197a8f14dc", "another bottle-c70544a0d240341c", "jar1-01b2e3533216d03e", "jar2-a95cca50d33f18fc", "large spoon-ed5d33601bd20c92", "spoon2-9ffea9d56e529f5f", "cap of sesame oil-1631c2df5aa94466", "lid of jar 2-96d852b968e5be51", "lid of jar1-757dd44a1e3ce4dc", "small piece of ginger-ff98dfea8cbdc77d", "grater-417f6ed8ab7a4741", "tea towel-400c5e66724aa07f", "seasoning jar2-2f6cf3209ff2fbd2", "salt seasoning-51c19f10a0bf6ace", "spatula-6f7cae0db67c6d63", "bag of garlic bulbs-404d97491af1fd87", "knife holder-a1500e2cc7d008dc", "sponge-119dd24110ca05b2", "bottle of dish soap-f5391f858262f25a", "piece of towel paper2-7b3b6938769bce94" ], "reasoning": "1. I watched the video and found a garlic bulb (garlic bulb-d5d2d28fbced78d6) at 05:36.\n2. I counted each clove the user peeled from the garlic bulb (garlic bulb-d5d2d28fbced78d6).\n3. At 05:53-05:57, I saw the user peel the first garlic clove (pieces of garlic-3f10d47ed489caed).\n4. At 06:07, I saw the user peel the second garlic clove (another piece of garlic-81cc05779900e214).\n5. At 06:11-06:14, I saw the user peel the third garlic clove (third piece of garlic-6b228109d85e9f82).\n6. At 06:20-06:24, I saw the user peel the fourth garlic clove .\n7. I calculated 4*2=8.\n8. Therefore, the user doubled the number of garlic cloves they peeled, they would have peeled 8.", "referenced_object_masks": [ "another piece of garlic-81cc05779900e214", "garlic bulb-d5d2d28fbced78d6", "third piece of garlic-6b228109d85e9f82", "pieces of garlic-3f10d47ed489caed" ] }, { "key": "P08-20240622-141902:201c15d104e361e1f9a9890add2eb3ccce7e53cd", "video_id": "P08-20240622-141902", "question": "What was the numerical value of the cucumber's weight?", "answer_choice_0": "30.3", "answer_choice_1": "33.0", "answer_choice_2": "0.3", "answer_choice_3": "3.0", "answer_choice_4": "0.0", "answer_id": 0, "answer": "30.3", "question_type": [ "Object Recognition", "Action Recognition", "Egocentric & Spatial Perception", "Reading" ], "objects_of_interest": [ "phone-ab28b182626e68fc", "green cover-9fab18d8fcc7cf13", "cucumber-6080177236aa1213", "red pot-001da57a3786cb57", "metal bowl-e664e31cdc52bf82", "espresso cup-7bf1a8d6684c6e98", "scale-b3a11e635a4b187c", "white chopping board-d47e2ed507befc8c", "knife-5263ddd1d58c7852", "entire piece of plastic casing-81cc398cc40127a2", "small piece of cucumber-6d446938758521f1", "small grey plate-963dfa32179d7607", "kitchen towel roll with stand-b85b7935b92267ed", "piece of paper towel-5ea215ea0d4999a1", "spoon-1e4efa53a8e1ccb9", "garlic bulb-d5d2d28fbced78d6", "piece of ginger-476c8deda11714a4", "garlic peel-36a8c06bece0b4c4", "garlic peel2-a2f01559e187c20a", "pieces of garlic-3f10d47ed489caed", "another piece of garlic-81cc05779900e214", "third piece of garlic-6b228109d85e9f82", "knife2-4230254947b1be7e", "larger knife-9f83679954e29dd0", "food scraps-17608dfd1ff9e4e9", "food recycle bin-773712746d9dd34f", "bottle of sauce-cd13ee7327c0b69e", "second bottle of sauce-82989a36a7fceddd", "seasoning jar-899df6197a8f14dc", "another bottle-c70544a0d240341c", "jar1-01b2e3533216d03e", "jar2-a95cca50d33f18fc", "large spoon-ed5d33601bd20c92", "spoon2-9ffea9d56e529f5f", "cap of sesame oil-1631c2df5aa94466", "lid of jar 2-96d852b968e5be51", "lid of jar1-757dd44a1e3ce4dc", "small piece of ginger-ff98dfea8cbdc77d", "grater-417f6ed8ab7a4741", "tea towel-400c5e66724aa07f", "seasoning jar2-2f6cf3209ff2fbd2", "salt seasoning-51c19f10a0bf6ace", "spatula-6f7cae0db67c6d63", "bag of garlic bulbs-404d97491af1fd87", "knife holder-a1500e2cc7d008dc", "sponge-119dd24110ca05b2", "bottle of dish soap-f5391f858262f25a", "piece of towel paper2-7b3b6938769bce94" ], "reasoning": "1. I watched the video and found the cucumber (cucumber-6080177236aa1213) at 00:10.\n2. I found the weight scale (scale-b3a11e635a4b187c) at 00:23.\n3. I saw the user weigh the cucumber (cucumber-6080177236aa1213) on the scale (scale-b3a11e635a4b187c) at 01:05.\n4. At 01:05, I saw the reading of the scale (scale-b3a11e635a4b187c) change from 0.0 to 30.3.\n5. Therefore, the numerical value of the weight of the cucumber (cucumber-6080177236aa1213) is 30.3.", "referenced_object_masks": [ "cucumber-6080177236aa1213", "scale-b3a11e635a4b187c" ] }, { "key": "P08-20240622-141902:4b58dd20d084122eac0d43f7d12be4aa338a8f4e", "video_id": "P08-20240622-141902", "question": "Which set contains all of the locations of the tea towel throughout the video in order?", "answer_choice_0": "hanger, the stovetop, the cutting board, the counter, and the oven", "answer_choice_1": "hanger, stovetop, counter, cutting board, oven", "answer_choice_2": "hanger, cutting board, counter, stovetop, oven", "answer_choice_3": "hanger, cutting board, stovetop, counter, oven", "answer_choice_4": "hanger, counter, stovetop, cutting board, oven", "answer_id": 0, "answer": "hanger, the stovetop, the cutting board, the counter, and the oven", "question_type": [ "Situational Awareness", "Temporal Reasoning", "Object Recognition" ], "objects_of_interest": [ "phone-ab28b182626e68fc", "green cover-9fab18d8fcc7cf13", "cucumber-6080177236aa1213", "red pot-001da57a3786cb57", "metal bowl-e664e31cdc52bf82", "espresso cup-7bf1a8d6684c6e98", "scale-b3a11e635a4b187c", "white chopping board-d47e2ed507befc8c", "knife-5263ddd1d58c7852", "entire piece of plastic casing-81cc398cc40127a2", "small piece of cucumber-6d446938758521f1", "small grey plate-963dfa32179d7607", "kitchen towel roll with stand-b85b7935b92267ed", "piece of paper towel-5ea215ea0d4999a1", "spoon-1e4efa53a8e1ccb9", "garlic bulb-d5d2d28fbced78d6", "piece of ginger-476c8deda11714a4", "garlic peel-36a8c06bece0b4c4", "garlic peel2-a2f01559e187c20a", "pieces of garlic-3f10d47ed489caed", "another piece of garlic-81cc05779900e214", "third piece of garlic-6b228109d85e9f82", "knife2-4230254947b1be7e", "larger knife-9f83679954e29dd0", "food scraps-17608dfd1ff9e4e9", "food recycle bin-773712746d9dd34f", "bottle of sauce-cd13ee7327c0b69e", "second bottle of sauce-82989a36a7fceddd", "seasoning jar-899df6197a8f14dc", "another bottle-c70544a0d240341c", "jar1-01b2e3533216d03e", "jar2-a95cca50d33f18fc", "large spoon-ed5d33601bd20c92", "spoon2-9ffea9d56e529f5f", "cap of sesame oil-1631c2df5aa94466", "lid of jar 2-96d852b968e5be51", "lid of jar1-757dd44a1e3ce4dc", "small piece of ginger-ff98dfea8cbdc77d", "grater-417f6ed8ab7a4741", "tea towel-400c5e66724aa07f", "seasoning jar2-2f6cf3209ff2fbd2", "salt seasoning-51c19f10a0bf6ace", "spatula-6f7cae0db67c6d63", "bag of garlic bulbs-404d97491af1fd87", "knife holder-a1500e2cc7d008dc", "sponge-119dd24110ca05b2", "bottle of dish soap-f5391f858262f25a", "piece of towel paper2-7b3b6938769bce94" ], "reasoning": "1. I identified the tea towel (tea towel-400c5e66724aa07f) at 14:34 when the user reaches for it. The initial location of the tea towel (tea towel-400c5e66724aa07f) is on the hanger on the refrigerator.\n2. The user then tosses the tea towel (tea towel-400c5e66724aa07f) onto the stovetop at 14:36.\n3. The tea towel (tea towel-400c5e66724aa07f) remains on the stove until the user grabs it at 16:39 and places it on top of the cutting board (white chopping board-d47e2ed507befc8c) next to the refrigerator at 16:41.\n4. After drying their hands, the user drops the tea towel (tea towel-400c5e66724aa07f) on the counter next to the sink at 18:07.\n5. Lastly, at 18:35, the user folds and hangs the tea towel (tea towel-400c5e66724aa07f) on the handle of the oven, where it remains until the end of the video.\n6. The locations of the tea towel (tea towel-400c5e66724aa07f) in order are the hanger, the stovetop, the cutting board, the counter, and the oven.", "referenced_object_masks": [ "tea towel-400c5e66724aa07f", "white chopping board-d47e2ed507befc8c" ] }, { "key": "P08-20240622-141902:71bc6ad8de367f1ef3be84a79073ca037d369623", "video_id": "P08-20240622-141902", "question": "If I added the code I pressed on the third keyboard to appear on my phone to the weight of the cucumber, what would be the result?", "answer_choice_0": "6433.3", "answer_choice_1": "7333.3", "answer_choice_2": "9436.3", "answer_choice_3": "8333.3", "answer_choice_4": "323.3", "answer_id": 3, "answer": "8333.3", "question_type": [ "Action Recognition", "Situational Awareness", "Reading", "Object Recognition" ], "objects_of_interest": [ "phone-ab28b182626e68fc", "green cover-9fab18d8fcc7cf13", "cucumber-6080177236aa1213", "red pot-001da57a3786cb57", "metal bowl-e664e31cdc52bf82", "espresso cup-7bf1a8d6684c6e98", "scale-b3a11e635a4b187c", "white chopping board-d47e2ed507befc8c", "knife-5263ddd1d58c7852", "entire piece of plastic casing-81cc398cc40127a2", "small piece of cucumber-6d446938758521f1", "small grey plate-963dfa32179d7607", "kitchen towel roll with stand-b85b7935b92267ed", "piece of paper towel-5ea215ea0d4999a1", "spoon-1e4efa53a8e1ccb9", "garlic bulb-d5d2d28fbced78d6", "piece of ginger-476c8deda11714a4", "garlic peel-36a8c06bece0b4c4", "garlic peel2-a2f01559e187c20a", "pieces of garlic-3f10d47ed489caed", "another piece of garlic-81cc05779900e214", "third piece of garlic-6b228109d85e9f82", "knife2-4230254947b1be7e", "larger knife-9f83679954e29dd0", "food scraps-17608dfd1ff9e4e9", "food recycle bin-773712746d9dd34f", "bottle of sauce-cd13ee7327c0b69e", "second bottle of sauce-82989a36a7fceddd", "seasoning jar-899df6197a8f14dc", "another bottle-c70544a0d240341c", "jar1-01b2e3533216d03e", "jar2-a95cca50d33f18fc", "large spoon-ed5d33601bd20c92", "spoon2-9ffea9d56e529f5f", "cap of sesame oil-1631c2df5aa94466", "lid of jar 2-96d852b968e5be51", "lid of jar1-757dd44a1e3ce4dc", "small piece of ginger-ff98dfea8cbdc77d", "grater-417f6ed8ab7a4741", "tea towel-400c5e66724aa07f", "seasoning jar2-2f6cf3209ff2fbd2", "salt seasoning-51c19f10a0bf6ace", "spatula-6f7cae0db67c6d63", "bag of garlic bulbs-404d97491af1fd87", "knife holder-a1500e2cc7d008dc", "sponge-119dd24110ca05b2", "bottle of dish soap-f5391f858262f25a", "piece of towel paper2-7b3b6938769bce94" ], "reasoning": "1. I observed the weight of the cucumber (cucumber-6080177236aa1213) at 01:05 when the user placed the cucumber (cucumber-6080177236aa1213) on the scale (scale-b3a11e635a4b187c). The weight reads 30.3.\n2. The user proceeds to take out their phone (phone-ab28b182626e68fc) at 01:06. I looked for each time a keyboard appeared on the screen, which happens at 01:13, 01:31, and 01:35.\n3. On the third keyboard, at 01:35, the user presses the number 8303.\n4. When added together, 8303 and 30.3 results in 8333.3.", "referenced_object_masks": [ "cucumber-6080177236aa1213", "phone-ab28b182626e68fc", "scale-b3a11e635a4b187c" ] }, { "key": "P08-20240622-141902:733f32d2fa8d73fdb2077b17c3366b80e49e7c76", "video_id": "P08-20240622-141902", "question": "What was the second item I put on the black kitchen countertop?", "answer_choice_0": "A pot", "answer_choice_1": "A bowl", "answer_choice_2": "A cucumber", "answer_choice_3": "A smartphone", "answer_choice_4": "A scale", "answer_id": 2, "answer": "A cucumber", "question_type": [ "Counting", "Action Recognition", "Object Recognition", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "phone-ab28b182626e68fc", "green cover-9fab18d8fcc7cf13", "cucumber-6080177236aa1213", "red pot-001da57a3786cb57", "metal bowl-e664e31cdc52bf82", "espresso cup-7bf1a8d6684c6e98", "scale-b3a11e635a4b187c", "white chopping board-d47e2ed507befc8c", "knife-5263ddd1d58c7852", "entire piece of plastic casing-81cc398cc40127a2", "small piece of cucumber-6d446938758521f1", "small grey plate-963dfa32179d7607", "kitchen towel roll with stand-b85b7935b92267ed", "piece of paper towel-5ea215ea0d4999a1", "spoon-1e4efa53a8e1ccb9", "garlic bulb-d5d2d28fbced78d6", "piece of ginger-476c8deda11714a4", "garlic peel-36a8c06bece0b4c4", "garlic peel2-a2f01559e187c20a", "pieces of garlic-3f10d47ed489caed", "another piece of garlic-81cc05779900e214", "third piece of garlic-6b228109d85e9f82", "knife2-4230254947b1be7e", "larger knife-9f83679954e29dd0", "food scraps-17608dfd1ff9e4e9", "food recycle bin-773712746d9dd34f", "bottle of sauce-cd13ee7327c0b69e", "second bottle of sauce-82989a36a7fceddd", "seasoning jar-899df6197a8f14dc", "another bottle-c70544a0d240341c", "jar1-01b2e3533216d03e", "jar2-a95cca50d33f18fc", "large spoon-ed5d33601bd20c92", "spoon2-9ffea9d56e529f5f", "cap of sesame oil-1631c2df5aa94466", "lid of jar 2-96d852b968e5be51", "lid of jar1-757dd44a1e3ce4dc", "small piece of ginger-ff98dfea8cbdc77d", "grater-417f6ed8ab7a4741", "tea towel-400c5e66724aa07f", "seasoning jar2-2f6cf3209ff2fbd2", "salt seasoning-51c19f10a0bf6ace", "spatula-6f7cae0db67c6d63", "bag of garlic bulbs-404d97491af1fd87", "knife holder-a1500e2cc7d008dc", "sponge-119dd24110ca05b2", "bottle of dish soap-f5391f858262f25a", "piece of towel paper2-7b3b6938769bce94" ], "reasoning": "1. I watched the video and found the black kitchen countertop at 00:01.\n2. I counted each item the user put on the black kitchen countertop.\n3. At 00:01, I saw the user put the first item on the black kitchen countertop, which was a smartphone (phone-ab28b182626e68fc).\n4. At 00:13, I saw the user put the second item on the black kitchen countertop, which was a cucumber (cucumber-6080177236aa1213).\n5. Therefore, the second item the user put on the black kitchen countertop was a cucumber (cucumber-6080177236aa1213).", "referenced_object_masks": [ "cucumber-6080177236aa1213", "phone-ab28b182626e68fc" ] }, { "key": "P08-20240622-141902:73e0394680c20be4787fbbd23efa3b534a4fad88", "video_id": "P08-20240622-141902", "question": "How many different ingredients from the bottom shelf of the cabinet did I add to the cucumber by the time I placed a spoon on the lid of one of the sauce containers?", "answer_choice_0": "4", "answer_choice_1": "3", "answer_choice_2": "5", "answer_choice_3": "1", "answer_choice_4": "2", "answer_id": 4, "answer": "2", "question_type": [ "Counting", "Action Recognition", "Egocentric & Spatial Perception", "Situational Awareness", "Object Recognition", "State Changes" ], "objects_of_interest": [ "phone-ab28b182626e68fc", "green cover-9fab18d8fcc7cf13", "cucumber-6080177236aa1213", "red pot-001da57a3786cb57", "metal bowl-e664e31cdc52bf82", "espresso cup-7bf1a8d6684c6e98", "scale-b3a11e635a4b187c", "white chopping board-d47e2ed507befc8c", "knife-5263ddd1d58c7852", "entire piece of plastic casing-81cc398cc40127a2", "small piece of cucumber-6d446938758521f1", "small grey plate-963dfa32179d7607", "kitchen towel roll with stand-b85b7935b92267ed", "piece of paper towel-5ea215ea0d4999a1", "spoon-1e4efa53a8e1ccb9", "garlic bulb-d5d2d28fbced78d6", "piece of ginger-476c8deda11714a4", "garlic peel-36a8c06bece0b4c4", "garlic peel2-a2f01559e187c20a", "pieces of garlic-3f10d47ed489caed", "another piece of garlic-81cc05779900e214", "third piece of garlic-6b228109d85e9f82", "knife2-4230254947b1be7e", "larger knife-9f83679954e29dd0", "food scraps-17608dfd1ff9e4e9", "food recycle bin-773712746d9dd34f", "bottle of sauce-cd13ee7327c0b69e", "second bottle of sauce-82989a36a7fceddd", "seasoning jar-899df6197a8f14dc", "another bottle-c70544a0d240341c", "jar1-01b2e3533216d03e", "jar2-a95cca50d33f18fc", "large spoon-ed5d33601bd20c92", "spoon2-9ffea9d56e529f5f", "cap of sesame oil-1631c2df5aa94466", "lid of jar 2-96d852b968e5be51", "lid of jar1-757dd44a1e3ce4dc", "small piece of ginger-ff98dfea8cbdc77d", "grater-417f6ed8ab7a4741", "tea towel-400c5e66724aa07f", "seasoning jar2-2f6cf3209ff2fbd2", "salt seasoning-51c19f10a0bf6ace", "spatula-6f7cae0db67c6d63", "bag of garlic bulbs-404d97491af1fd87", "knife holder-a1500e2cc7d008dc", "sponge-119dd24110ca05b2", "bottle of dish soap-f5391f858262f25a", "piece of towel paper2-7b3b6938769bce94" ], "reasoning": "1. I watched the video to identify which ingredients came from the bottom shelf of the cabinet. The user starts to grab ingredients from the cabinet at 08:45, beginning with the first bottle (bottle of sauce-cd13ee7327c0b69e), which is also from the bottom shelf of the cabinet.\n2. The user grabs the second bottle of sauce (second bottle of sauce-82989a36a7fceddd) from the bottom shelf at 08:46. The user grabs a seasoning jar (seasoning jar-899df6197a8f14dc) from the bottom shelf at 08:51. These constitute all of the items from the bottom shelf of the cabinet.\n3. I watched the video until the user began adding ingredients to the cucumber (cucumber-6080177236aa1213), looking out for when the user placed a spoon on a lid.\n4. The user begins adding ingredients at 09:29, beginning with the first bottle (bottle of sauce-cd13ee7327c0b69e). The user adds the second ingredient, which is the second bottle of sauce (second bottle of sauce-82989a36a7fceddd), at 10:42. The user adds the third ingredient at 11:33, but it is not from the bottom shelf. The user adds the fourth ingredient at 12:10, but it is not from the bottom shelf. the user adds the fifth ingredient at 12:23, but it is not from the bottom shelf.\n5. The user places a spoon (spoon2-9ffea9d56e529f5f) on the lid of the fifth ingredient (jar2-a95cca50d33f18fc) at 12:32.\n6. At this time, the user has added 2 ingredients from the bottom shelf of the cabinet.", "referenced_object_masks": [ "bottle of sauce-cd13ee7327c0b69e", "seasoning jar-899df6197a8f14dc", "spoon2-9ffea9d56e529f5f", "second bottle of sauce-82989a36a7fceddd", "cucumber-6080177236aa1213", "jar2-a95cca50d33f18fc" ] }, { "key": "P08-20240622-141902:958ca3a9c03d1bf8b211a5c0814fd109fe048544", "video_id": "P08-20240622-141902", "question": "Which set of directions would lead me to the TV from where I cut the cucumber?", "answer_choice_0": "Turn 180 degrees, walk to the dining table, turn right", "answer_choice_1": "Turn 90 degrees left, walk ahead past the couch", "answer_choice_2": "Turn 90 degrees right, walk ahead past the dining table", "answer_choice_3": "Turn 180 degrees, walk to the couch, turn left", "answer_choice_4": "Turn 180 degrees, walk past the dining table, walk past the couch", "answer_id": 4, "answer": "Turn 180 degrees, walk past the dining table, walk past the couch", "question_type": [ "Counterfactual", "Egocentric & Spatial Perception", "Situational Awareness", "State Changes", "Action Recognition", "Object Recognition" ], "objects_of_interest": [ "phone-ab28b182626e68fc", "green cover-9fab18d8fcc7cf13", "cucumber-6080177236aa1213", "red pot-001da57a3786cb57", "metal bowl-e664e31cdc52bf82", "espresso cup-7bf1a8d6684c6e98", "scale-b3a11e635a4b187c", "white chopping board-d47e2ed507befc8c", "knife-5263ddd1d58c7852", "entire piece of plastic casing-81cc398cc40127a2", "small piece of cucumber-6d446938758521f1", "small grey plate-963dfa32179d7607", "kitchen towel roll with stand-b85b7935b92267ed", "piece of paper towel-5ea215ea0d4999a1", "spoon-1e4efa53a8e1ccb9", "garlic bulb-d5d2d28fbced78d6", "piece of ginger-476c8deda11714a4", "garlic peel-36a8c06bece0b4c4", "garlic peel2-a2f01559e187c20a", "pieces of garlic-3f10d47ed489caed", "another piece of garlic-81cc05779900e214", "third piece of garlic-6b228109d85e9f82", "knife2-4230254947b1be7e", "larger knife-9f83679954e29dd0", "food scraps-17608dfd1ff9e4e9", "food recycle bin-773712746d9dd34f", "bottle of sauce-cd13ee7327c0b69e", "second bottle of sauce-82989a36a7fceddd", "seasoning jar-899df6197a8f14dc", "another bottle-c70544a0d240341c", "jar1-01b2e3533216d03e", "jar2-a95cca50d33f18fc", "large spoon-ed5d33601bd20c92", "spoon2-9ffea9d56e529f5f", "cap of sesame oil-1631c2df5aa94466", "lid of jar 2-96d852b968e5be51", "lid of jar1-757dd44a1e3ce4dc", "small piece of ginger-ff98dfea8cbdc77d", "grater-417f6ed8ab7a4741", "tea towel-400c5e66724aa07f", "seasoning jar2-2f6cf3209ff2fbd2", "salt seasoning-51c19f10a0bf6ace", "spatula-6f7cae0db67c6d63", "bag of garlic bulbs-404d97491af1fd87", "knife holder-a1500e2cc7d008dc", "sponge-119dd24110ca05b2", "bottle of dish soap-f5391f858262f25a", "piece of towel paper2-7b3b6938769bce94" ], "reasoning": "1. The user begins cutting the cucumber (cucumber-6080177236aa1213) at 02:03. At this time, the user stands at the corner of the kitchen counter on the right of the sink.\n2. I continued watching until more of the environment is shown to identify where the TV is. At 05:33, the user turns around to walk toward the dining table.\n3. At 05:34, the TV is visible on the far wall ahead, past the blue couch.\n4. From where the user cut the cucumber (cucumber-6080177236aa1213), the user would have to turn around 180 degrees, walk past the dining table, then walk past the couch ahead to reach the TV.", "referenced_object_masks": [ "cucumber-6080177236aa1213" ] }, { "key": "P08-20240622-141902:a46558dc5635f31b33e176ce4a50713dd5a860ea", "video_id": "P08-20240622-141902", "question": "Where did I move the cucumber to after dicing it?", "answer_choice_0": "Into the refrigerator", "answer_choice_1": "Into a metal bowl", "answer_choice_2": "Onto a countertop", "answer_choice_3": "Onto a cutting board", "answer_choice_4": "Into the kitchen sink", "answer_id": 1, "answer": "Into a metal bowl", "question_type": [ "Object Recognition", "Action Recognition", "Temporal Reasoning", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "phone-ab28b182626e68fc", "green cover-9fab18d8fcc7cf13", "cucumber-6080177236aa1213", "red pot-001da57a3786cb57", "metal bowl-e664e31cdc52bf82", "espresso cup-7bf1a8d6684c6e98", "scale-b3a11e635a4b187c", "white chopping board-d47e2ed507befc8c", "knife-5263ddd1d58c7852", "entire piece of plastic casing-81cc398cc40127a2", "small piece of cucumber-6d446938758521f1", "small grey plate-963dfa32179d7607", "kitchen towel roll with stand-b85b7935b92267ed", "piece of paper towel-5ea215ea0d4999a1", "spoon-1e4efa53a8e1ccb9", "garlic bulb-d5d2d28fbced78d6", "piece of ginger-476c8deda11714a4", "garlic peel-36a8c06bece0b4c4", "garlic peel2-a2f01559e187c20a", "pieces of garlic-3f10d47ed489caed", "another piece of garlic-81cc05779900e214", "third piece of garlic-6b228109d85e9f82", "knife2-4230254947b1be7e", "larger knife-9f83679954e29dd0", "food scraps-17608dfd1ff9e4e9", "food recycle bin-773712746d9dd34f", "bottle of sauce-cd13ee7327c0b69e", "second bottle of sauce-82989a36a7fceddd", "seasoning jar-899df6197a8f14dc", "another bottle-c70544a0d240341c", "jar1-01b2e3533216d03e", "jar2-a95cca50d33f18fc", "large spoon-ed5d33601bd20c92", "spoon2-9ffea9d56e529f5f", "cap of sesame oil-1631c2df5aa94466", "lid of jar 2-96d852b968e5be51", "lid of jar1-757dd44a1e3ce4dc", "small piece of ginger-ff98dfea8cbdc77d", "grater-417f6ed8ab7a4741", "tea towel-400c5e66724aa07f", "seasoning jar2-2f6cf3209ff2fbd2", "salt seasoning-51c19f10a0bf6ace", "spatula-6f7cae0db67c6d63", "bag of garlic bulbs-404d97491af1fd87", "knife holder-a1500e2cc7d008dc", "sponge-119dd24110ca05b2", "bottle of dish soap-f5391f858262f25a", "piece of towel paper2-7b3b6938769bce94" ], "reasoning": "1. I watched the video and found the cucumber (cucumber-6080177236aa1213) at 00:10.\n2. I saw the user dice the cucumber (cucumber-6080177236aa1213) with a knife on a cutting board (white chopping board-d47e2ed507befc8c) at 02:00-02:45.\n3. After the user diced the cucumber (cucumber-6080177236aa1213) at 02:45, I saw the user move it from the cutting board (white chopping board-d47e2ed507befc8c) into a metal bowl (metal bowl-e664e31cdc52bf82) at 02:48.\n4. Therefore, the user moved the cucumber (cucumber-6080177236aa1213) into a metal bowl (metal bowl-e664e31cdc52bf82) after dicing it.", "referenced_object_masks": [ "cucumber-6080177236aa1213", "white chopping board-d47e2ed507befc8c", "metal bowl-e664e31cdc52bf82" ] }, { "key": "P08-20240622-141902:c6046234aec316ad1eb15f03f98025a661db7752", "video_id": "P08-20240622-141902", "question": "What rotational degree and direction would I have had to turn towards in order to look at the paper towel roll if it were in the opposite direction while I was cutting the cucumber?", "answer_choice_0": "135 degrees to the left", "answer_choice_1": "90 degrees to the left", "answer_choice_2": "135 degrees to the right", "answer_choice_3": "90 degrees to the right", "answer_choice_4": "180 degrees behind", "answer_id": 3, "answer": "90 degrees to the right", "question_type": [ "Object Recognition", "Action Recognition", "Temporal Reasoning", "Egocentric & Spatial Perception", "Counterfactual" ], "objects_of_interest": [ "phone-ab28b182626e68fc", "green cover-9fab18d8fcc7cf13", "cucumber-6080177236aa1213", "red pot-001da57a3786cb57", "metal bowl-e664e31cdc52bf82", "espresso cup-7bf1a8d6684c6e98", "scale-b3a11e635a4b187c", "white chopping board-d47e2ed507befc8c", "knife-5263ddd1d58c7852", "entire piece of plastic casing-81cc398cc40127a2", "small piece of cucumber-6d446938758521f1", "small grey plate-963dfa32179d7607", "kitchen towel roll with stand-b85b7935b92267ed", "piece of paper towel-5ea215ea0d4999a1", "spoon-1e4efa53a8e1ccb9", "garlic bulb-d5d2d28fbced78d6", "piece of ginger-476c8deda11714a4", "garlic peel-36a8c06bece0b4c4", "garlic peel2-a2f01559e187c20a", "pieces of garlic-3f10d47ed489caed", "another piece of garlic-81cc05779900e214", "third piece of garlic-6b228109d85e9f82", "knife2-4230254947b1be7e", "larger knife-9f83679954e29dd0", "food scraps-17608dfd1ff9e4e9", "food recycle bin-773712746d9dd34f", "bottle of sauce-cd13ee7327c0b69e", "second bottle of sauce-82989a36a7fceddd", "seasoning jar-899df6197a8f14dc", "another bottle-c70544a0d240341c", "jar1-01b2e3533216d03e", "jar2-a95cca50d33f18fc", "large spoon-ed5d33601bd20c92", "spoon2-9ffea9d56e529f5f", "cap of sesame oil-1631c2df5aa94466", "lid of jar 2-96d852b968e5be51", "lid of jar1-757dd44a1e3ce4dc", "small piece of ginger-ff98dfea8cbdc77d", "grater-417f6ed8ab7a4741", "tea towel-400c5e66724aa07f", "seasoning jar2-2f6cf3209ff2fbd2", "salt seasoning-51c19f10a0bf6ace", "spatula-6f7cae0db67c6d63", "bag of garlic bulbs-404d97491af1fd87", "knife holder-a1500e2cc7d008dc", "sponge-119dd24110ca05b2", "bottle of dish soap-f5391f858262f25a", "piece of towel paper2-7b3b6938769bce94" ], "reasoning": "1. I watched the video from the beginning and found the cucumber (cucumber-6080177236aa1213) appearing for the first time at 00:10.\n2. From 00:37 to 1:04, the user removes the plastic wrapper (entire piece of plastic casing-81cc398cc40127a2) from the cucumber (cucumber-6080177236aa1213), cuts the tip of each end (OBJ, small piece of cucumber-6d446938758521f1), and places it on the food scale (scale-b3a11e635a4b187c) to weigh it. \n3. At 01:42, the user walks toward the sink to wash the cucumber (cucumber-6080177236aa1213). I saw the paper towel roll (kitchen towel roll with stand-b85b7935b92267ed) in front of the user, diagonally to the left, from 01:47-01:58. \n4. At 01:58, the user turns 90 degrees to the right to walk towards the cutting board (white chopping board-d47e2ed507befc8c), and starts cutting the cucumber at the 02:00 mark. \n5. Since the user turned 90 degrees to the right to move from the paper towel roll (kitchen towel roll with stand-b85b7935b92267ed) to the cutting board (white chopping board-d47e2ed507befc8c) at 01:58, the user consequently would have had to turn 90 degrees to the left to look at the paper towel roll (kitchen towel roll with stand-b85b7935b92267ed) while cutting the cucumber (cucumber-6080177236aa1213).\n6. The opposite of 90 degrees to the left is 90 degrees to the right.\n7. Therefore, the user would have had to turn 90 degrees to the right to look at the paper towel roll (kitchen towel roll with stand-b85b7935b92267ed) if it were in the opposite direction while they were cutting the cucumber (cucumber-6080177236aa1213).", "referenced_object_masks": [ "scale-b3a11e635a4b187c", "small piece of cucumber-6d446938758521f1", "entire piece of plastic casing-81cc398cc40127a2", "cucumber-6080177236aa1213", "kitchen towel roll with stand-b85b7935b92267ed", "white chopping board-d47e2ed507befc8c" ] }, { "key": "P09-20240621-093545:1d21400731dbce5a89ccc88cdfabbd3409ce2730", "video_id": "P09-20240621-093545", "question": "If I had weighed all 4 chicken breasts on the scale, what would the approximate numerical value of its combined weight have been?", "answer_choice_0": "31", "answer_choice_1": "62", "answer_choice_2": "46", "answer_choice_3": "60", "answer_choice_4": "30", "answer_id": 1, "answer": "62", "question_type": [ "Object Recognition", "Counting", "Action Recognition", "State Changes", "Reading", "Numerical Reasoning", "Counterfactual" ], "objects_of_interest": [ "yoghurt-65ad708d1cda5542", "small bowl of yogurt-2da04b4f9bf804c9", "pack of chicken-b8790276dc4edf19", "jar-ce984a52f7841857", "knife-b8e0ec4b5c6bd6ea", "bottle of washing up liquid-fc76e4531e81ca3d", "sponge-0ceb1670caafdfdb", "knife block-b5bf07daf90b2db3", "knife2-b457565b7bc67648", "white chopping board-1c6798257b4130f8", "chicken breast1-cf2c5d7c76f9f9e6", "chicken breast2-8e44a9c6cfec4198", "bowl-4ac981522f0c93c7", "hand towels-5c1d4a41dc268f57", "another hand towel-4160938a0d7d84bb", "hand towels2-ca4d7c40e6e75128", "measuring scale-6d8ca5c698a32767", "washed chickens-490cbd4a97384842", "washed chicken 2-582a187efb5804d8", "washed chicken 1-8922ccce16c077a4", "chicken breast piece-91d97e2f2c98b33a", "another chicken breast-7142d4f277b0799a", "chicken breast packet-76a185c93d44530c", "pack of cling film-5ec93b8161fdf484", "pack of cling film2-21a4420f5e094ca3", "package of cling film3-c149b09ac54c7d8b", "roll of freezer bags-5b10923e683b6a6a", "one freezer bag-c099b4767049f7cc", "another freezer bag-eed75d0dd3f0f980", "retrieved chicken breast1-59cca663cfa30c90", "second chicken breast-f719084be9673a86", "chicken breast chunk-4d2e0317e7e5cb84", "slices of cut chicken breast-cbd8f3bf66b11255", "slices of cut chicken breast1-c1560616b3521f87", "slices of cut chicken breast2-db5ee47a2aab094a", "slices of cut chicken breast3-b1ce1cb8f30e28b2", "lid-9a6adc3900db92f8", "pot cover-708f95a399f58163", "pot-457ffd4355e76336", "bottle of oil-609a6a6968c52a9a", "bottle of salt-6311ffe14b3e81e7", "cheese-4f74916c93d23b4b", "red curry paste-02ae600e3f946f70", "courgette-020e1378635e95a1", "bag of sugar-0e59174cfb192136", "tin coconut milk-6bfb9ce76128eebb", "tin of ghee-56d53c3e3e5d107b", "tin of bamboo shoot-5e12e3317a5fcd03", "courgettes2-3ebd5032dc3ea9c1", "pack of chilli-79b2afbaf9346e1b", "plantains-c691f08c29d039c0", "courgette3-99e8103d31f0f077", "courgettes together-ce68bcd45ab2bbdf", "bowl2-e591e857e8ef4972", "sliced courgette-f71e53eaa4568958", "sliced courgette2-143d9d1f0ae18d44", "fork-133ad7eebaa13290", "plate2-4c521b7283aeb2e3", "bowl3-5079d1f8fffa10df", "colander-5c766dede154b009", "some chillies-5bde19241755ea5f", "bowl4-7ecae136566af053", "spoon-297e798483a73e1c", "lid of curry paste-7c0bdd8329c7d268", "curry paste bag-55e980f49ba4f445", "can opener-901d3a839f5234eb", "kettle-59226e425988a1b4", "some vegetables-d5d0e9eea71474c8", "green bowl-2cf737ee66d122a9", "another chilli-ae93cb958cb2466a", "bin-cf41357d5d16801d", "hand-bb426c14cd0833c6", "tupperware lid-a0c8b567e4eaa15a", "tupperware-744f140a0c860332", "bowl5-202bd59f460a5d38", "hot chocolate whisk-9538e6e1f8fbdf7e", "chocolate machine-1c8a9697b79953bb", "basil plant-e61521c12f3074dc", "some chillies2-2246ccb232aa24cd", "pouch of chillies-25de5b01414e3b82", "tablespoon-7fd55c9342415a7e", "ceramic tumblers-a3b1032bcc9d7f55", "cup-0228c30b8d90c236", "closed tupperware-f00534c2b54ad11e", "rag-3f53d8014bb15ed0", "lunch bag-775abbb72f1f7677", "bag of microwave rice-baab17aa73970e45", "plate3-713696f2ea77f1c4", "tuuperware3-ed95484fc33fb950", "serving spoon-780a719423598113", "tupperware cover-d961f49932f43bd8", "sponge2-db23d2c789ec7e19", "kitchen rag-a6caebb05411dcc4", "towel-054b5bae99ecca01" ], "reasoning": "1. I watched the video and found the pack of chicken breasts (pack of chicken-b8790276dc4edf19) at 00:22.\n2. I counted 4 chicken breasts inside the pack (pack of chicken-b8790276dc4edf19) at 01:04.\n3. I counted 2 chicken breasts (chicken breast1-cf2c5d7c76f9f9e6, chicken breast2-8e44a9c6cfec4198) the user removed from the pack (pack of chicken-b8790276dc4edf19) at 01:04.\n4. I found the scale (measuring scale-6d8ca5c698a32767) at 01:58.\n5. I saw the user weigh the 2 chicken breasts (chicken breast1-cf2c5d7c76f9f9e6, chicken breast2-8e44a9c6cfec4198) together on the scale (measuring scale-6d8ca5c698a32767) at 02:30.\n6. I saw the reading of the scale (measuring scale-6d8ca5c698a32767) at 02:33, which read \"30.8\".\n7. I calculated 30.8*2=61.6, which approximates to 62.\n8. Therefore, if the user had weighed all 4 chicken breasts on the scale (measuring scale-6d8ca5c698a32767), the approximate numerical value of its combined weight would have been 62.", "referenced_object_masks": [ "measuring scale-6d8ca5c698a32767", "pack of chicken-b8790276dc4edf19", "chicken breast1-cf2c5d7c76f9f9e6", "chicken breast2-8e44a9c6cfec4198" ] }, { "key": "P09-20240621-093545:248b1f8af8fea3068f754160102e36ef419e0dfa", "video_id": "P09-20240621-093545", "question": "How many items do I place out on the cupboard and leave out of the cupboard after looking through it?", "answer_choice_0": "2", "answer_choice_1": "4", "answer_choice_2": "5", "answer_choice_3": "1", "answer_choice_4": "3", "answer_id": 0, "answer": "2", "question_type": [ "Object Recognition", "Counting", "Action Recognition" ], "objects_of_interest": [ "yoghurt-65ad708d1cda5542", "small bowl of yogurt-2da04b4f9bf804c9", "pack of chicken-b8790276dc4edf19", "jar-ce984a52f7841857", "knife-b8e0ec4b5c6bd6ea", "bottle of washing up liquid-fc76e4531e81ca3d", "sponge-0ceb1670caafdfdb", "knife block-b5bf07daf90b2db3", "knife2-b457565b7bc67648", "white chopping board-1c6798257b4130f8", "chicken breast1-cf2c5d7c76f9f9e6", "chicken breast2-8e44a9c6cfec4198", "bowl-4ac981522f0c93c7", "hand towels-5c1d4a41dc268f57", "another hand towel-4160938a0d7d84bb", "hand towels2-ca4d7c40e6e75128", "measuring scale-6d8ca5c698a32767", "washed chickens-490cbd4a97384842", "washed chicken 2-582a187efb5804d8", "washed chicken 1-8922ccce16c077a4", "chicken breast piece-91d97e2f2c98b33a", "another chicken breast-7142d4f277b0799a", "chicken breast packet-76a185c93d44530c", "pack of cling film-5ec93b8161fdf484", "pack of cling film2-21a4420f5e094ca3", "package of cling film3-c149b09ac54c7d8b", "roll of freezer bags-5b10923e683b6a6a", "one freezer bag-c099b4767049f7cc", "another freezer bag-eed75d0dd3f0f980", "retrieved chicken breast1-59cca663cfa30c90", "second chicken breast-f719084be9673a86", "chicken breast chunk-4d2e0317e7e5cb84", "slices of cut chicken breast-cbd8f3bf66b11255", "slices of cut chicken breast1-c1560616b3521f87", "slices of cut chicken breast2-db5ee47a2aab094a", "slices of cut chicken breast3-b1ce1cb8f30e28b2", "lid-9a6adc3900db92f8", "pot cover-708f95a399f58163", "pot-457ffd4355e76336", "bottle of oil-609a6a6968c52a9a", "bottle of salt-6311ffe14b3e81e7", "cheese-4f74916c93d23b4b", "red curry paste-02ae600e3f946f70", "courgette-020e1378635e95a1", "bag of sugar-0e59174cfb192136", "tin coconut milk-6bfb9ce76128eebb", "tin of ghee-56d53c3e3e5d107b", "tin of bamboo shoot-5e12e3317a5fcd03", "courgettes2-3ebd5032dc3ea9c1", "pack of chilli-79b2afbaf9346e1b", "plantains-c691f08c29d039c0", "courgette3-99e8103d31f0f077", "courgettes together-ce68bcd45ab2bbdf", "bowl2-e591e857e8ef4972", "sliced courgette-f71e53eaa4568958", "sliced courgette2-143d9d1f0ae18d44", "fork-133ad7eebaa13290", "plate2-4c521b7283aeb2e3", "bowl3-5079d1f8fffa10df", "colander-5c766dede154b009", "some chillies-5bde19241755ea5f", "bowl4-7ecae136566af053", "spoon-297e798483a73e1c", "lid of curry paste-7c0bdd8329c7d268", "curry paste bag-55e980f49ba4f445", "can opener-901d3a839f5234eb", "kettle-59226e425988a1b4", "some vegetables-d5d0e9eea71474c8", "green bowl-2cf737ee66d122a9", "another chilli-ae93cb958cb2466a", "bin-cf41357d5d16801d", "hand-bb426c14cd0833c6", "tupperware lid-a0c8b567e4eaa15a", "tupperware-744f140a0c860332", "bowl5-202bd59f460a5d38", "hot chocolate whisk-9538e6e1f8fbdf7e", "chocolate machine-1c8a9697b79953bb", "basil plant-e61521c12f3074dc", "some chillies2-2246ccb232aa24cd", "pouch of chillies-25de5b01414e3b82", "tablespoon-7fd55c9342415a7e", "ceramic tumblers-a3b1032bcc9d7f55", "cup-0228c30b8d90c236", "closed tupperware-f00534c2b54ad11e", "rag-3f53d8014bb15ed0", "lunch bag-775abbb72f1f7677", "bag of microwave rice-baab17aa73970e45", "plate3-713696f2ea77f1c4", "tuuperware3-ed95484fc33fb950", "serving spoon-780a719423598113", "tupperware cover-d961f49932f43bd8", "sponge2-db23d2c789ec7e19", "kitchen rag-a6caebb05411dcc4", "towel-054b5bae99ecca01" ], "reasoning": "1. I watched for the user to remove items from a cupboard.\n2. I saw the user open the cupboard beneath the counter at 36.43.\n3. I saw the user take out the first item, a cup (cup-0228c30b8d90c236) at 36:54. \n4. I saw the user take out a tupperware lid at 36:56.\n5. I saw the user hold in their left hand a tupperware container at 36:56. \n6. I saw the user take out another tupperware lid at 36:58.\n7. I saw the user close the tupperware with the lid at 37:07.\n8.I saw the user put the closed tupperware on the counter at 37:08\n9. I saw the user put one tupperware lid back into the cupboard at 37:09.\n10. I saw the user put the cup (cup-0228c30b8d90c236) back into the cupboard at 37:13.\n11. I saw the user close the cupboard door at 37:14. \n12. I saw the user leave the closed tupperware (closed tupperware-f00534c2b54ad11e) out at 37:15.\n13. The only item taken out and left out is the closed tupperware (closed tupperware-f00534c2b54ad11e), which consitutes two separate items.", "referenced_object_masks": [ "cup-0228c30b8d90c236", "closed tupperware-f00534c2b54ad11e" ] }, { "key": "P09-20240621-093545:45a3ffee3a37e92afb86c226bb57b5a6f5ae184c", "video_id": "P09-20240621-093545", "question": "How many times do I clank the fork on the bowl, after the can of white cream is placed on the scale?", "answer_choice_0": "5", "answer_choice_1": "4", "answer_choice_2": "1", "answer_choice_3": "3", "answer_choice_4": "2", "answer_id": 4, "answer": "2", "question_type": [ "Counting", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "yoghurt-65ad708d1cda5542", "small bowl of yogurt-2da04b4f9bf804c9", "pack of chicken-b8790276dc4edf19", "jar-ce984a52f7841857", "knife-b8e0ec4b5c6bd6ea", "bottle of washing up liquid-fc76e4531e81ca3d", "sponge-0ceb1670caafdfdb", "knife block-b5bf07daf90b2db3", "knife2-b457565b7bc67648", "white chopping board-1c6798257b4130f8", "chicken breast1-cf2c5d7c76f9f9e6", "chicken breast2-8e44a9c6cfec4198", "bowl-4ac981522f0c93c7", "hand towels-5c1d4a41dc268f57", "another hand towel-4160938a0d7d84bb", "hand towels2-ca4d7c40e6e75128", "measuring scale-6d8ca5c698a32767", "washed chickens-490cbd4a97384842", "washed chicken 2-582a187efb5804d8", "washed chicken 1-8922ccce16c077a4", "chicken breast piece-91d97e2f2c98b33a", "another chicken breast-7142d4f277b0799a", "chicken breast packet-76a185c93d44530c", "pack of cling film-5ec93b8161fdf484", "pack of cling film2-21a4420f5e094ca3", "package of cling film3-c149b09ac54c7d8b", "roll of freezer bags-5b10923e683b6a6a", "one freezer bag-c099b4767049f7cc", "another freezer bag-eed75d0dd3f0f980", "retrieved chicken breast1-59cca663cfa30c90", "second chicken breast-f719084be9673a86", "chicken breast chunk-4d2e0317e7e5cb84", "slices of cut chicken breast-cbd8f3bf66b11255", "slices of cut chicken breast1-c1560616b3521f87", "slices of cut chicken breast2-db5ee47a2aab094a", "slices of cut chicken breast3-b1ce1cb8f30e28b2", "lid-9a6adc3900db92f8", "pot cover-708f95a399f58163", "pot-457ffd4355e76336", "bottle of oil-609a6a6968c52a9a", "bottle of salt-6311ffe14b3e81e7", "cheese-4f74916c93d23b4b", "red curry paste-02ae600e3f946f70", "courgette-020e1378635e95a1", "bag of sugar-0e59174cfb192136", "tin coconut milk-6bfb9ce76128eebb", "tin of ghee-56d53c3e3e5d107b", "tin of bamboo shoot-5e12e3317a5fcd03", "courgettes2-3ebd5032dc3ea9c1", "pack of chilli-79b2afbaf9346e1b", "plantains-c691f08c29d039c0", "courgette3-99e8103d31f0f077", "courgettes together-ce68bcd45ab2bbdf", "bowl2-e591e857e8ef4972", "sliced courgette-f71e53eaa4568958", "sliced courgette2-143d9d1f0ae18d44", "fork-133ad7eebaa13290", "plate2-4c521b7283aeb2e3", "bowl3-5079d1f8fffa10df", "colander-5c766dede154b009", "some chillies-5bde19241755ea5f", "bowl4-7ecae136566af053", "spoon-297e798483a73e1c", "lid of curry paste-7c0bdd8329c7d268", "curry paste bag-55e980f49ba4f445", "can opener-901d3a839f5234eb", "kettle-59226e425988a1b4", "some vegetables-d5d0e9eea71474c8", "green bowl-2cf737ee66d122a9", "another chilli-ae93cb958cb2466a", "bin-cf41357d5d16801d", "hand-bb426c14cd0833c6", "tupperware lid-a0c8b567e4eaa15a", "tupperware-744f140a0c860332", "bowl5-202bd59f460a5d38", "hot chocolate whisk-9538e6e1f8fbdf7e", "chocolate machine-1c8a9697b79953bb", "basil plant-e61521c12f3074dc", "some chillies2-2246ccb232aa24cd", "pouch of chillies-25de5b01414e3b82", "tablespoon-7fd55c9342415a7e", "ceramic tumblers-a3b1032bcc9d7f55", "cup-0228c30b8d90c236", "closed tupperware-f00534c2b54ad11e", "rag-3f53d8014bb15ed0", "lunch bag-775abbb72f1f7677", "bag of microwave rice-baab17aa73970e45", "plate3-713696f2ea77f1c4", "tuuperware3-ed95484fc33fb950", "serving spoon-780a719423598113", "tupperware cover-d961f49932f43bd8", "sponge2-db23d2c789ec7e19", "kitchen rag-a6caebb05411dcc4", "towel-054b5bae99ecca01" ], "reasoning": "1. I saw the user set down the can of cream on the scale (scale-6d8ca5c698a32767) at 18:43.\n2. I saw the user clank the fork on the side of bowl (bowl2-e591e857e8ef4972)) once at 18:43.\n3. I saw the user clank the fork a second time on the side of the bowl bowl2-e591e857e8ef4972) at 18:44.", "referenced_object_masks": [ "bowl2-e591e857e8ef4972", "scale-6d8ca5c698a32767" ] }, { "key": "P09-20240621-093545:4af426c49a75a39a79a75c7b7ae7944f72703742", "video_id": "P09-20240621-093545", "question": "Where did I get the second item I put on the kitchen countertop from?", "answer_choice_0": "A knife holder", "answer_choice_1": "The kitchen sink", "answer_choice_2": "A countertop", "answer_choice_3": "The dishwasher", "answer_choice_4": "The refrigerator", "answer_id": 0, "answer": "A knife holder", "question_type": [ "Situational Awareness", "Counting", "Object Recognition", "Action Recognition", "State Changes", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "yoghurt-65ad708d1cda5542", "small bowl of yogurt-2da04b4f9bf804c9", "pack of chicken-b8790276dc4edf19", "jar-ce984a52f7841857", "knife-b8e0ec4b5c6bd6ea", "bottle of washing up liquid-fc76e4531e81ca3d", "sponge-0ceb1670caafdfdb", "knife block-b5bf07daf90b2db3", "knife2-b457565b7bc67648", "white chopping board-1c6798257b4130f8", "chicken breast1-cf2c5d7c76f9f9e6", "chicken breast2-8e44a9c6cfec4198", "bowl-4ac981522f0c93c7", "hand towels-5c1d4a41dc268f57", "another hand towel-4160938a0d7d84bb", "hand towels2-ca4d7c40e6e75128", "measuring scale-6d8ca5c698a32767", "washed chickens-490cbd4a97384842", "washed chicken 2-582a187efb5804d8", "washed chicken 1-8922ccce16c077a4", "chicken breast piece-91d97e2f2c98b33a", "another chicken breast-7142d4f277b0799a", "chicken breast packet-76a185c93d44530c", "pack of cling film-5ec93b8161fdf484", "pack of cling film2-21a4420f5e094ca3", "package of cling film3-c149b09ac54c7d8b", "roll of freezer bags-5b10923e683b6a6a", "one freezer bag-c099b4767049f7cc", "another freezer bag-eed75d0dd3f0f980", "retrieved chicken breast1-59cca663cfa30c90", "second chicken breast-f719084be9673a86", "chicken breast chunk-4d2e0317e7e5cb84", "slices of cut chicken breast-cbd8f3bf66b11255", "slices of cut chicken breast1-c1560616b3521f87", "slices of cut chicken breast2-db5ee47a2aab094a", "slices of cut chicken breast3-b1ce1cb8f30e28b2", "lid-9a6adc3900db92f8", "pot cover-708f95a399f58163", "pot-457ffd4355e76336", "bottle of oil-609a6a6968c52a9a", "bottle of salt-6311ffe14b3e81e7", "cheese-4f74916c93d23b4b", "red curry paste-02ae600e3f946f70", "courgette-020e1378635e95a1", "bag of sugar-0e59174cfb192136", "tin coconut milk-6bfb9ce76128eebb", "tin of ghee-56d53c3e3e5d107b", "tin of bamboo shoot-5e12e3317a5fcd03", "courgettes2-3ebd5032dc3ea9c1", "pack of chilli-79b2afbaf9346e1b", "plantains-c691f08c29d039c0", "courgette3-99e8103d31f0f077", "courgettes together-ce68bcd45ab2bbdf", "bowl2-e591e857e8ef4972", "sliced courgette-f71e53eaa4568958", "sliced courgette2-143d9d1f0ae18d44", "fork-133ad7eebaa13290", "plate2-4c521b7283aeb2e3", "bowl3-5079d1f8fffa10df", "colander-5c766dede154b009", "some chillies-5bde19241755ea5f", "bowl4-7ecae136566af053", "spoon-297e798483a73e1c", "lid of curry paste-7c0bdd8329c7d268", "curry paste bag-55e980f49ba4f445", "can opener-901d3a839f5234eb", "kettle-59226e425988a1b4", "some vegetables-d5d0e9eea71474c8", "green bowl-2cf737ee66d122a9", "another chilli-ae93cb958cb2466a", "bin-cf41357d5d16801d", "hand-bb426c14cd0833c6", "tupperware lid-a0c8b567e4eaa15a", "tupperware-744f140a0c860332", "bowl5-202bd59f460a5d38", "hot chocolate whisk-9538e6e1f8fbdf7e", "chocolate machine-1c8a9697b79953bb", "basil plant-e61521c12f3074dc", "some chillies2-2246ccb232aa24cd", "pouch of chillies-25de5b01414e3b82", "tablespoon-7fd55c9342415a7e", "ceramic tumblers-a3b1032bcc9d7f55", "cup-0228c30b8d90c236", "closed tupperware-f00534c2b54ad11e", "rag-3f53d8014bb15ed0", "lunch bag-775abbb72f1f7677", "bag of microwave rice-baab17aa73970e45", "plate3-713696f2ea77f1c4", "tuuperware3-ed95484fc33fb950", "serving spoon-780a719423598113", "tupperware cover-d961f49932f43bd8", "sponge2-db23d2c789ec7e19", "kitchen rag-a6caebb05411dcc4", "towel-054b5bae99ecca01" ], "reasoning": "1. I watched the video and found the kitchen countertop at 00:33.\n2. I counted and identified each item the user put on the kitchen countertop.\n3. I saw the user put the first item on the kitchen countertop at 00:35, which was a pack of chicken breasts (pack of chicken-b8790276dc4edf19).\n4. I saw the user put the second item on the kitchen countertop at 00:47, which was a small steak knife (knife-b8e0ec4b5c6bd6ea).\n5. Before 00:47, I saw the user get the small steak knife (knife-b8e0ec4b5c6bd6ea) from a knife holder at 00:35.\n6. Therefore, the user got the second item they put on the kitchen countertop from a knife holder .", "referenced_object_masks": [ "pack of chicken-b8790276dc4edf19", "knife-b8e0ec4b5c6bd6ea" ] }, { "key": "P09-20240621-093545:522d9758454f954ad80652d9ab6b6159b7180826", "video_id": "P09-20240621-093545", "question": "After I pull out the plate from underneath the cabinet under the microwave, what do I stir the food in the pot with?", "answer_choice_0": "Knife", "answer_choice_1": "Spoon", "answer_choice_2": "Fork", "answer_choice_3": "Finger", "answer_choice_4": "Spatula", "answer_id": 2, "answer": "Fork", "question_type": [ "Action Recognition", "Situational Awareness", "Object Recognition" ], "objects_of_interest": [ "yoghurt-65ad708d1cda5542", "small bowl of yogurt-2da04b4f9bf804c9", "pack of chicken-b8790276dc4edf19", "jar-ce984a52f7841857", "knife-b8e0ec4b5c6bd6ea", "bottle of washing up liquid-fc76e4531e81ca3d", "sponge-0ceb1670caafdfdb", "knife block-b5bf07daf90b2db3", "knife2-b457565b7bc67648", "white chopping board-1c6798257b4130f8", "chicken breast1-cf2c5d7c76f9f9e6", "chicken breast2-8e44a9c6cfec4198", "bowl-4ac981522f0c93c7", "hand towels-5c1d4a41dc268f57", "another hand towel-4160938a0d7d84bb", "hand towels2-ca4d7c40e6e75128", "measuring scale-6d8ca5c698a32767", "washed chickens-490cbd4a97384842", "washed chicken 2-582a187efb5804d8", "washed chicken 1-8922ccce16c077a4", "chicken breast piece-91d97e2f2c98b33a", "another chicken breast-7142d4f277b0799a", "chicken breast packet-76a185c93d44530c", "pack of cling film-5ec93b8161fdf484", "pack of cling film2-21a4420f5e094ca3", "package of cling film3-c149b09ac54c7d8b", "roll of freezer bags-5b10923e683b6a6a", "one freezer bag-c099b4767049f7cc", "another freezer bag-eed75d0dd3f0f980", "retrieved chicken breast1-59cca663cfa30c90", "second chicken breast-f719084be9673a86", "chicken breast chunk-4d2e0317e7e5cb84", "slices of cut chicken breast-cbd8f3bf66b11255", "slices of cut chicken breast1-c1560616b3521f87", "slices of cut chicken breast2-db5ee47a2aab094a", "slices of cut chicken breast3-b1ce1cb8f30e28b2", "lid-9a6adc3900db92f8", "pot cover-708f95a399f58163", "pot-457ffd4355e76336", "bottle of oil-609a6a6968c52a9a", "bottle of salt-6311ffe14b3e81e7", "cheese-4f74916c93d23b4b", "red curry paste-02ae600e3f946f70", "courgette-020e1378635e95a1", "bag of sugar-0e59174cfb192136", "tin coconut milk-6bfb9ce76128eebb", "tin of ghee-56d53c3e3e5d107b", "tin of bamboo shoot-5e12e3317a5fcd03", "courgettes2-3ebd5032dc3ea9c1", "pack of chilli-79b2afbaf9346e1b", "plantains-c691f08c29d039c0", "courgette3-99e8103d31f0f077", "courgettes together-ce68bcd45ab2bbdf", "bowl2-e591e857e8ef4972", "sliced courgette-f71e53eaa4568958", "sliced courgette2-143d9d1f0ae18d44", "fork-133ad7eebaa13290", "plate2-4c521b7283aeb2e3", "bowl3-5079d1f8fffa10df", "colander-5c766dede154b009", "some chillies-5bde19241755ea5f", "bowl4-7ecae136566af053", "spoon-297e798483a73e1c", "lid of curry paste-7c0bdd8329c7d268", "curry paste bag-55e980f49ba4f445", "can opener-901d3a839f5234eb", "kettle-59226e425988a1b4", "some vegetables-d5d0e9eea71474c8", "green bowl-2cf737ee66d122a9", "another chilli-ae93cb958cb2466a", "bin-cf41357d5d16801d", "hand-bb426c14cd0833c6", "tupperware lid-a0c8b567e4eaa15a", "tupperware-744f140a0c860332", "bowl5-202bd59f460a5d38", "hot chocolate whisk-9538e6e1f8fbdf7e", "chocolate machine-1c8a9697b79953bb", "basil plant-e61521c12f3074dc", "some chillies2-2246ccb232aa24cd", "pouch of chillies-25de5b01414e3b82", "tablespoon-7fd55c9342415a7e", "ceramic tumblers-a3b1032bcc9d7f55", "cup-0228c30b8d90c236", "closed tupperware-f00534c2b54ad11e", "rag-3f53d8014bb15ed0", "lunch bag-775abbb72f1f7677", "bag of microwave rice-baab17aa73970e45", "plate3-713696f2ea77f1c4", "tuuperware3-ed95484fc33fb950", "serving spoon-780a719423598113", "tupperware cover-d961f49932f43bd8", "sponge2-db23d2c789ec7e19", "kitchen rag-a6caebb05411dcc4", "towel-054b5bae99ecca01" ], "reasoning": "1. I noticed the user grab a plate (plate3-713696f2ea77f1c4) from the cabinet underneath the microwave at 39:36. \n2. I saw the user grab a fork (fork-133ad7eebaa13290) at 39:44.\n3. I saw the user stir the contents of the pot (pot-457ffd4355e76336) with the fork (fork-133ad7eebaa13290) at 39:47.", "referenced_object_masks": [ "fork-133ad7eebaa13290", "plate3-713696f2ea77f1c4", "pot-457ffd4355e76336" ] }, { "key": "P09-20240621-093545:7e75d41263c4284a7f6ee9618473a809c61369c7", "video_id": "P09-20240621-093545", "question": "Where did I put the chicken breasts after removing them from the package and before washing them?", "answer_choice_0": "Onto the kitchen stove", "answer_choice_1": "Into the refrigerator", "answer_choice_2": "Into a saucepan", "answer_choice_3": "Onto a cutting board", "answer_choice_4": "Into the kitchen sink", "answer_id": 3, "answer": "Onto a cutting board", "question_type": [ "Object Recognition", "Action Recognition", "State Changes", "Situational Awareness", "Temporal Reasoning", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "yoghurt-65ad708d1cda5542", "small bowl of yogurt-2da04b4f9bf804c9", "pack of chicken-b8790276dc4edf19", "jar-ce984a52f7841857", "knife-b8e0ec4b5c6bd6ea", "bottle of washing up liquid-fc76e4531e81ca3d", "sponge-0ceb1670caafdfdb", "knife block-b5bf07daf90b2db3", "knife2-b457565b7bc67648", "white chopping board-1c6798257b4130f8", "chicken breast1-cf2c5d7c76f9f9e6", "chicken breast2-8e44a9c6cfec4198", "bowl-4ac981522f0c93c7", "hand towels-5c1d4a41dc268f57", "another hand towel-4160938a0d7d84bb", "hand towels2-ca4d7c40e6e75128", "measuring scale-6d8ca5c698a32767", "washed chickens-490cbd4a97384842", "washed chicken 2-582a187efb5804d8", "washed chicken 1-8922ccce16c077a4", "chicken breast piece-91d97e2f2c98b33a", "another chicken breast-7142d4f277b0799a", "chicken breast packet-76a185c93d44530c", "pack of cling film-5ec93b8161fdf484", "pack of cling film2-21a4420f5e094ca3", "package of cling film3-c149b09ac54c7d8b", "roll of freezer bags-5b10923e683b6a6a", "one freezer bag-c099b4767049f7cc", "another freezer bag-eed75d0dd3f0f980", "retrieved chicken breast1-59cca663cfa30c90", "second chicken breast-f719084be9673a86", "chicken breast chunk-4d2e0317e7e5cb84", "slices of cut chicken breast-cbd8f3bf66b11255", "slices of cut chicken breast1-c1560616b3521f87", "slices of cut chicken breast2-db5ee47a2aab094a", "slices of cut chicken breast3-b1ce1cb8f30e28b2", "lid-9a6adc3900db92f8", "pot cover-708f95a399f58163", "pot-457ffd4355e76336", "bottle of oil-609a6a6968c52a9a", "bottle of salt-6311ffe14b3e81e7", "cheese-4f74916c93d23b4b", "red curry paste-02ae600e3f946f70", "courgette-020e1378635e95a1", "bag of sugar-0e59174cfb192136", "tin coconut milk-6bfb9ce76128eebb", "tin of ghee-56d53c3e3e5d107b", "tin of bamboo shoot-5e12e3317a5fcd03", "courgettes2-3ebd5032dc3ea9c1", "pack of chilli-79b2afbaf9346e1b", "plantains-c691f08c29d039c0", "courgette3-99e8103d31f0f077", "courgettes together-ce68bcd45ab2bbdf", "bowl2-e591e857e8ef4972", "sliced courgette-f71e53eaa4568958", "sliced courgette2-143d9d1f0ae18d44", "fork-133ad7eebaa13290", "plate2-4c521b7283aeb2e3", "bowl3-5079d1f8fffa10df", "colander-5c766dede154b009", "some chillies-5bde19241755ea5f", "bowl4-7ecae136566af053", "spoon-297e798483a73e1c", "lid of curry paste-7c0bdd8329c7d268", "curry paste bag-55e980f49ba4f445", "can opener-901d3a839f5234eb", "kettle-59226e425988a1b4", "some vegetables-d5d0e9eea71474c8", "green bowl-2cf737ee66d122a9", "another chilli-ae93cb958cb2466a", "bin-cf41357d5d16801d", "hand-bb426c14cd0833c6", "tupperware lid-a0c8b567e4eaa15a", "tupperware-744f140a0c860332", "bowl5-202bd59f460a5d38", "hot chocolate whisk-9538e6e1f8fbdf7e", "chocolate machine-1c8a9697b79953bb", "basil plant-e61521c12f3074dc", "some chillies2-2246ccb232aa24cd", "pouch of chillies-25de5b01414e3b82", "tablespoon-7fd55c9342415a7e", "ceramic tumblers-a3b1032bcc9d7f55", "cup-0228c30b8d90c236", "closed tupperware-f00534c2b54ad11e", "rag-3f53d8014bb15ed0", "lunch bag-775abbb72f1f7677", "bag of microwave rice-baab17aa73970e45", "plate3-713696f2ea77f1c4", "tuuperware3-ed95484fc33fb950", "serving spoon-780a719423598113", "tupperware cover-d961f49932f43bd8", "sponge2-db23d2c789ec7e19", "kitchen rag-a6caebb05411dcc4", "towel-054b5bae99ecca01" ], "reasoning": "1. I watched the video and found the pack of chicken breasts (pack of chicken-b8790276dc4edf19) at 00:22.\n2. I saw the user remove 2 chicken breasts (chicken breast1-cf2c5d7c76f9f9e6, chicken breast2-8e44a9c6cfec4198) from the package (pack of chicken-b8790276dc4edf19) at 01:04.\n3. After 01:04, I saw the user wash the chicken breasts (chicken breast1-cf2c5d7c76f9f9e6, chicken breast2-8e44a9c6cfec4198) in the sink at 01:11.\n4. After 01:04 and before 01:11, I saw the user put the chicken breasts (chicken breast1-cf2c5d7c76f9f9e6, chicken breast2-8e44a9c6cfec4198) onto a white cutting board (white chopping board-1c6798257b4130f8) at 01:05-01:09.\n5. Therefore, the user put the chicken breasts (chicken breast1-cf2c5d7c76f9f9e6, chicken breast2-8e44a9c6cfec4198) onto a cutting board (white chopping board-1c6798257b4130f8) after removing them from the package (pack of chicken-b8790276dc4edf19) and before washing them.", "referenced_object_masks": [ "pack of chicken-b8790276dc4edf19", "chicken breast1-cf2c5d7c76f9f9e6", "chicken breast2-8e44a9c6cfec4198", "white chopping board-1c6798257b4130f8" ] }, { "key": "P09-20240621-093545:8e7b45d8b5a35067ebc764b473798f04ba4e8387", "video_id": "P09-20240621-093545", "question": "After I pick up the pan from the left burner on the stoveand and stir it more, what do I pour into the pan?", "answer_choice_0": "Milk", "answer_choice_1": ", Cream", "answer_choice_2": "Water", "answer_choice_3": "Juice", "answer_choice_4": "Oil", "answer_id": 4, "answer": "Oil", "question_type": [ "Action Recognition", "Object Recognition" ], "objects_of_interest": [ "yoghurt-65ad708d1cda5542", "small bowl of yogurt-2da04b4f9bf804c9", "pack of chicken-b8790276dc4edf19", "jar-ce984a52f7841857", "knife-b8e0ec4b5c6bd6ea", "bottle of washing up liquid-fc76e4531e81ca3d", "sponge-0ceb1670caafdfdb", "knife block-b5bf07daf90b2db3", "knife2-b457565b7bc67648", "white chopping board-1c6798257b4130f8", "chicken breast1-cf2c5d7c76f9f9e6", "chicken breast2-8e44a9c6cfec4198", "bowl-4ac981522f0c93c7", "hand towels-5c1d4a41dc268f57", "another hand towel-4160938a0d7d84bb", "hand towels2-ca4d7c40e6e75128", "measuring scale-6d8ca5c698a32767", "washed chickens-490cbd4a97384842", "washed chicken 2-582a187efb5804d8", "washed chicken 1-8922ccce16c077a4", "chicken breast piece-91d97e2f2c98b33a", "another chicken breast-7142d4f277b0799a", "chicken breast packet-76a185c93d44530c", "pack of cling film-5ec93b8161fdf484", "pack of cling film2-21a4420f5e094ca3", "package of cling film3-c149b09ac54c7d8b", "roll of freezer bags-5b10923e683b6a6a", "one freezer bag-c099b4767049f7cc", "another freezer bag-eed75d0dd3f0f980", "retrieved chicken breast1-59cca663cfa30c90", "second chicken breast-f719084be9673a86", "chicken breast chunk-4d2e0317e7e5cb84", "slices of cut chicken breast-cbd8f3bf66b11255", "slices of cut chicken breast1-c1560616b3521f87", "slices of cut chicken breast2-db5ee47a2aab094a", "slices of cut chicken breast3-b1ce1cb8f30e28b2", "lid-9a6adc3900db92f8", "pot cover-708f95a399f58163", "pot-457ffd4355e76336", "bottle of oil-609a6a6968c52a9a", "bottle of salt-6311ffe14b3e81e7", "cheese-4f74916c93d23b4b", "red curry paste-02ae600e3f946f70", "courgette-020e1378635e95a1", "bag of sugar-0e59174cfb192136", "tin coconut milk-6bfb9ce76128eebb", "tin of ghee-56d53c3e3e5d107b", "tin of bamboo shoot-5e12e3317a5fcd03", "courgettes2-3ebd5032dc3ea9c1", "pack of chilli-79b2afbaf9346e1b", "plantains-c691f08c29d039c0", "courgette3-99e8103d31f0f077", "courgettes together-ce68bcd45ab2bbdf", "bowl2-e591e857e8ef4972", "sliced courgette-f71e53eaa4568958", "sliced courgette2-143d9d1f0ae18d44", "fork-133ad7eebaa13290", "plate2-4c521b7283aeb2e3", "bowl3-5079d1f8fffa10df", "colander-5c766dede154b009", "some chillies-5bde19241755ea5f", "bowl4-7ecae136566af053", "spoon-297e798483a73e1c", "lid of curry paste-7c0bdd8329c7d268", "curry paste bag-55e980f49ba4f445", "can opener-901d3a839f5234eb", "kettle-59226e425988a1b4", "some vegetables-d5d0e9eea71474c8", "green bowl-2cf737ee66d122a9", "another chilli-ae93cb958cb2466a", "bin-cf41357d5d16801d", "hand-bb426c14cd0833c6", "tupperware lid-a0c8b567e4eaa15a", "tupperware-744f140a0c860332", "bowl5-202bd59f460a5d38", "hot chocolate whisk-9538e6e1f8fbdf7e", "chocolate machine-1c8a9697b79953bb", "basil plant-e61521c12f3074dc", "some chillies2-2246ccb232aa24cd", "pouch of chillies-25de5b01414e3b82", "tablespoon-7fd55c9342415a7e", "ceramic tumblers-a3b1032bcc9d7f55", "cup-0228c30b8d90c236", "closed tupperware-f00534c2b54ad11e", "rag-3f53d8014bb15ed0", "lunch bag-775abbb72f1f7677", "bag of microwave rice-baab17aa73970e45", "plate3-713696f2ea77f1c4", "tuuperware3-ed95484fc33fb950", "serving spoon-780a719423598113", "tupperware cover-d961f49932f43bd8", "sponge2-db23d2c789ec7e19", "kitchen rag-a6caebb05411dcc4", "towel-054b5bae99ecca01" ], "reasoning": "1. I saw the user pick up the pan from the left burner on the stove at 15:48. \n2. I saw the user stir the sauce pan at 16:10.\n2. I saw the user look over to the right at 16:11.\n3. I saw the user grab the bottle of oil (bottle of oil-609a6a6968c52a9a) at 16:12. \n4. I saw the user pour the oil in the pan at 16:16.", "referenced_object_masks": [ "bottle of oil-609a6a6968c52a9a" ] }, { "key": "P09-20240621-093545:8fbf5d94236aa62234b10513c21982154ad08f46", "video_id": "P09-20240621-093545", "question": "What direction would I have turned to look at the stove if it were in the opposite direction while I was at the refrigerator?", "answer_choice_0": "90 degrees to the right", "answer_choice_1": "90 degrees to the left", "answer_choice_2": "90 degrees upward", "answer_choice_3": "180 degrees behind", "answer_choice_4": "45 degrees downward", "answer_id": 0, "answer": "90 degrees to the right", "question_type": [ "Situational Awareness", "Action Recognition", "Egocentric & Spatial Perception", "Temporal Reasoning", "Counterfactual" ], "objects_of_interest": [ "yoghurt-65ad708d1cda5542", "small bowl of yogurt-2da04b4f9bf804c9", "pack of chicken-b8790276dc4edf19", "jar-ce984a52f7841857", "knife-b8e0ec4b5c6bd6ea", "bottle of washing up liquid-fc76e4531e81ca3d", "sponge-0ceb1670caafdfdb", "knife block-b5bf07daf90b2db3", "knife2-b457565b7bc67648", "white chopping board-1c6798257b4130f8", "chicken breast1-cf2c5d7c76f9f9e6", "chicken breast2-8e44a9c6cfec4198", "bowl-4ac981522f0c93c7", "hand towels-5c1d4a41dc268f57", "another hand towel-4160938a0d7d84bb", "hand towels2-ca4d7c40e6e75128", "measuring scale-6d8ca5c698a32767", "washed chickens-490cbd4a97384842", "washed chicken 2-582a187efb5804d8", "washed chicken 1-8922ccce16c077a4", "chicken breast piece-91d97e2f2c98b33a", "another chicken breast-7142d4f277b0799a", "chicken breast packet-76a185c93d44530c", "pack of cling film-5ec93b8161fdf484", "pack of cling film2-21a4420f5e094ca3", "package of cling film3-c149b09ac54c7d8b", "roll of freezer bags-5b10923e683b6a6a", "one freezer bag-c099b4767049f7cc", "another freezer bag-eed75d0dd3f0f980", "retrieved chicken breast1-59cca663cfa30c90", "second chicken breast-f719084be9673a86", "chicken breast chunk-4d2e0317e7e5cb84", "slices of cut chicken breast-cbd8f3bf66b11255", "slices of cut chicken breast1-c1560616b3521f87", "slices of cut chicken breast2-db5ee47a2aab094a", "slices of cut chicken breast3-b1ce1cb8f30e28b2", "lid-9a6adc3900db92f8", "pot cover-708f95a399f58163", "pot-457ffd4355e76336", "bottle of oil-609a6a6968c52a9a", "bottle of salt-6311ffe14b3e81e7", "cheese-4f74916c93d23b4b", "red curry paste-02ae600e3f946f70", "courgette-020e1378635e95a1", "bag of sugar-0e59174cfb192136", "tin coconut milk-6bfb9ce76128eebb", "tin of ghee-56d53c3e3e5d107b", "tin of bamboo shoot-5e12e3317a5fcd03", "courgettes2-3ebd5032dc3ea9c1", "pack of chilli-79b2afbaf9346e1b", "plantains-c691f08c29d039c0", "courgette3-99e8103d31f0f077", "courgettes together-ce68bcd45ab2bbdf", "bowl2-e591e857e8ef4972", "sliced courgette-f71e53eaa4568958", "sliced courgette2-143d9d1f0ae18d44", "fork-133ad7eebaa13290", "plate2-4c521b7283aeb2e3", "bowl3-5079d1f8fffa10df", "colander-5c766dede154b009", "some chillies-5bde19241755ea5f", "bowl4-7ecae136566af053", "spoon-297e798483a73e1c", "lid of curry paste-7c0bdd8329c7d268", "curry paste bag-55e980f49ba4f445", "can opener-901d3a839f5234eb", "kettle-59226e425988a1b4", "some vegetables-d5d0e9eea71474c8", "green bowl-2cf737ee66d122a9", "another chilli-ae93cb958cb2466a", "bin-cf41357d5d16801d", "hand-bb426c14cd0833c6", "tupperware lid-a0c8b567e4eaa15a", "tupperware-744f140a0c860332", "bowl5-202bd59f460a5d38", "hot chocolate whisk-9538e6e1f8fbdf7e", "chocolate machine-1c8a9697b79953bb", "basil plant-e61521c12f3074dc", "some chillies2-2246ccb232aa24cd", "pouch of chillies-25de5b01414e3b82", "tablespoon-7fd55c9342415a7e", "ceramic tumblers-a3b1032bcc9d7f55", "cup-0228c30b8d90c236", "closed tupperware-f00534c2b54ad11e", "rag-3f53d8014bb15ed0", "lunch bag-775abbb72f1f7677", "bag of microwave rice-baab17aa73970e45", "plate3-713696f2ea77f1c4", "tuuperware3-ed95484fc33fb950", "serving spoon-780a719423598113", "tupperware cover-d961f49932f43bd8", "sponge2-db23d2c789ec7e19", "kitchen rag-a6caebb05411dcc4", "towel-054b5bae99ecca01" ], "reasoning": "1. I watched the video and found the refrigerator at 00:10.\n2. At 00:30, I saw the stove when the user turned 90 degrees to the left to look from the refrigerator and towards the stove.\n3. Since the user turned to the left to view the stove, the stove was consequently to the left of the user while they were at the refrigerator.\n4. The opposite of \"90 degrees to the left\" is \"90 degrees to the right.\"\n5. Therefore, the user would have turned 90 degrees to the right to look at the stove if it were in the opposite direction while they were at the refrigerator.", "referenced_object_masks": [] }, { "key": "P09-20240621-093545:b3a14854c531ef1ee2a892270e8ed38d4069defe", "video_id": "P09-20240621-093545", "question": "How many times do I transfer items from the blue bowl into the pan, with the first completely blue bowl I use?", "answer_choice_0": "3", "answer_choice_1": "4", "answer_choice_2": "5", "answer_choice_3": "7", "answer_choice_4": "6", "answer_id": 3, "answer": "7", "question_type": [ "Action Recognition", "Counting" ], "objects_of_interest": [ "yoghurt-65ad708d1cda5542", "small bowl of yogurt-2da04b4f9bf804c9", "pack of chicken-b8790276dc4edf19", "jar-ce984a52f7841857", "knife-b8e0ec4b5c6bd6ea", "bottle of washing up liquid-fc76e4531e81ca3d", "sponge-0ceb1670caafdfdb", "knife block-b5bf07daf90b2db3", "knife2-b457565b7bc67648", "white chopping board-1c6798257b4130f8", "chicken breast1-cf2c5d7c76f9f9e6", "chicken breast2-8e44a9c6cfec4198", "bowl-4ac981522f0c93c7", "hand towels-5c1d4a41dc268f57", "another hand towel-4160938a0d7d84bb", "hand towels2-ca4d7c40e6e75128", "measuring scale-6d8ca5c698a32767", "washed chickens-490cbd4a97384842", "washed chicken 2-582a187efb5804d8", "washed chicken 1-8922ccce16c077a4", "chicken breast piece-91d97e2f2c98b33a", "another chicken breast-7142d4f277b0799a", "chicken breast packet-76a185c93d44530c", "pack of cling film-5ec93b8161fdf484", "pack of cling film2-21a4420f5e094ca3", "package of cling film3-c149b09ac54c7d8b", "roll of freezer bags-5b10923e683b6a6a", "one freezer bag-c099b4767049f7cc", "another freezer bag-eed75d0dd3f0f980", "retrieved chicken breast1-59cca663cfa30c90", "second chicken breast-f719084be9673a86", "chicken breast chunk-4d2e0317e7e5cb84", "slices of cut chicken breast-cbd8f3bf66b11255", "slices of cut chicken breast1-c1560616b3521f87", "slices of cut chicken breast2-db5ee47a2aab094a", "slices of cut chicken breast3-b1ce1cb8f30e28b2", "lid-9a6adc3900db92f8", "pot cover-708f95a399f58163", "pot-457ffd4355e76336", "bottle of oil-609a6a6968c52a9a", "bottle of salt-6311ffe14b3e81e7", "cheese-4f74916c93d23b4b", "red curry paste-02ae600e3f946f70", "courgette-020e1378635e95a1", "bag of sugar-0e59174cfb192136", "tin coconut milk-6bfb9ce76128eebb", "tin of ghee-56d53c3e3e5d107b", "tin of bamboo shoot-5e12e3317a5fcd03", "courgettes2-3ebd5032dc3ea9c1", "pack of chilli-79b2afbaf9346e1b", "plantains-c691f08c29d039c0", "courgette3-99e8103d31f0f077", "courgettes together-ce68bcd45ab2bbdf", "bowl2-e591e857e8ef4972", "sliced courgette-f71e53eaa4568958", "sliced courgette2-143d9d1f0ae18d44", "fork-133ad7eebaa13290", "plate2-4c521b7283aeb2e3", "bowl3-5079d1f8fffa10df", "colander-5c766dede154b009", "some chillies-5bde19241755ea5f", "bowl4-7ecae136566af053", "spoon-297e798483a73e1c", "lid of curry paste-7c0bdd8329c7d268", "curry paste bag-55e980f49ba4f445", "can opener-901d3a839f5234eb", "kettle-59226e425988a1b4", "some vegetables-d5d0e9eea71474c8", "green bowl-2cf737ee66d122a9", "another chilli-ae93cb958cb2466a", "bin-cf41357d5d16801d", "hand-bb426c14cd0833c6", "tupperware lid-a0c8b567e4eaa15a", "tupperware-744f140a0c860332", "bowl5-202bd59f460a5d38", "hot chocolate whisk-9538e6e1f8fbdf7e", "chocolate machine-1c8a9697b79953bb", "basil plant-e61521c12f3074dc", "some chillies2-2246ccb232aa24cd", "pouch of chillies-25de5b01414e3b82", "tablespoon-7fd55c9342415a7e", "ceramic tumblers-a3b1032bcc9d7f55", "cup-0228c30b8d90c236", "closed tupperware-f00534c2b54ad11e", "rag-3f53d8014bb15ed0", "lunch bag-775abbb72f1f7677", "bag of microwave rice-baab17aa73970e45", "plate3-713696f2ea77f1c4", "tuuperware3-ed95484fc33fb950", "serving spoon-780a719423598113", "tupperware cover-d961f49932f43bd8", "sponge2-db23d2c789ec7e19", "kitchen rag-a6caebb05411dcc4", "towel-054b5bae99ecca01" ], "reasoning": "1. I noticed the user grab a completely blue bowl (bowl3-5079d1f8fffa10df) from 11:33-11:35. This is the first time I have seen a completely blue bowl.\n2. I noticed the user grab a handful of white cooked chicken pieces from the bowl (bowl3-5079d1f8fffa10df) at 16:51 and put them in the pan (pot-457ffd4355e76336).\n3. I noticed the user transfers white chicken pieces from the bowl (bowl3-5079d1f8fffa10df) into the pan (pot-457ffd4355e76336) a total of 7 times by 17:02. These transfers happen from 16:51-16:54, 16:54-16:56, 16:56-16:57, 16:58-16:59, 17:00-17:02, 17:02-17:04, and from 17:04-17:06.\n4. Therefore, the answer is 7.", "referenced_object_masks": [ "bowl3-5079d1f8fffa10df", "pot-457ffd4355e76336" ] }, { "key": "P09-20240621-093545:bada30e13995853888c02182c68f629830dbe237", "video_id": "P09-20240621-093545", "question": "If I took twice as much time to cut the 2 chicken breasts into strips, approximately how much time would I have taken to cut all 4 of them?", "answer_choice_0": "5 minutes and 50 seconds", "answer_choice_1": "4 minutes and 40 seconds", "answer_choice_2": "1 minute and 10 seconds", "answer_choice_3": "2 minutes and 20 seconds", "answer_choice_4": "3 minutes and 30 seconds", "answer_id": 3, "answer": "2 minutes and 20 seconds", "question_type": [ "Object Recognition", "Counting", "Action Recognition", "State Changes", "Temporal Reasoning", "Numerical Reasoning", "Counterfactual" ], "objects_of_interest": [ "yoghurt-65ad708d1cda5542", "small bowl of yogurt-2da04b4f9bf804c9", "pack of chicken-b8790276dc4edf19", "jar-ce984a52f7841857", "knife-b8e0ec4b5c6bd6ea", "bottle of washing up liquid-fc76e4531e81ca3d", "sponge-0ceb1670caafdfdb", "knife block-b5bf07daf90b2db3", "knife2-b457565b7bc67648", "white chopping board-1c6798257b4130f8", "chicken breast1-cf2c5d7c76f9f9e6", "chicken breast2-8e44a9c6cfec4198", "bowl-4ac981522f0c93c7", "hand towels-5c1d4a41dc268f57", "another hand towel-4160938a0d7d84bb", "hand towels2-ca4d7c40e6e75128", "measuring scale-6d8ca5c698a32767", "washed chickens-490cbd4a97384842", "washed chicken 2-582a187efb5804d8", "washed chicken 1-8922ccce16c077a4", "chicken breast piece-91d97e2f2c98b33a", "another chicken breast-7142d4f277b0799a", "chicken breast packet-76a185c93d44530c", "pack of cling film-5ec93b8161fdf484", "pack of cling film2-21a4420f5e094ca3", "package of cling film3-c149b09ac54c7d8b", "roll of freezer bags-5b10923e683b6a6a", "one freezer bag-c099b4767049f7cc", "another freezer bag-eed75d0dd3f0f980", "retrieved chicken breast1-59cca663cfa30c90", "second chicken breast-f719084be9673a86", "chicken breast chunk-4d2e0317e7e5cb84", "slices of cut chicken breast-cbd8f3bf66b11255", "slices of cut chicken breast1-c1560616b3521f87", "slices of cut chicken breast2-db5ee47a2aab094a", "slices of cut chicken breast3-b1ce1cb8f30e28b2", "lid-9a6adc3900db92f8", "pot cover-708f95a399f58163", "pot-457ffd4355e76336", "bottle of oil-609a6a6968c52a9a", "bottle of salt-6311ffe14b3e81e7", "cheese-4f74916c93d23b4b", "red curry paste-02ae600e3f946f70", "courgette-020e1378635e95a1", "bag of sugar-0e59174cfb192136", "tin coconut milk-6bfb9ce76128eebb", "tin of ghee-56d53c3e3e5d107b", "tin of bamboo shoot-5e12e3317a5fcd03", "courgettes2-3ebd5032dc3ea9c1", "pack of chilli-79b2afbaf9346e1b", "plantains-c691f08c29d039c0", "courgette3-99e8103d31f0f077", "courgettes together-ce68bcd45ab2bbdf", "bowl2-e591e857e8ef4972", "sliced courgette-f71e53eaa4568958", "sliced courgette2-143d9d1f0ae18d44", "fork-133ad7eebaa13290", "plate2-4c521b7283aeb2e3", "bowl3-5079d1f8fffa10df", "colander-5c766dede154b009", "some chillies-5bde19241755ea5f", "bowl4-7ecae136566af053", "spoon-297e798483a73e1c", "lid of curry paste-7c0bdd8329c7d268", "curry paste bag-55e980f49ba4f445", "can opener-901d3a839f5234eb", "kettle-59226e425988a1b4", "some vegetables-d5d0e9eea71474c8", "green bowl-2cf737ee66d122a9", "another chilli-ae93cb958cb2466a", "bin-cf41357d5d16801d", "hand-bb426c14cd0833c6", "tupperware lid-a0c8b567e4eaa15a", "tupperware-744f140a0c860332", "bowl5-202bd59f460a5d38", "hot chocolate whisk-9538e6e1f8fbdf7e", "chocolate machine-1c8a9697b79953bb", "basil plant-e61521c12f3074dc", "some chillies2-2246ccb232aa24cd", "pouch of chillies-25de5b01414e3b82", "tablespoon-7fd55c9342415a7e", "ceramic tumblers-a3b1032bcc9d7f55", "cup-0228c30b8d90c236", "closed tupperware-f00534c2b54ad11e", "rag-3f53d8014bb15ed0", "lunch bag-775abbb72f1f7677", "bag of microwave rice-baab17aa73970e45", "plate3-713696f2ea77f1c4", "tuuperware3-ed95484fc33fb950", "serving spoon-780a719423598113", "tupperware cover-d961f49932f43bd8", "sponge2-db23d2c789ec7e19", "kitchen rag-a6caebb05411dcc4", "towel-054b5bae99ecca01" ], "reasoning": "1. I watched the video and found the pack of chicken breasts (pack of chicken-b8790276dc4edf19) at 00:22.\n2. I counted 4 chicken breasts inside the pack (pack of chicken-b8790276dc4edf19) at 01:04.\n3. I counted 2 chicken breasts (chicken breast1-cf2c5d7c76f9f9e6, chicken breast2-8e44a9c6cfec4198) the user removed from the pack (pack of chicken-b8790276dc4edf19) at 01:04.\n4. I saw the user start cutting the first chicken breast (chicken breast1-cf2c5d7c76f9f9e6) into strips at 02:41.\n5. I saw the user finish cutting the first chicken breast (chicken breast1-cf2c5d7c76f9f9e6) into strips at 02:54.\n6. I calculated 02:54-02:41=00:13.\n7. I saw the user start cutting the second chicken breast (chicken breast2-8e44a9c6cfec4198) into strips at 04:06.\n8. I saw the user finish cutting the second chicken breast (chicken breast2-8e44a9c6cfec4198) into strips at 04:29.\n9. I calculated 04:29-04:06=00:23.\n10. I calculated 00:13+00:23=00:36.\n11. I calculated 36*2=72.\n12. I calculated 72*2=144.\n13. I converted 144 seconds to 2 minutes and 24 seconds, which approximates to 2 minutes and 20 seconds.\n14. Therefore, if the user took twice as much time to cut the 2 chicken breasts (chicken breast1-cf2c5d7c76f9f9e6, chicken breast2-8e44a9c6cfec4198) into strips, they would have taken approximately 2 minutes and 20 seconds to cut all 4 of them.", "referenced_object_masks": [ "pack of chicken-b8790276dc4edf19", "chicken breast2-8e44a9c6cfec4198", "chicken breast1-cf2c5d7c76f9f9e6" ] }, { "key": "P09-20240621-153208:0b46067c498a3a78eb2984c6538fc9f1f41b269e", "video_id": "P09-20240621-153208", "question": "What is the difference between the number of items sitting on the black mat to the right of the stovetop and the amount of frying pans I put away from the dish drying rack?", "answer_choice_0": "2", "answer_choice_1": "3", "answer_choice_2": "0", "answer_choice_3": "1", "answer_choice_4": "4", "answer_id": 0, "answer": "2", "question_type": [ "Action Recognition", "Event Occurence", "Object Recognition", "Counting", "Numerical Reasoning" ], "objects_of_interest": [ "packet1-caa64c36f9230875", "weighing scales-07a7bdb86f519819", "bag of ginger-332a5a68afbf7632", "ginger root-6623199d4318b2e4", "piece of ginger1-75f6c08f824a8783", "saucepan-a6c988e3a34fd214", "knife-a953f013ba65d18a", "packet of freezer bags-ab2609ec278240e0", "freezer bag1-517858839d8838e4", "freezer bag2-9f02030ed27c975e", "chopping board-fd8a9f4e7602f1c0", "saucepan2-9abab4497042e3cd", "saucepan3-96510105da1f0c04", "saucepan lid-b9ca1eb8fd96f3c1", "frying pan-d114b0713768d371", "frying pan2-17d9ace5f3331085", "colander-63c484cbca0c0492", "metal colander-22eb7eeabe142878", "plastic box-34654e54d822c66b", "serving spoon-087b540fb53ba990", "saucepan lid2-e1c42b7b084c9479", "cup1-9ce14d9dac0146a6", "cup2-25ec4d4a87b6deb2", "cup3-7d92edb83473be44", "mug1-90869e62e1a10691", "spoon1-090982fce1e3386e", "spoon2-84ed24b26503b75e", "sponge-9e2107cababc8708", "bottle of washing up liquid-310ab08333cb90b9", "plastic bottle-148e3b691902a389", "tea towel-9323086df0257151", "kettle-ec6d98316af5c9cc", "jar-ee109fb7ed8f78a8", "lid of jar-735d370142d171b8", "jar2-1afedd4f86fef78c", "lid of jar2-2d0f3590a81e0b13", "blue cloth-1644e846b3e9d46a", "sticker-c7cd3932247635ab", "carton of orange juice-7f5e9cdb13a7ab97", "cap of orange juice-9b750a68f95831ca", "spatula-11ac2e7c1ad490dd", "slotted spoon-2a4095823fa0cda4", "bag of sugar-80f7cea2668443a4", "jar of sugar-f31ca9c8d2683b50", "lid of jar of sugar-2c070ca8a84e0ef0", "blender jar-284afab995eb2695", "blender jar2-a6cb9461e289ec27", "bowl1-af6c96bc86a1a212", "glass bowl-2ace6e6e60343d1e", "spoon3-4d0a1a0ff0b581d3" ], "reasoning": "1. I watched the video to see the user standing to the right of the stovetop , and specifically at 02:12, I observed 4 items (kettle-ec6d98316af5c9cc).\n2. Then, I watched the video to see the user begin putting the dishes on the drying rack away at 03:20.\n3. Watching for frying pans specifically, I saw the user stack two frying pans (frying pan-d114b0713768d371, frying pan2-17d9ace5f3331085) and put them away from 03:34-03:40.\n4. I continued to see when the user finished putting the dishes away and if any of those were frying pans.\n5. At 04:15, I saw the user move to perform another task, ending the process of putting the dishes away, with no additional frying pans being put away.\n6. The final math then is 4 - 2 = 2.", "referenced_object_masks": [ "frying pan-d114b0713768d371", "frying pan2-17d9ace5f3331085", "kettle-ec6d98316af5c9cc" ] }, { "key": "P09-20240621-153208:160a3232fc395c4345b14087ece1181ebdb708fe", "video_id": "P09-20240621-153208", "question": "How many scoops does it take to extract all of the ingredients from the saucepan?", "answer_choice_0": "6", "answer_choice_1": "8", "answer_choice_2": "10", "answer_choice_3": "9", "answer_choice_4": "11", "answer_id": 3, "answer": "9", "question_type": [ "Action Recognition", "Counting", "Event Occurence" ], "objects_of_interest": [ "packet1-caa64c36f9230875", "weighing scales-07a7bdb86f519819", "bag of ginger-332a5a68afbf7632", "ginger root-6623199d4318b2e4", "piece of ginger1-75f6c08f824a8783", "saucepan-a6c988e3a34fd214", "knife-a953f013ba65d18a", "packet of freezer bags-ab2609ec278240e0", "freezer bag1-517858839d8838e4", "freezer bag2-9f02030ed27c975e", "chopping board-fd8a9f4e7602f1c0", "saucepan2-9abab4497042e3cd", "saucepan3-96510105da1f0c04", "saucepan lid-b9ca1eb8fd96f3c1", "frying pan-d114b0713768d371", "frying pan2-17d9ace5f3331085", "colander-63c484cbca0c0492", "metal colander-22eb7eeabe142878", "plastic box-34654e54d822c66b", "serving spoon-087b540fb53ba990", "saucepan lid2-e1c42b7b084c9479", "cup1-9ce14d9dac0146a6", "cup2-25ec4d4a87b6deb2", "cup3-7d92edb83473be44", "mug1-90869e62e1a10691", "spoon1-090982fce1e3386e", "spoon2-84ed24b26503b75e", "sponge-9e2107cababc8708", "bottle of washing up liquid-310ab08333cb90b9", "plastic bottle-148e3b691902a389", "tea towel-9323086df0257151", "kettle-ec6d98316af5c9cc", "jar-ee109fb7ed8f78a8", "lid of jar-735d370142d171b8", "jar2-1afedd4f86fef78c", "lid of jar2-2d0f3590a81e0b13", "blue cloth-1644e846b3e9d46a", "sticker-c7cd3932247635ab", "carton of orange juice-7f5e9cdb13a7ab97", "cap of orange juice-9b750a68f95831ca", "spatula-11ac2e7c1ad490dd", "slotted spoon-2a4095823fa0cda4", "bag of sugar-80f7cea2668443a4", "jar of sugar-f31ca9c8d2683b50", "lid of jar of sugar-2c070ca8a84e0ef0", "blender jar-284afab995eb2695", "blender jar2-a6cb9461e289ec27", "bowl1-af6c96bc86a1a212", "glass bowl-2ace6e6e60343d1e", "spoon3-4d0a1a0ff0b581d3" ], "reasoning": "1. I watched the video to see the user prepare to scoop the ingredients out of the saucepan (saucepan-a6c988e3a34fd214) at 13:25.\n2. Specifically, the user executes the first scoop with the slotted spoon (slotted spoon-2a4095823fa0cda4) from 13:29-13:34.\n3. The user does a second scoop with the spoon (slotted spoon-2a4095823fa0cda4) from 13:35-13:39.\n4. A third scoop occurs from 13:40-13:46.\n5. The user does a fourth scoop from 13:47-13:50.\n6. Next, the user scoops again, for a fifth time, from 13:51-13:54.\n7. Then, the user continues, scooping for a sixth time from 13:55-14:01.\n8. A seventh scoop occurs from 14:02-14:07.\n9. The user continues to scoop for the eighth time from 14:08-14:11.\n10. The user scoops again, this being the ninth scoop, from 14:12-14:16.\n111. Then, from 14:16-14:19, the user puts the spoon (slotted spoon-2a4095823fa0cda4) into the pan (saucepan-a6c988e3a34fd214) before removing it and moving to another task.\n12. Thus, the user does 9 scoops to get all the ingredients out of the pan (saucepan-a6c988e3a34fd214).", "referenced_object_masks": [ "slotted spoon-2a4095823fa0cda4", "saucepan-a6c988e3a34fd214" ] }, { "key": "P09-20240621-153208:29fce53fafbcdba933b878babed46fd0de656b61", "video_id": "P09-20240621-153208", "question": "During the span where I clean the first small hexagonal jar in the sink, at what point do I put soap on the sponge?", "answer_choice_0": "Before scrubbing the jar, scrubbing the lid, and washing out the jar", "answer_choice_1": "Before scrubbing both the jar and the lid, but after washing out the jar", "answer_choice_2": "Before scrubbing the jar and washing out the jar, but after scrubbing the lid", "answer_choice_3": "After scrubbing the jar, washing out the jar, and scrubbing the lid", "answer_choice_4": "Rater 1 was correct", "answer_id": 4, "answer": "Rater 1 was correct", "question_type": [ "Action Recognition", "Temporal Reasoning", "Event Occurence" ], "objects_of_interest": [ "packet1-caa64c36f9230875", "weighing scales-07a7bdb86f519819", "bag of ginger-332a5a68afbf7632", "ginger root-6623199d4318b2e4", "piece of ginger1-75f6c08f824a8783", "saucepan-a6c988e3a34fd214", "knife-a953f013ba65d18a", "packet of freezer bags-ab2609ec278240e0", "freezer bag1-517858839d8838e4", "freezer bag2-9f02030ed27c975e", "chopping board-fd8a9f4e7602f1c0", "saucepan2-9abab4497042e3cd", "saucepan3-96510105da1f0c04", "saucepan lid-b9ca1eb8fd96f3c1", "frying pan-d114b0713768d371", "frying pan2-17d9ace5f3331085", "colander-63c484cbca0c0492", "metal colander-22eb7eeabe142878", "plastic box-34654e54d822c66b", "serving spoon-087b540fb53ba990", "saucepan lid2-e1c42b7b084c9479", "cup1-9ce14d9dac0146a6", "cup2-25ec4d4a87b6deb2", "cup3-7d92edb83473be44", "mug1-90869e62e1a10691", "spoon1-090982fce1e3386e", "spoon2-84ed24b26503b75e", "sponge-9e2107cababc8708", "bottle of washing up liquid-310ab08333cb90b9", "plastic bottle-148e3b691902a389", "tea towel-9323086df0257151", "kettle-ec6d98316af5c9cc", "jar-ee109fb7ed8f78a8", "lid of jar-735d370142d171b8", "jar2-1afedd4f86fef78c", "lid of jar2-2d0f3590a81e0b13", "blue cloth-1644e846b3e9d46a", "sticker-c7cd3932247635ab", "carton of orange juice-7f5e9cdb13a7ab97", "cap of orange juice-9b750a68f95831ca", "spatula-11ac2e7c1ad490dd", "slotted spoon-2a4095823fa0cda4", "bag of sugar-80f7cea2668443a4", "jar of sugar-f31ca9c8d2683b50", "lid of jar of sugar-2c070ca8a84e0ef0", "blender jar-284afab995eb2695", "blender jar2-a6cb9461e289ec27", "bowl1-af6c96bc86a1a212", "glass bowl-2ace6e6e60343d1e", "spoon3-4d0a1a0ff0b581d3" ], "reasoning": "1. To find this answer, I first determine when the user handles the first small hexagonal jar (jar-ee109fb7ed8f78a8) at the sink - I watch this sequence begin at 06:48, when the user picks up the jar (jar-ee109fb7ed8f78a8) and takes its lid off.\n2. I watch the user use the sponge (sponge-9e2107cababc8708) to scrub the jar (jar-ee109fb7ed8f78a8) from 06:51-07:00, noting this as the first event in the sequence.\n3. Next, I watch the user place soap (bottle of washing up liquid-310ab08333cb90b9) onto the sponge from 07:01-07:03, (sponge-9e2107cababc8708) making this the second event in the sequence.\n4. For the third chronological event in the sequence, I watch the user scrub the lid of the jar (lid of jar-735d370142d171b8) from 07:05-07:10.\n5. From 07:11-07:17, I watched the user wash the jar (jar-ee109fb7ed8f78a8) and its lid (lid of jar-735d370142d171b8) under the faucet, making that the fourth and final event of the sequence.\n6. I consider the order of these events to identify when the sponge (sponge-9e2107cababc8708) has soap (bottle of washing up liquid-310ab08333cb90b9) placed on it relative to the other events - this occurs after the jar (jar-ee109fb7ed8f78a8) is scrubbed, but before the lid (lid of jar-735d370142d171b8) is scribbed or the jar (jar-ee109fb7ed8f78a8) is washed, aligning with answer choice C.", "referenced_object_masks": [ "lid of jar-735d370142d171b8", "jar-ee109fb7ed8f78a8", "bottle of washing up liquid-310ab08333cb90b9", "sponge-9e2107cababc8708" ] }, { "key": "P09-20240621-153208:30fc50eaef394f774fb6879340f6fa8c527625d3", "video_id": "P09-20240621-153208", "question": "What is the ratio of the weight of the ginger root when it is first weighed to the number of pieces I cut off from the original piece of ginger on the cutting board?", "answer_choice_0": "47:5", "answer_choice_1": "47:10", "answer_choice_2": "24:5", "answer_choice_3": "48:5", "answer_choice_4": "16:3", "answer_id": 2, "answer": "24:5", "question_type": [ "Action Recognition", "Reading", "Object Recognition", "Numerical Reasoning" ], "objects_of_interest": [ "packet1-caa64c36f9230875", "weighing scales-07a7bdb86f519819", "bag of ginger-332a5a68afbf7632", "ginger root-6623199d4318b2e4", "piece of ginger1-75f6c08f824a8783", "saucepan-a6c988e3a34fd214", "knife-a953f013ba65d18a", "packet of freezer bags-ab2609ec278240e0", "freezer bag1-517858839d8838e4", "freezer bag2-9f02030ed27c975e", "chopping board-fd8a9f4e7602f1c0", "saucepan2-9abab4497042e3cd", "saucepan3-96510105da1f0c04", "saucepan lid-b9ca1eb8fd96f3c1", "frying pan-d114b0713768d371", "frying pan2-17d9ace5f3331085", "colander-63c484cbca0c0492", "metal colander-22eb7eeabe142878", "plastic box-34654e54d822c66b", "serving spoon-087b540fb53ba990", "saucepan lid2-e1c42b7b084c9479", "cup1-9ce14d9dac0146a6", "cup2-25ec4d4a87b6deb2", "cup3-7d92edb83473be44", "mug1-90869e62e1a10691", "spoon1-090982fce1e3386e", "spoon2-84ed24b26503b75e", "sponge-9e2107cababc8708", "bottle of washing up liquid-310ab08333cb90b9", "plastic bottle-148e3b691902a389", "tea towel-9323086df0257151", "kettle-ec6d98316af5c9cc", "jar-ee109fb7ed8f78a8", "lid of jar-735d370142d171b8", "jar2-1afedd4f86fef78c", "lid of jar2-2d0f3590a81e0b13", "blue cloth-1644e846b3e9d46a", "sticker-c7cd3932247635ab", "carton of orange juice-7f5e9cdb13a7ab97", "cap of orange juice-9b750a68f95831ca", "spatula-11ac2e7c1ad490dd", "slotted spoon-2a4095823fa0cda4", "bag of sugar-80f7cea2668443a4", "jar of sugar-f31ca9c8d2683b50", "lid of jar of sugar-2c070ca8a84e0ef0", "blender jar-284afab995eb2695", "blender jar2-a6cb9461e289ec27", "bowl1-af6c96bc86a1a212", "glass bowl-2ace6e6e60343d1e", "spoon3-4d0a1a0ff0b581d3" ], "reasoning": "1. To determine this, I first identify when the ginger root (piece of ginger1-75f6c08f824a8783) is first weighed - I observe this from 00:25-00:30, when the root (piece of ginger1-75f6c08f824a8783) is placed on the food scale (weighing scales-07a7bdb86f519819). During this span, I read the weight on the scale (weighing scales-07a7bdb86f519819): though it vascillates slightly, the number on the scale (weighing scales-07a7bdb86f519819) clearly reads 48, giving me my first number of the ratio.\n2. Next, I watch the video to determine when the user cuts the ginger root (piece of ginger1-75f6c08f824a8783) - this occurs from 02:49 to 03:03. During this span, I observe the user place the ginger (piece of ginger1-75f6c08f824a8783) onto the cutting board (chopping board-fd8a9f4e7602f1c0), and use the knife (knife-a953f013ba65d18a) to cut it - I count that the user cuts off 10 smaller ginger pieces, giving me my second number of the ratio.\n3. I combine the numbers I obtained in steps 1 and 2 to get a ratio of 48:10; simplified to the lowest common denominator, this is a ratio of 24:5, making C the correct answer.", "referenced_object_masks": [ "weighing scales-07a7bdb86f519819", "chopping board-fd8a9f4e7602f1c0", "knife-a953f013ba65d18a", "piece of ginger1-75f6c08f824a8783" ] }, { "key": "P09-20240621-153208:6462b4646b3f08eb5af77c9b6775903432604352", "video_id": "P09-20240621-153208", "question": "When I put away the food scale, where is the red saucepan in relation to where I'm standing?", "answer_choice_0": "At about 1 o'clock", "answer_choice_1": "At about 12 o'clock", "answer_choice_2": "At about 10 o'clock", "answer_choice_3": "At about 8 o'clock", "answer_choice_4": "At about 3 o'clock", "answer_id": 2, "answer": "At about 10 o'clock", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "packet1-caa64c36f9230875", "weighing scales-07a7bdb86f519819", "bag of ginger-332a5a68afbf7632", "ginger root-6623199d4318b2e4", "piece of ginger1-75f6c08f824a8783", "saucepan-a6c988e3a34fd214", "knife-a953f013ba65d18a", "packet of freezer bags-ab2609ec278240e0", "freezer bag1-517858839d8838e4", "freezer bag2-9f02030ed27c975e", "chopping board-fd8a9f4e7602f1c0", "saucepan2-9abab4497042e3cd", "saucepan3-96510105da1f0c04", "saucepan lid-b9ca1eb8fd96f3c1", "frying pan-d114b0713768d371", "frying pan2-17d9ace5f3331085", "colander-63c484cbca0c0492", "metal colander-22eb7eeabe142878", "plastic box-34654e54d822c66b", "serving spoon-087b540fb53ba990", "saucepan lid2-e1c42b7b084c9479", "cup1-9ce14d9dac0146a6", "cup2-25ec4d4a87b6deb2", "cup3-7d92edb83473be44", "mug1-90869e62e1a10691", "spoon1-090982fce1e3386e", "spoon2-84ed24b26503b75e", "sponge-9e2107cababc8708", "bottle of washing up liquid-310ab08333cb90b9", "plastic bottle-148e3b691902a389", "tea towel-9323086df0257151", "kettle-ec6d98316af5c9cc", "jar-ee109fb7ed8f78a8", "lid of jar-735d370142d171b8", "jar2-1afedd4f86fef78c", "lid of jar2-2d0f3590a81e0b13", "blue cloth-1644e846b3e9d46a", "sticker-c7cd3932247635ab", "carton of orange juice-7f5e9cdb13a7ab97", "cap of orange juice-9b750a68f95831ca", "spatula-11ac2e7c1ad490dd", "slotted spoon-2a4095823fa0cda4", "bag of sugar-80f7cea2668443a4", "jar of sugar-f31ca9c8d2683b50", "lid of jar of sugar-2c070ca8a84e0ef0", "blender jar-284afab995eb2695", "blender jar2-a6cb9461e289ec27", "bowl1-af6c96bc86a1a212", "glass bowl-2ace6e6e60343d1e", "spoon3-4d0a1a0ff0b581d3" ], "reasoning": "1. To determine this, I first find the point in the video where the user puts away the food scale (weighing scales-07a7bdb86f519819). I observe this at 05:05, when the user picks up the food scale (weighing scales-07a7bdb86f519819) and places it in a tucked-away place at the back right of the countertop.\n2. I note that the red saucepan (saucepan-a6c988e3a34fd214) is on the stovetop right after this moment, at 05:06.\n3. From here, I note the location and direction in which the user is standing, and its relation to the stovetop: the user is standing on the right side of the kitchen, facing directly toward the counter. This is to the right of the stovetop.\n3. I then note that the red saucepan (saucepan-a6c988e3a34fd214), in relation to the user, is to their left and very slightly in front of them, corresponding to a relative clockwise location of about 10 o'clock. This makes E the correct answer.", "referenced_object_masks": [ "weighing scales-07a7bdb86f519819", "saucepan-a6c988e3a34fd214" ] }, { "key": "P09-20240621-153208:65088133e2df098e634e894511f8667ab9ed8f8e", "video_id": "P09-20240621-153208", "question": "After opening the bag of spices with the knife, what is the third unique item I touch with my left hand?", "answer_choice_0": "Bag of ginger", "answer_choice_1": "Freezer bag box", "answer_choice_2": "Microwave", "answer_choice_3": "Weighing scale", "answer_choice_4": "Freezer bag", "answer_id": 4, "answer": "Freezer bag", "question_type": [ "Action Recognition", "Object Recognition", "Situational Awareness", "Temporal Reasoning" ], "objects_of_interest": [ "packet1-caa64c36f9230875", "weighing scales-07a7bdb86f519819", "bag of ginger-332a5a68afbf7632", "ginger root-6623199d4318b2e4", "piece of ginger1-75f6c08f824a8783", "saucepan-a6c988e3a34fd214", "knife-a953f013ba65d18a", "packet of freezer bags-ab2609ec278240e0", "freezer bag1-517858839d8838e4", "freezer bag2-9f02030ed27c975e", "chopping board-fd8a9f4e7602f1c0", "saucepan2-9abab4497042e3cd", "saucepan3-96510105da1f0c04", "saucepan lid-b9ca1eb8fd96f3c1", "frying pan-d114b0713768d371", "frying pan2-17d9ace5f3331085", "colander-63c484cbca0c0492", "metal colander-22eb7eeabe142878", "plastic box-34654e54d822c66b", "serving spoon-087b540fb53ba990", "saucepan lid2-e1c42b7b084c9479", "cup1-9ce14d9dac0146a6", "cup2-25ec4d4a87b6deb2", "cup3-7d92edb83473be44", "mug1-90869e62e1a10691", "spoon1-090982fce1e3386e", "spoon2-84ed24b26503b75e", "sponge-9e2107cababc8708", "bottle of washing up liquid-310ab08333cb90b9", "plastic bottle-148e3b691902a389", "tea towel-9323086df0257151", "kettle-ec6d98316af5c9cc", "jar-ee109fb7ed8f78a8", "lid of jar-735d370142d171b8", "jar2-1afedd4f86fef78c", "lid of jar2-2d0f3590a81e0b13", "blue cloth-1644e846b3e9d46a", "sticker-c7cd3932247635ab", "carton of orange juice-7f5e9cdb13a7ab97", "cap of orange juice-9b750a68f95831ca", "spatula-11ac2e7c1ad490dd", "slotted spoon-2a4095823fa0cda4", "bag of sugar-80f7cea2668443a4", "jar of sugar-f31ca9c8d2683b50", "lid of jar of sugar-2c070ca8a84e0ef0", "blender jar-284afab995eb2695", "blender jar2-a6cb9461e289ec27", "bowl1-af6c96bc86a1a212", "glass bowl-2ace6e6e60343d1e", "spoon3-4d0a1a0ff0b581d3" ], "reasoning": "1. I watched the video to see the user open the bag of spices (packet1-caa64c36f9230875) with the knife (knife-a953f013ba65d18a) from 00:57-01:04.\n2. At this time, the user is touching the packet of spices (packet1-caa64c36f9230875) with the left hand, making this the first unique item touched with the left hand.\n3. The user touches this packet (packet1-caa64c36f9230875) until 01:30, when the user grabs a bag of freezer bags (packet of freezer bags-ab2609ec278240e0) with both hands.\n4. The user holds the bag of freezer bags (packet of freezer bags-ab2609ec278240e0) from 01:30-01:39, making it the second unique item touched with the left hand.\n5. After placing the box of bags (packet of freezer bags-ab2609ec278240e0) back in the drawer, the user then grabs a single freezer bag (freezer bag1-517858839d8838e4) at 01:44 with their left hand. Thus, this is the third unique item touched with their left hand after opening the spice packet (packet1-caa64c36f9230875).", "referenced_object_masks": [ "freezer bag1-517858839d8838e4", "packet1-caa64c36f9230875", "packet of freezer bags-ab2609ec278240e0", "knife-a953f013ba65d18a" ] }, { "key": "P09-20240621-153208:6c8e99284b6b63acd0489ecf712966049f89f3a7", "video_id": "P09-20240621-153208", "question": "What do I do with ginger in the video during the second time I handle it?", "answer_choice_0": "Remove it from the food scale", "answer_choice_1": "Place it in the saucepan", "answer_choice_2": "Slice it into smaller pieces", "answer_choice_3": "Place it on the food scale", "answer_choice_4": "Break it in half", "answer_id": 0, "answer": "Remove it from the food scale", "question_type": [ "Action Recognition", "Counting", "Event Occurence", "Object Recognition" ], "objects_of_interest": [ "packet1-caa64c36f9230875", "weighing scales-07a7bdb86f519819", "bag of ginger-332a5a68afbf7632", "ginger root-6623199d4318b2e4", "piece of ginger1-75f6c08f824a8783", "saucepan-a6c988e3a34fd214", "knife-a953f013ba65d18a", "packet of freezer bags-ab2609ec278240e0", "freezer bag1-517858839d8838e4", "freezer bag2-9f02030ed27c975e", "chopping board-fd8a9f4e7602f1c0", "saucepan2-9abab4497042e3cd", "saucepan3-96510105da1f0c04", "saucepan lid-b9ca1eb8fd96f3c1", "frying pan-d114b0713768d371", "frying pan2-17d9ace5f3331085", "colander-63c484cbca0c0492", "metal colander-22eb7eeabe142878", "plastic box-34654e54d822c66b", "serving spoon-087b540fb53ba990", "saucepan lid2-e1c42b7b084c9479", "cup1-9ce14d9dac0146a6", "cup2-25ec4d4a87b6deb2", "cup3-7d92edb83473be44", "mug1-90869e62e1a10691", "spoon1-090982fce1e3386e", "spoon2-84ed24b26503b75e", "sponge-9e2107cababc8708", "bottle of washing up liquid-310ab08333cb90b9", "plastic bottle-148e3b691902a389", "tea towel-9323086df0257151", "kettle-ec6d98316af5c9cc", "jar-ee109fb7ed8f78a8", "lid of jar-735d370142d171b8", "jar2-1afedd4f86fef78c", "lid of jar2-2d0f3590a81e0b13", "blue cloth-1644e846b3e9d46a", "sticker-c7cd3932247635ab", "carton of orange juice-7f5e9cdb13a7ab97", "cap of orange juice-9b750a68f95831ca", "spatula-11ac2e7c1ad490dd", "slotted spoon-2a4095823fa0cda4", "bag of sugar-80f7cea2668443a4", "jar of sugar-f31ca9c8d2683b50", "lid of jar of sugar-2c070ca8a84e0ef0", "blender jar-284afab995eb2695", "blender jar2-a6cb9461e289ec27", "bowl1-af6c96bc86a1a212", "glass bowl-2ace6e6e60343d1e", "spoon3-4d0a1a0ff0b581d3" ], "reasoning": "1. To determine the answer to this question, I watch the video and find where the user first handles ginger (ginger root-6623199d4318b2e4) - this happens at 00:20, when the user takes a large ginger root (ginger root-6623199d4318b2e4) out of a bag (bag of ginger-332a5a68afbf7632). From this point on, until the ginger leaves the user's hands, I count as the first handling of ginger. I observe that during the handling, the user breaks the ginger root (ginger root-6623199d4318b2e4) in half at 00:24, and places one of the halves (piece of ginger1-75f6c08f824a8783) on the food scale (weighing scales-07a7bdb86f519819) at 00:25, and the other half back in the bag (bag of ginger-332a5a68afbf7632) at 00:27. This handling, from 00:20-00:27, I count as the first handling of ginger.\n2. I continue watching the video for when the user next handles ginger (piece of ginger1-75f6c08f824a8783) - this occurs from 00:43-00:45, when the user removes the piece of ginger (piece of ginger1-75f6c08f824a8783) from the food scale (weighing scales-07a7bdb86f519819) and places it onto the counter. This is all that happens during this second handling, making B the correct answer.", "referenced_object_masks": [ "piece of ginger1-75f6c08f824a8783", "weighing scales-07a7bdb86f519819", "bag of ginger-332a5a68afbf7632", "ginger root-6623199d4318b2e4" ] }, { "key": "P09-20240621-153208:d7b4f6abd385a8014c4fb63c5d8bbfdc82238c8d", "video_id": "P09-20240621-153208", "question": "When I search in the utensil drawer for a large spoon, in what relative direction is my coffee cup from me?", "answer_choice_0": "6 o'clock", "answer_choice_1": "5 o'clock", "answer_choice_2": "1 o'clock", "answer_choice_3": "12 o'clock", "answer_choice_4": "3 o'clock", "answer_id": 1, "answer": "5 o'clock", "question_type": [ "Egocentric & Spatial Perception", "Action Recognition", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "packet1-caa64c36f9230875", "weighing scales-07a7bdb86f519819", "bag of ginger-332a5a68afbf7632", "ginger root-6623199d4318b2e4", "piece of ginger1-75f6c08f824a8783", "saucepan-a6c988e3a34fd214", "knife-a953f013ba65d18a", "packet of freezer bags-ab2609ec278240e0", "freezer bag1-517858839d8838e4", "freezer bag2-9f02030ed27c975e", "chopping board-fd8a9f4e7602f1c0", "saucepan2-9abab4497042e3cd", "saucepan3-96510105da1f0c04", "saucepan lid-b9ca1eb8fd96f3c1", "frying pan-d114b0713768d371", "frying pan2-17d9ace5f3331085", "colander-63c484cbca0c0492", "metal colander-22eb7eeabe142878", "plastic box-34654e54d822c66b", "serving spoon-087b540fb53ba990", "saucepan lid2-e1c42b7b084c9479", "cup1-9ce14d9dac0146a6", "cup2-25ec4d4a87b6deb2", "cup3-7d92edb83473be44", "mug1-90869e62e1a10691", "spoon1-090982fce1e3386e", "spoon2-84ed24b26503b75e", "sponge-9e2107cababc8708", "bottle of washing up liquid-310ab08333cb90b9", "plastic bottle-148e3b691902a389", "tea towel-9323086df0257151", "kettle-ec6d98316af5c9cc", "jar-ee109fb7ed8f78a8", "lid of jar-735d370142d171b8", "jar2-1afedd4f86fef78c", "lid of jar2-2d0f3590a81e0b13", "blue cloth-1644e846b3e9d46a", "sticker-c7cd3932247635ab", "carton of orange juice-7f5e9cdb13a7ab97", "cap of orange juice-9b750a68f95831ca", "spatula-11ac2e7c1ad490dd", "slotted spoon-2a4095823fa0cda4", "bag of sugar-80f7cea2668443a4", "jar of sugar-f31ca9c8d2683b50", "lid of jar of sugar-2c070ca8a84e0ef0", "blender jar-284afab995eb2695", "blender jar2-a6cb9461e289ec27", "bowl1-af6c96bc86a1a212", "glass bowl-2ace6e6e60343d1e", "spoon3-4d0a1a0ff0b581d3" ], "reasoning": "1. I watched the video to see that from 12:19-12:27, the user searches a drawer under the sink, pulling out a large spoon (slotted spoon-2a4095823fa0cda4).\n2. The user then turns to the right towards the stovetop from 12:27-12:28.\n3. At this point, the mug (mug1-90869e62e1a10691) comes into view on the counter, to the right of the stovetop .\n4. Once I identified the mug (mug1-90869e62e1a10691) on the counter, I rewound back to when the user is looking in the drawer, and observed that the mug is approximately at 5 o'clock from the user.", "referenced_object_masks": [ "mug1-90869e62e1a10691", "slotted spoon-2a4095823fa0cda4" ] }, { "key": "P09-20240621-153208:dbc771b43681d9332c28f32ef1c804e65790a682", "video_id": "P09-20240621-153208", "question": "If I hadn't ripped the label off the plastic bottle, how long would the video have been?", "answer_choice_0": "17:11", "answer_choice_1": "16:02", "answer_choice_2": "16:57", "answer_choice_3": "16:34", "answer_choice_4": "16:46", "answer_id": 3, "answer": "16:34", "question_type": [ "Action Recognition", "Temporal Reasoning", "Counterfactual", "Object Recognition" ], "objects_of_interest": [ "packet1-caa64c36f9230875", "weighing scales-07a7bdb86f519819", "bag of ginger-332a5a68afbf7632", "ginger root-6623199d4318b2e4", "piece of ginger1-75f6c08f824a8783", "saucepan-a6c988e3a34fd214", "knife-a953f013ba65d18a", "packet of freezer bags-ab2609ec278240e0", "freezer bag1-517858839d8838e4", "freezer bag2-9f02030ed27c975e", "chopping board-fd8a9f4e7602f1c0", "saucepan2-9abab4497042e3cd", "saucepan3-96510105da1f0c04", "saucepan lid-b9ca1eb8fd96f3c1", "frying pan-d114b0713768d371", "frying pan2-17d9ace5f3331085", "colander-63c484cbca0c0492", "metal colander-22eb7eeabe142878", "plastic box-34654e54d822c66b", "serving spoon-087b540fb53ba990", "saucepan lid2-e1c42b7b084c9479", "cup1-9ce14d9dac0146a6", "cup2-25ec4d4a87b6deb2", "cup3-7d92edb83473be44", "mug1-90869e62e1a10691", "spoon1-090982fce1e3386e", "spoon2-84ed24b26503b75e", "sponge-9e2107cababc8708", "bottle of washing up liquid-310ab08333cb90b9", "plastic bottle-148e3b691902a389", "tea towel-9323086df0257151", "kettle-ec6d98316af5c9cc", "jar-ee109fb7ed8f78a8", "lid of jar-735d370142d171b8", "jar2-1afedd4f86fef78c", "lid of jar2-2d0f3590a81e0b13", "blue cloth-1644e846b3e9d46a", "sticker-c7cd3932247635ab", "carton of orange juice-7f5e9cdb13a7ab97", "cap of orange juice-9b750a68f95831ca", "spatula-11ac2e7c1ad490dd", "slotted spoon-2a4095823fa0cda4", "bag of sugar-80f7cea2668443a4", "jar of sugar-f31ca9c8d2683b50", "lid of jar of sugar-2c070ca8a84e0ef0", "blender jar-284afab995eb2695", "blender jar2-a6cb9461e289ec27", "bowl1-af6c96bc86a1a212", "glass bowl-2ace6e6e60343d1e", "spoon3-4d0a1a0ff0b581d3" ], "reasoning": "1. I watched the video to find when the user begins ripping the label off the plastic bottle (plastic bottle-148e3b691902a389), which I saw first start to occur at 08:53.\n2. I continued to watch the user pull the label off the bottle (plastic bottle-148e3b691902a389), which lasts until 09:19. \n3. Because ripping the label lasts from 08:53-09:19, I calculated that this is 0:26 total.\n4. Noting that the video is 17:00, I did the final math of 17:00 - 00:26 = 16:34.", "referenced_object_masks": [ "plastic bottle-148e3b691902a389" ] }, { "key": "P09-20240621-153208:eea290b8a9fde1f7890bba86530a0a99de277517", "video_id": "P09-20240621-153208", "question": "During the span in which I unload the dish drying rack and place those dishes into cupboards and cabinets, how many of the dishes that I put away are primarily made of plastic?", "answer_choice_0": "3", "answer_choice_1": "2", "answer_choice_2": "1", "answer_choice_3": "0", "answer_choice_4": "4", "answer_id": 3, "answer": "0", "question_type": [ "Action Recognition", "Counting", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "packet1-caa64c36f9230875", "weighing scales-07a7bdb86f519819", "bag of ginger-332a5a68afbf7632", "ginger root-6623199d4318b2e4", "piece of ginger1-75f6c08f824a8783", "saucepan-a6c988e3a34fd214", "knife-a953f013ba65d18a", "packet of freezer bags-ab2609ec278240e0", "freezer bag1-517858839d8838e4", "freezer bag2-9f02030ed27c975e", "chopping board-fd8a9f4e7602f1c0", "saucepan2-9abab4497042e3cd", "saucepan3-96510105da1f0c04", "saucepan lid-b9ca1eb8fd96f3c1", "frying pan-d114b0713768d371", "frying pan2-17d9ace5f3331085", "colander-63c484cbca0c0492", "metal colander-22eb7eeabe142878", "plastic box-34654e54d822c66b", "serving spoon-087b540fb53ba990", "saucepan lid2-e1c42b7b084c9479", "cup1-9ce14d9dac0146a6", "cup2-25ec4d4a87b6deb2", "cup3-7d92edb83473be44", "mug1-90869e62e1a10691", "spoon1-090982fce1e3386e", "spoon2-84ed24b26503b75e", "sponge-9e2107cababc8708", "bottle of washing up liquid-310ab08333cb90b9", "plastic bottle-148e3b691902a389", "tea towel-9323086df0257151", "kettle-ec6d98316af5c9cc", "jar-ee109fb7ed8f78a8", "lid of jar-735d370142d171b8", "jar2-1afedd4f86fef78c", "lid of jar2-2d0f3590a81e0b13", "blue cloth-1644e846b3e9d46a", "sticker-c7cd3932247635ab", "carton of orange juice-7f5e9cdb13a7ab97", "cap of orange juice-9b750a68f95831ca", "spatula-11ac2e7c1ad490dd", "slotted spoon-2a4095823fa0cda4", "bag of sugar-80f7cea2668443a4", "jar of sugar-f31ca9c8d2683b50", "lid of jar of sugar-2c070ca8a84e0ef0", "blender jar-284afab995eb2695", "blender jar2-a6cb9461e289ec27", "bowl1-af6c96bc86a1a212", "glass bowl-2ace6e6e60343d1e", "spoon3-4d0a1a0ff0b581d3" ], "reasoning": "1. I watch the video to identify the span where the user unloads the dish drying rack and puts away dishes - I observe that this begins at about 03:20, and lasts until about 04:12, when the user becomes distracted by other kitchen tasks.\n2. At 03:20, I observe the user take a pot lid (saucepan lid-b9ca1eb8fd96f3c1) out of the drying rack and place it onto the saucepan (saucepan-a6c988e3a34fd214) - I note that this is primarily made of glass, and do not count it toward the total.\n3. At 03:21, I observe the user take two metal pans (saucepan2-9abab4497042e3cd, saucepan3-96510105da1f0c04) out of the drying rack and place them in a cupboard at 03:28. Since both are primarily made of metal, I do not count them toward the total.\n4. At 03:32, I observe the user grab two more metal pans (frying pan-d114b0713768d371) out of the drying rack and place them in a cupboard directly below. Since they are both made primarily of metal, I count neither toward the total. While these are put away, I observe the user lift a white plastic colander (colander-63c484cbca0c0492) and place it back to make room for them, but since this colander (colander-63c484cbca0c0492) was not originally in the drying rack, I do not count it toward the total either.\n5. At 03:41, I observe the user grab a large metal colander (metal colander-22eb7eeabe142878) and place it in the cupboard below - since it is metal, I do not count it toward the total.\n6. At 03:44, I observe the user pick up a clear plastic tupperware container (plastic box-34654e54d822c66b) off the floor and put it back in the drying rack - I do not count it toward the total because it was not placed into a cupboard or cabinet, per the question's parameters.\n7. At 03:47, I observe the user place a large metal spoon (serving spoon-087b540fb53ba990) into a drawer - since it is not plastic, I do not count it toward the total.\n8. At 03:49, I observe the user place a saucepan lid (saucepan lid2-e1c42b7b084c9479) off to the side on the counter, and do not count it because of its material.\n9. From 03:51-04:01, I observe the user handle four plastic cups from the drying rack , three of which (cup1-9ce14d9dac0146a6, cup2-25ec4d4a87b6deb2) are black, and one of which (cup3-7d92edb83473be44) is clear. The user stacks these cups and places them back into the rack, and since they do not go into a cupboard or cabinet, I do not count them toward the total.\n10. At 04:10, I watch the user place the saucepan lid that was placed on the counter (saucepan lid2-e1c42b7b084c9479) into a cupboard. I do not count it because of its material.\n11. I watch the remainder of the span to ensure that no plastic items are placed into cupboards or cabinets, and then tally the final number: since I observed 0 plastic items being placed into cupboards or cabinets in this span, the answer is A.", "referenced_object_masks": [ "cup3-7d92edb83473be44", "cup1-9ce14d9dac0146a6", "cup2-25ec4d4a87b6deb2", "saucepan-a6c988e3a34fd214", "saucepan lid-b9ca1eb8fd96f3c1", "metal colander-22eb7eeabe142878", "saucepan lid2-e1c42b7b084c9479", "frying pan-d114b0713768d371", "saucepan2-9abab4497042e3cd", "saucepan3-96510105da1f0c04", "colander-63c484cbca0c0492", "plastic box-34654e54d822c66b", "serving spoon-087b540fb53ba990" ] }, { "key": "P09-20240622-150155:3c6fc54beb31e502be0a1aa14e91d07e0a097f80", "video_id": "P09-20240622-150155", "question": "Which was greater, and by how much: The number of times I opened a cabinet before 05:00, or the amount of time (in seconds) I washed my hands while the chicken was cooking in the saucepan - before washing the vegetable peeler?", "answer_choice_0": "Seconds washing hands--by 1", "answer_choice_1": "Cabinets opened--by 1", "answer_choice_2": "Cabinets opened--by 2", "answer_choice_3": "Seconds washing hands--by 2", "answer_choice_4": "Neither--the numbers were the same", "answer_id": 2, "answer": "Cabinets opened--by 2", "question_type": [ "Action Recognition", "Counting", "Numerical Reasoning", "Object Recognition", "State Changes", "Temporal Reasoning" ], "objects_of_interest": [ "saucepan-eaaa0de116893699", "frying pan-d510045435eca92b", "bowl-ea981a91dfb4db94", "bag of chicken-fe3ba4b0a41f9b19", "chicken-1eb0ba2f3be85466", "pyrex bowl-7fd62b5c0e617c12", "discarded chicken bag-669e77c13d83a0c6", "towel-06d457c3d18b0910", "towel2-fea06dac4ea4a30f", "potato peeler-c319f80b11a0a592", "cup of bread crumbs-21252de25de2d8ad", "cup of bread crumbs cap-5d84b03bd77bef5f", "scale-f09148f04fa5d168", "bowl2-d2721bd426e5c19c", "bowl3-34c64810cbc69dc4", "bag of flour-0890c068caf37f46", "pouch of rice-c95b6d502de90cdc", "bottle of ground chili-57ec31dc93a4c02b", "teaspoon-e94f9e5636637939", "bottle of garlic powder-ce5db1acdb4c63a1", "bottle of ground black pepper-ee2a4084493bb086", "bottle of salt-0434f4abf51cc5a5", "tupperware-f062676dc1eb2055", "bowl4-9d7df688cdef6926", "egg-5deb0b311b0b6c63", "egg2-e293348c62e7f37f", "electric kettle-91e63c2cb8515de3", "pot cover-3fda277021cf3664", "saucepan2-85b54cc17ec1671a", "red saucepan-31701be2c3ca6617", "pot cover2-a532e882a3a733be", "bottle of washing up liquid-747a3f68e8427a9c", "sponge-8090a791330975b2", "spoon-a5bc07287813f070", "potatoes-bf87bed0ccee3f9d", "ginger-68836b8eddf285c9", "bag of ginger-d91eb0cfc44f83bd", "pack of dried hibiscus flowers-dc02bcfa55ddf1de", "saucepan cover-ea642f32c4e6490d", "some dried hibiscus flowers-47389b49cd317eb9", "knife-c88b7b52fc975faf", "chunks of sliced potato-8b06f30ac9601bc9", "potato bits-918cf9c5f49a8dff", "potato-0121203fec36c708", "more potato bits-8beb52e430b0bc78", "more potato bits2-11e03b271b1e0134", "chicken breast-04e0d6d6b2971042", "chicken breast2-3c4cc2e9007379e1", "chicken chunk-fcd07ee885138d63", "chicken strip-661cea196e0d16e3", "chicken strip2-02f73767db97f73b", "chicken strip3-ea82baf3e87b6299", "chicken strip4-7a84a44acc9fac15", "chicken strips5-8b4dde3cb0e45483", "spoon 1-c162f4c69a135827", "plastic chopping board-aff6cf3571b5e746", "potato peels-a129023c2e975289", "slice of potato peel-a7c2d019073e057e", "egg shell-f37d5171d244b593", "pair of tongs-b2cb60919ca4589b", "bottle of oil-429f4383d7550753", "table spoon-1e1514d81bd95cbf", "fork-0ccf59091cbd3221", "fork2-16f46ef8e1f52cc8", "potato masher-8ecaad0bee4a2482", "bag of cheese-05b11d84f980233b", "bowl5-4a909ddf142486d7", "fork3-acee27d6717a322b", "serving spoon-db3fc3c2d7b899f4", "foil-efd9a8b5e899fe1e", "packet-a1bb131de6a58410", "potato chicken ball-a396ef3f39417275", "potato chicken ball2-fd1ff80272e8d015", "potato chicken ball3-36f01a64517663bb", "potato chicken ball4-4ce990da51f12cbf", "potato chicken ball5-b9b830a0af24d7a3", "potato chicken ball6-8b9a9e03e054d476", "potato chicken ball7-ef7ef274207eec42" ], "reasoning": "1. I watched the video and counted the number of times the user opened a cabinet before 05:00. This occurred at: 01:12, 02:56, 03:04, 03:46, and 04:48. This makes a total of 5 times. \n2. I saw the user slice 2 chicken breasts (chicken breast-04e0d6d6b2971042, chicken breast2-3c4cc2e9007379e1) and add the pieces to a saucepan (saucepan-eaaa0de116893699) beginning at 15:25.\n3. I saw the user remove the strips of cooked chicken from the saucepan (saucepan-eaaa0de116893699) from 22:44-23:14. \n4. I looked for the times that the user washed their hands after 15:45 and before 22:44, keeping in mind that I'm looking for the instance that occurs before the washing of the vegetable peeler (potato peeler-c319f80b11a0a592). The first instance of handwashing occurred from 18:13-18:16, followed immediately by the washing of the potato peeler (potato peeler-c319f80b11a0a592) from 18:18-18:24. Therefore, I find that 18:13-18:16 is my target time range.\n5. I found the duration of handwashing by subtracting the start time from the end time: 18:16-18:13=00:03. \n6. I found the difference between the number of cabinets opened and the time in seconds washing hands: 5-3=2.", "referenced_object_masks": [ "potato peeler-c319f80b11a0a592", "saucepan-eaaa0de116893699", "chicken breast-04e0d6d6b2971042", "chicken breast2-3c4cc2e9007379e1" ] }, { "key": "P09-20240622-150155:44c5200d5cfbe2336f90af4a5ca92173ddf88d98", "video_id": "P09-20240622-150155", "question": "What ingredient(s) which I weigh in the video are equal to or exceed 100 units of weight?", "answer_choice_0": "Ginger and Mozzarella", "answer_choice_1": "Potato and Chicken", "answer_choice_2": "Flour and Bread Crumbs", "answer_choice_3": "Chicken", "answer_choice_4": "Potato", "answer_id": 0, "answer": "Ginger and Mozzarella", "question_type": [ "Reading", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "saucepan-eaaa0de116893699", "frying pan-d510045435eca92b", "bowl-ea981a91dfb4db94", "bag of chicken-fe3ba4b0a41f9b19", "chicken-1eb0ba2f3be85466", "pyrex bowl-7fd62b5c0e617c12", "discarded chicken bag-669e77c13d83a0c6", "towel-06d457c3d18b0910", "towel2-fea06dac4ea4a30f", "potato peeler-c319f80b11a0a592", "cup of bread crumbs-21252de25de2d8ad", "cup of bread crumbs cap-5d84b03bd77bef5f", "scale-f09148f04fa5d168", "bowl2-d2721bd426e5c19c", "bowl3-34c64810cbc69dc4", "bag of flour-0890c068caf37f46", "pouch of rice-c95b6d502de90cdc", "bottle of ground chili-57ec31dc93a4c02b", "teaspoon-e94f9e5636637939", "bottle of garlic powder-ce5db1acdb4c63a1", "bottle of ground black pepper-ee2a4084493bb086", "bottle of salt-0434f4abf51cc5a5", "tupperware-f062676dc1eb2055", "bowl4-9d7df688cdef6926", "egg-5deb0b311b0b6c63", "egg2-e293348c62e7f37f", "electric kettle-91e63c2cb8515de3", "pot cover-3fda277021cf3664", "saucepan2-85b54cc17ec1671a", "red saucepan-31701be2c3ca6617", "pot cover2-a532e882a3a733be", "bottle of washing up liquid-747a3f68e8427a9c", "sponge-8090a791330975b2", "spoon-a5bc07287813f070", "potatoes-bf87bed0ccee3f9d", "ginger-68836b8eddf285c9", "bag of ginger-d91eb0cfc44f83bd", "pack of dried hibiscus flowers-dc02bcfa55ddf1de", "saucepan cover-ea642f32c4e6490d", "some dried hibiscus flowers-47389b49cd317eb9", "knife-c88b7b52fc975faf", "chunks of sliced potato-8b06f30ac9601bc9", "potato bits-918cf9c5f49a8dff", "potato-0121203fec36c708", "more potato bits-8beb52e430b0bc78", "more potato bits2-11e03b271b1e0134", "chicken breast-04e0d6d6b2971042", "chicken breast2-3c4cc2e9007379e1", "chicken chunk-fcd07ee885138d63", "chicken strip-661cea196e0d16e3", "chicken strip2-02f73767db97f73b", "chicken strip3-ea82baf3e87b6299", "chicken strip4-7a84a44acc9fac15", "chicken strips5-8b4dde3cb0e45483", "spoon 1-c162f4c69a135827", "plastic chopping board-aff6cf3571b5e746", "potato peels-a129023c2e975289", "slice of potato peel-a7c2d019073e057e", "egg shell-f37d5171d244b593", "pair of tongs-b2cb60919ca4589b", "bottle of oil-429f4383d7550753", "table spoon-1e1514d81bd95cbf", "fork-0ccf59091cbd3221", "fork2-16f46ef8e1f52cc8", "potato masher-8ecaad0bee4a2482", "bag of cheese-05b11d84f980233b", "bowl5-4a909ddf142486d7", "fork3-acee27d6717a322b", "serving spoon-db3fc3c2d7b899f4", "foil-efd9a8b5e899fe1e", "packet-a1bb131de6a58410", "potato chicken ball-a396ef3f39417275", "potato chicken ball2-fd1ff80272e8d015", "potato chicken ball3-36f01a64517663bb", "potato chicken ball4-4ce990da51f12cbf", "potato chicken ball5-b9b830a0af24d7a3", "potato chicken ball6-8b9a9e03e054d476", "potato chicken ball7-ef7ef274207eec42" ], "reasoning": "1. I look for individual ingredients weighed by the user, looking for ingredients that surpass or is equal to 100 units of weight by themselves.\n2. At 11:57, after paring down the weight of the ginger (ginger-68836b8eddf285c9), the user stops when the weight of the item reaches \"101\" units of weight.\n3. At 24:33, the user finishes weighing a pile of mozzarella cheese poured from a bag (bag of cheese-05b11d84f980233b), leaving the final weight at \"151\" units of weight.\n4. The user never weighs the potatoes (potatoes-bf87bed0ccee3f9d, chunks of sliced potato-8b06f30ac9601bc9, potato-0121203fec36c708) or the chicken (chicken breast-04e0d6d6b2971042, chicken breast2-3c4cc2e9007379e1). The bread crumbs poured from a cup (cup of bread crumbs-21252de25de2d8ad), weighed at 02:53, comes to \"52\" units of weight, while the flour from the bag (bag of flour-0890c068caf37f46), weighed at 03:42, only comes to \"60\" units of weight.\n5. Therefore, the answer is ginger (ginger-68836b8eddf285c9) and mozzarella .", "referenced_object_masks": [ "cup of bread crumbs-21252de25de2d8ad", "bag of flour-0890c068caf37f46", "ginger-68836b8eddf285c9", "chicken breast-04e0d6d6b2971042", "chicken breast2-3c4cc2e9007379e1", "potatoes-bf87bed0ccee3f9d", "chunks of sliced potato-8b06f30ac9601bc9", "potato-0121203fec36c708", "bag of cheese-05b11d84f980233b" ] }, { "key": "P09-20240622-150155:4bbd2a21af42655db0039a39611c84abfb898232", "video_id": "P09-20240622-150155", "question": "How many more times do I stir the pot filled with ginger and hibiscus clockwise the first time than I stir it counterclockwise the second time (if there is a half-stir, round up)?", "answer_choice_0": "4", "answer_choice_1": "3", "answer_choice_2": "2", "answer_choice_3": "5", "answer_choice_4": "0", "answer_id": 4, "answer": "0", "question_type": [ "Egocentric & Spatial Perception", "Action Recognition", "Object Recognition", "Counting", "Temporal Reasoning" ], "objects_of_interest": [ "saucepan-eaaa0de116893699", "frying pan-d510045435eca92b", "bowl-ea981a91dfb4db94", "bag of chicken-fe3ba4b0a41f9b19", "chicken-1eb0ba2f3be85466", "pyrex bowl-7fd62b5c0e617c12", "discarded chicken bag-669e77c13d83a0c6", "towel-06d457c3d18b0910", "towel2-fea06dac4ea4a30f", "potato peeler-c319f80b11a0a592", "cup of bread crumbs-21252de25de2d8ad", "cup of bread crumbs cap-5d84b03bd77bef5f", "scale-f09148f04fa5d168", "bowl2-d2721bd426e5c19c", "bowl3-34c64810cbc69dc4", "bag of flour-0890c068caf37f46", "pouch of rice-c95b6d502de90cdc", "bottle of ground chili-57ec31dc93a4c02b", "teaspoon-e94f9e5636637939", "bottle of garlic powder-ce5db1acdb4c63a1", "bottle of ground black pepper-ee2a4084493bb086", "bottle of salt-0434f4abf51cc5a5", "tupperware-f062676dc1eb2055", "bowl4-9d7df688cdef6926", "egg-5deb0b311b0b6c63", "egg2-e293348c62e7f37f", "electric kettle-91e63c2cb8515de3", "pot cover-3fda277021cf3664", "saucepan2-85b54cc17ec1671a", "red saucepan-31701be2c3ca6617", "pot cover2-a532e882a3a733be", "bottle of washing up liquid-747a3f68e8427a9c", "sponge-8090a791330975b2", "spoon-a5bc07287813f070", "potatoes-bf87bed0ccee3f9d", "ginger-68836b8eddf285c9", "bag of ginger-d91eb0cfc44f83bd", "pack of dried hibiscus flowers-dc02bcfa55ddf1de", "saucepan cover-ea642f32c4e6490d", "some dried hibiscus flowers-47389b49cd317eb9", "knife-c88b7b52fc975faf", "chunks of sliced potato-8b06f30ac9601bc9", "potato bits-918cf9c5f49a8dff", "potato-0121203fec36c708", "more potato bits-8beb52e430b0bc78", "more potato bits2-11e03b271b1e0134", "chicken breast-04e0d6d6b2971042", "chicken breast2-3c4cc2e9007379e1", "chicken chunk-fcd07ee885138d63", "chicken strip-661cea196e0d16e3", "chicken strip2-02f73767db97f73b", "chicken strip3-ea82baf3e87b6299", "chicken strip4-7a84a44acc9fac15", "chicken strips5-8b4dde3cb0e45483", "spoon 1-c162f4c69a135827", "plastic chopping board-aff6cf3571b5e746", "potato peels-a129023c2e975289", "slice of potato peel-a7c2d019073e057e", "egg shell-f37d5171d244b593", "pair of tongs-b2cb60919ca4589b", "bottle of oil-429f4383d7550753", "table spoon-1e1514d81bd95cbf", "fork-0ccf59091cbd3221", "fork2-16f46ef8e1f52cc8", "potato masher-8ecaad0bee4a2482", "bag of cheese-05b11d84f980233b", "bowl5-4a909ddf142486d7", "fork3-acee27d6717a322b", "serving spoon-db3fc3c2d7b899f4", "foil-efd9a8b5e899fe1e", "packet-a1bb131de6a58410", "potato chicken ball-a396ef3f39417275", "potato chicken ball2-fd1ff80272e8d015", "potato chicken ball3-36f01a64517663bb", "potato chicken ball4-4ce990da51f12cbf", "potato chicken ball5-b9b830a0af24d7a3", "potato chicken ball6-8b9a9e03e054d476", "potato chicken ball7-ef7ef274207eec42" ], "reasoning": "1. I look for each time the user stirs the pot (saucepan-eaaa0de116893699) with ginger (ginger-68836b8eddf285c9) and hibiscus flower (some dried hibiscus flowers-47389b49cd317eb9), and I find 3 separate times: 1 from 16:15-16:26, another from 18:59-19:07, and a final one from 20:32-20:41.\n2. I look at the first time 16:15-16:26, and I examine how many clockwise stirs the user makes.\n3. I count 5 clockwise stirs.\n4. I look at the second time from 18:59-19:07.\n5. I count 4.5 counterclockwise stirs, but according to the rules of the question I round up to 5.\n6. Calculating the difference, I realize that the user stirs the pot (saucepan-eaaa0de116893699) 0 more times clockwise the first time stirring than counterclockwise the second time.\n7. Therefore, the answer is 0.", "referenced_object_masks": [ "saucepan-eaaa0de116893699", "some dried hibiscus flowers-47389b49cd317eb9", "ginger-68836b8eddf285c9" ] }, { "key": "P09-20240622-150155:54519e69999f452ff59a4e5369bb8d43702ecc7f", "video_id": "P09-20240622-150155", "question": "If the bowl of egg mixture was marked a 1, the square tupperware a 2, the bowl of bread crumbs marked a 3, and the second bowl of flour marked a 4, in what order do I pick from these bowls to finish the first and second chicken-potato balls I make?", "answer_choice_0": "4, 2, 3, 1: 2, 1, 3", "answer_choice_1": "2, 4, 1, 3: 2, 1, 3", "answer_choice_2": "4, 2, 3, 1: 1, 2, 3", "answer_choice_3": "3, 1, 4, 2: 1, 3, 2, 4", "answer_choice_4": "1, 3, 4, 2: 4, 1, 2, 3", "answer_id": 1, "answer": "2, 4, 1, 3: 2, 1, 3", "question_type": [ "Counterfactual", "Action Recognition", "Object Recognition", "Counting", "Temporal Reasoning" ], "objects_of_interest": [ "saucepan-eaaa0de116893699", "frying pan-d510045435eca92b", "bowl-ea981a91dfb4db94", "bag of chicken-fe3ba4b0a41f9b19", "chicken-1eb0ba2f3be85466", "pyrex bowl-7fd62b5c0e617c12", "discarded chicken bag-669e77c13d83a0c6", "towel-06d457c3d18b0910", "towel2-fea06dac4ea4a30f", "potato peeler-c319f80b11a0a592", "cup of bread crumbs-21252de25de2d8ad", "cup of bread crumbs cap-5d84b03bd77bef5f", "scale-f09148f04fa5d168", "bowl2-d2721bd426e5c19c", "bowl3-34c64810cbc69dc4", "bag of flour-0890c068caf37f46", "pouch of rice-c95b6d502de90cdc", "bottle of ground chili-57ec31dc93a4c02b", "teaspoon-e94f9e5636637939", "bottle of garlic powder-ce5db1acdb4c63a1", "bottle of ground black pepper-ee2a4084493bb086", "bottle of salt-0434f4abf51cc5a5", "tupperware-f062676dc1eb2055", "bowl4-9d7df688cdef6926", "egg-5deb0b311b0b6c63", "egg2-e293348c62e7f37f", "electric kettle-91e63c2cb8515de3", "pot cover-3fda277021cf3664", "saucepan2-85b54cc17ec1671a", "red saucepan-31701be2c3ca6617", "pot cover2-a532e882a3a733be", "bottle of washing up liquid-747a3f68e8427a9c", "sponge-8090a791330975b2", "spoon-a5bc07287813f070", "potatoes-bf87bed0ccee3f9d", "ginger-68836b8eddf285c9", "bag of ginger-d91eb0cfc44f83bd", "pack of dried hibiscus flowers-dc02bcfa55ddf1de", "saucepan cover-ea642f32c4e6490d", "some dried hibiscus flowers-47389b49cd317eb9", "knife-c88b7b52fc975faf", "chunks of sliced potato-8b06f30ac9601bc9", "potato bits-918cf9c5f49a8dff", "potato-0121203fec36c708", "more potato bits-8beb52e430b0bc78", "more potato bits2-11e03b271b1e0134", "chicken breast-04e0d6d6b2971042", "chicken breast2-3c4cc2e9007379e1", "chicken chunk-fcd07ee885138d63", "chicken strip-661cea196e0d16e3", "chicken strip2-02f73767db97f73b", "chicken strip3-ea82baf3e87b6299", "chicken strip4-7a84a44acc9fac15", "chicken strips5-8b4dde3cb0e45483", "spoon 1-c162f4c69a135827", "plastic chopping board-aff6cf3571b5e746", "potato peels-a129023c2e975289", "slice of potato peel-a7c2d019073e057e", "egg shell-f37d5171d244b593", "pair of tongs-b2cb60919ca4589b", "bottle of oil-429f4383d7550753", "table spoon-1e1514d81bd95cbf", "fork-0ccf59091cbd3221", "fork2-16f46ef8e1f52cc8", "potato masher-8ecaad0bee4a2482", "bag of cheese-05b11d84f980233b", "bowl5-4a909ddf142486d7", "fork3-acee27d6717a322b", "serving spoon-db3fc3c2d7b899f4", "foil-efd9a8b5e899fe1e", "packet-a1bb131de6a58410", "potato chicken ball-a396ef3f39417275", "potato chicken ball2-fd1ff80272e8d015", "potato chicken ball3-36f01a64517663bb", "potato chicken ball4-4ce990da51f12cbf", "potato chicken ball5-b9b830a0af24d7a3", "potato chicken ball6-8b9a9e03e054d476", "potato chicken ball7-ef7ef274207eec42" ], "reasoning": "1. I look for the time the user creates their first and second potato balls (potato chicken ball-a396ef3f39417275, potato chicken ball2-fd1ff80272e8d015), finding the time from 36:20-37:47.\n2. During this time range, I notice a bowl (bowl2-d2721bd426e5c19c) of egg mixture , a square tupperware (tupperware-f062676dc1eb2055) filled with flour , a bowl (bowl-ea981a91dfb4db94) of bread crumbs , and another bowl (bowl4-9d7df688cdef6926) of flour sitting just above the aluminum foil where the chicken-potato balls (potato chicken ball-a396ef3f39417275, potato chicken ball2-fd1ff80272e8d015) are made.\n3. I mentally mark the bowl (bowl2-d2721bd426e5c19c) of egg mixture with a 1, the tupperware (tupperware-f062676dc1eb2055) with a 2, the bowl (bowl-ea981a91dfb4db94) of bread crumbs with a 3, and the second bowl (bowl4-9d7df688cdef6926) of flour with a 4.\n4. I notice that, in creating the first chicken-potato ball (potato chicken ball-a396ef3f39417275) from 36:20-37:04, the user grabs from the square tupperware (tupperware-f062676dc1eb2055) first at 36:33, the bowl (bowl4-9d7df688cdef6926) of flour second from 36:35-36:36; then the user places the ball (potato chicken ball-a396ef3f39417275) in the egg mixture third at 36:47, and the bowl (bowl-ea981a91dfb4db94) of bread crumbs last at 36:51.\n5. I reframe this order with the numeric values suggested by the question and I come to an order of 2, 4, 1, 3.\n6. I look at the second time the user makes a chicken potato ball (potato chicken ball2-fd1ff80272e8d015), from 37:06-37:47, and I examine the order of ingredients used. At 37:20, the user sprinkles flour from the tupperware (tupperware-f062676dc1eb2055) onto the ball (potato chicken ball2-fd1ff80272e8d015). At 37:32, the user places the ball (potato chicken ball2-fd1ff80272e8d015) in the egg mixture , and, at 37:36, the user places the ball (potato chicken ball2-fd1ff80272e8d015) in the bowl (bowl-ea981a91dfb4db94) of bread crumbs .\n7. Reframing this order with the numeric values suggested by the question gives me a 2, 1, 3.\n8. Therefore, 2, 4, 1, 3; 2, 1, 3 is the answer.", "referenced_object_masks": [ "bowl-ea981a91dfb4db94", "potato chicken ball2-fd1ff80272e8d015", "bowl2-d2721bd426e5c19c", "tupperware-f062676dc1eb2055", "bowl4-9d7df688cdef6926", "potato chicken ball-a396ef3f39417275" ] }, { "key": "P09-20240622-150155:73bf9f1f812a4659818d99da00100a6c6ed09d74", "video_id": "P09-20240622-150155", "question": "Which is greater, and by how much: The amount of time (in seconds) I spent whisking eggs, or the time (in seconds) it took to prepare the last chicken potato ball?", "answer_choice_0": "Whisking eggs--by 8", "answer_choice_1": "Neither--the times were the same", "answer_choice_2": "Preparing last chicken potato ball--by 8", "answer_choice_3": "Preparing last chicken potato ball--by 4", "answer_choice_4": "Whisking eggs--by 4", "answer_id": 2, "answer": "Preparing last chicken potato ball--by 8", "question_type": [ "Action Recognition", "Counting", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "saucepan-eaaa0de116893699", "frying pan-d510045435eca92b", "bowl-ea981a91dfb4db94", "bag of chicken-fe3ba4b0a41f9b19", "chicken-1eb0ba2f3be85466", "pyrex bowl-7fd62b5c0e617c12", "discarded chicken bag-669e77c13d83a0c6", "towel-06d457c3d18b0910", "towel2-fea06dac4ea4a30f", "potato peeler-c319f80b11a0a592", "cup of bread crumbs-21252de25de2d8ad", "cup of bread crumbs cap-5d84b03bd77bef5f", "scale-f09148f04fa5d168", "bowl2-d2721bd426e5c19c", "bowl3-34c64810cbc69dc4", "bag of flour-0890c068caf37f46", "pouch of rice-c95b6d502de90cdc", "bottle of ground chili-57ec31dc93a4c02b", "teaspoon-e94f9e5636637939", "bottle of garlic powder-ce5db1acdb4c63a1", "bottle of ground black pepper-ee2a4084493bb086", "bottle of salt-0434f4abf51cc5a5", "tupperware-f062676dc1eb2055", "bowl4-9d7df688cdef6926", "egg-5deb0b311b0b6c63", "egg2-e293348c62e7f37f", "electric kettle-91e63c2cb8515de3", "pot cover-3fda277021cf3664", "saucepan2-85b54cc17ec1671a", "red saucepan-31701be2c3ca6617", "pot cover2-a532e882a3a733be", "bottle of washing up liquid-747a3f68e8427a9c", "sponge-8090a791330975b2", "spoon-a5bc07287813f070", "potatoes-bf87bed0ccee3f9d", "ginger-68836b8eddf285c9", "bag of ginger-d91eb0cfc44f83bd", "pack of dried hibiscus flowers-dc02bcfa55ddf1de", "saucepan cover-ea642f32c4e6490d", "some dried hibiscus flowers-47389b49cd317eb9", "knife-c88b7b52fc975faf", "chunks of sliced potato-8b06f30ac9601bc9", "potato bits-918cf9c5f49a8dff", "potato-0121203fec36c708", "more potato bits-8beb52e430b0bc78", "more potato bits2-11e03b271b1e0134", "chicken breast-04e0d6d6b2971042", "chicken breast2-3c4cc2e9007379e1", "chicken chunk-fcd07ee885138d63", "chicken strip-661cea196e0d16e3", "chicken strip2-02f73767db97f73b", "chicken strip3-ea82baf3e87b6299", "chicken strip4-7a84a44acc9fac15", "chicken strips5-8b4dde3cb0e45483", "spoon 1-c162f4c69a135827", "plastic chopping board-aff6cf3571b5e746", "potato peels-a129023c2e975289", "slice of potato peel-a7c2d019073e057e", "egg shell-f37d5171d244b593", "pair of tongs-b2cb60919ca4589b", "bottle of oil-429f4383d7550753", "table spoon-1e1514d81bd95cbf", "fork-0ccf59091cbd3221", "fork2-16f46ef8e1f52cc8", "potato masher-8ecaad0bee4a2482", "bag of cheese-05b11d84f980233b", "bowl5-4a909ddf142486d7", "fork3-acee27d6717a322b", "serving spoon-db3fc3c2d7b899f4", "foil-efd9a8b5e899fe1e", "packet-a1bb131de6a58410", "potato chicken ball-a396ef3f39417275", "potato chicken ball2-fd1ff80272e8d015", "potato chicken ball3-36f01a64517663bb", "potato chicken ball4-4ce990da51f12cbf", "potato chicken ball5-b9b830a0af24d7a3", "potato chicken ball6-8b9a9e03e054d476", "potato chicken ball7-ef7ef274207eec42" ], "reasoning": "1. I looked for when the user whisked a bowl of eggs . From 05:48-05:50, I saw the user crack 2 eggs (egg-5deb0b311b0b6c63, egg2-e293348c62e7f37f) and add the yolks into a bowl (bowl4-9d7df688cdef6926). The user whisked the bowl of eggs from 06:17-06:42. \n2. I found the duration by subtracting the start time from the end time: 06:42-06:17=00:25.\n3. I saw the user prepare 7 chicken potato balls (potato chicken ball-a396ef3f39417275, potato chicken ball2-fd1ff80272e8d015, potato chicken ball3-36f01a64517663bb, potato chicken ball4-4ce990da51f12cbf, potato chicken ball5-b9b830a0af24d7a3, potato chicken ball6-8b9a9e03e054d476, potato chicken ball7-ef7ef274207eec42). The user prepared the seventh chicken potato ball (potato chicken ball7-ef7ef274207eec42) from 40:30-41:03. \n4. I found the duration by subtracting the start time from the end time: 41:03-40:30=00:33. \n5. I learned that preparing the last chicken ball took longer.\n6. I also found the difference between the two durations: 00:33-00:25=00:08.", "referenced_object_masks": [ "potato chicken ball-a396ef3f39417275", "potato chicken ball2-fd1ff80272e8d015", "potato chicken ball3-36f01a64517663bb", "potato chicken ball4-4ce990da51f12cbf", "potato chicken ball5-b9b830a0af24d7a3", "potato chicken ball6-8b9a9e03e054d476", "potato chicken ball7-ef7ef274207eec42", "egg-5deb0b311b0b6c63", "egg2-e293348c62e7f37f", "bowl4-9d7df688cdef6926" ] }, { "key": "P09-20240622-150155:9592569957b5d7ac3abaacbaf90fa597d879a192", "video_id": "P09-20240622-150155", "question": "Rounding to the nearest whole number, what is the average (mean) time it took me to prepare (roll, dredge, dip, and coat in seasoning) each chicken potato ball?", "answer_choice_0": "22 seconds", "answer_choice_1": "40 seconds", "answer_choice_2": "34 seconds", "answer_choice_3": "28 seconds", "answer_choice_4": "46 seconds", "answer_id": 2, "answer": "34 seconds", "question_type": [ "Action Recognition", "Counting", "Object Recognition", "Numerical Reasoning", "Temporal Reasoning", "State Changes" ], "objects_of_interest": [ "saucepan-eaaa0de116893699", "frying pan-d510045435eca92b", "bowl-ea981a91dfb4db94", "bag of chicken-fe3ba4b0a41f9b19", "chicken-1eb0ba2f3be85466", "pyrex bowl-7fd62b5c0e617c12", "discarded chicken bag-669e77c13d83a0c6", "towel-06d457c3d18b0910", "towel2-fea06dac4ea4a30f", "potato peeler-c319f80b11a0a592", "cup of bread crumbs-21252de25de2d8ad", "cup of bread crumbs cap-5d84b03bd77bef5f", "scale-f09148f04fa5d168", "bowl2-d2721bd426e5c19c", "bowl3-34c64810cbc69dc4", "bag of flour-0890c068caf37f46", "pouch of rice-c95b6d502de90cdc", "bottle of ground chili-57ec31dc93a4c02b", "teaspoon-e94f9e5636637939", "bottle of garlic powder-ce5db1acdb4c63a1", "bottle of ground black pepper-ee2a4084493bb086", "bottle of salt-0434f4abf51cc5a5", "tupperware-f062676dc1eb2055", "bowl4-9d7df688cdef6926", "egg-5deb0b311b0b6c63", "egg2-e293348c62e7f37f", "electric kettle-91e63c2cb8515de3", "pot cover-3fda277021cf3664", "saucepan2-85b54cc17ec1671a", "red saucepan-31701be2c3ca6617", "pot cover2-a532e882a3a733be", "bottle of washing up liquid-747a3f68e8427a9c", "sponge-8090a791330975b2", "spoon-a5bc07287813f070", "potatoes-bf87bed0ccee3f9d", "ginger-68836b8eddf285c9", "bag of ginger-d91eb0cfc44f83bd", "pack of dried hibiscus flowers-dc02bcfa55ddf1de", "saucepan cover-ea642f32c4e6490d", "some dried hibiscus flowers-47389b49cd317eb9", "knife-c88b7b52fc975faf", "chunks of sliced potato-8b06f30ac9601bc9", "potato bits-918cf9c5f49a8dff", "potato-0121203fec36c708", "more potato bits-8beb52e430b0bc78", "more potato bits2-11e03b271b1e0134", "chicken breast-04e0d6d6b2971042", "chicken breast2-3c4cc2e9007379e1", "chicken chunk-fcd07ee885138d63", "chicken strip-661cea196e0d16e3", "chicken strip2-02f73767db97f73b", "chicken strip3-ea82baf3e87b6299", "chicken strip4-7a84a44acc9fac15", "chicken strips5-8b4dde3cb0e45483", "spoon 1-c162f4c69a135827", "plastic chopping board-aff6cf3571b5e746", "potato peels-a129023c2e975289", "slice of potato peel-a7c2d019073e057e", "egg shell-f37d5171d244b593", "pair of tongs-b2cb60919ca4589b", "bottle of oil-429f4383d7550753", "table spoon-1e1514d81bd95cbf", "fork-0ccf59091cbd3221", "fork2-16f46ef8e1f52cc8", "potato masher-8ecaad0bee4a2482", "bag of cheese-05b11d84f980233b", "bowl5-4a909ddf142486d7", "fork3-acee27d6717a322b", "serving spoon-db3fc3c2d7b899f4", "foil-efd9a8b5e899fe1e", "packet-a1bb131de6a58410", "potato chicken ball-a396ef3f39417275", "potato chicken ball2-fd1ff80272e8d015", "potato chicken ball3-36f01a64517663bb", "potato chicken ball4-4ce990da51f12cbf", "potato chicken ball5-b9b830a0af24d7a3", "potato chicken ball6-8b9a9e03e054d476", "potato chicken ball7-ef7ef274207eec42" ], "reasoning": "1. I watched the video and found that the user began preparing the first chicken potato ball (potato chicken ball-a396ef3f39417275) at 36:25 and finished it at 37:04.\n2. I saw the user prepare the second chicken potato ball (potato chicken ball2-fd1ff80272e8d015) from 37:10-37:46. \n3. I saw the user prepare the third chicken potato ball (potato chicken ball3-36f01a64517663bb) from 37:55-38:32. \n4. I saw the user prepare the fourth chicken potato ball (potato chicken ball4-4ce990da51f12cbf) from 38:40-39:06.\n5. I saw the user prepare the fifth chicken potato ball (potato chicken ball5-b9b830a0af24d7a3) from 39:13-39:46. \n6. I saw the user prepare the sixth chicken potato ball (potato chicken ball6-8b9a9e03e054d476) from 39:51-40:23. \n7. I saw the user prepare the seventh chicken potato ball (potato chicken ball7-ef7ef274207eec42) from 40:30-41:03. \n8. I subtracted the start times from the end times for each occurrence: 37:04-36:25=00:39; 37:46-37:10=00:36; 38:32-37:55=00:37; 39:06-38:40=00:26; 39:46-39:13=00:33; 40:23-39:51=00:32; 41:03-40:30=00:33. \n9. I took the average (mean) of all 7 times: (39+36+37+26+33+32+33)/7=33.71.\n10. I rounded the average to the nearest whole number: 34.", "referenced_object_masks": [ "potato chicken ball-a396ef3f39417275", "potato chicken ball6-8b9a9e03e054d476", "potato chicken ball3-36f01a64517663bb", "potato chicken ball2-fd1ff80272e8d015", "potato chicken ball5-b9b830a0af24d7a3", "potato chicken ball4-4ce990da51f12cbf", "potato chicken ball7-ef7ef274207eec42" ] }, { "key": "P09-20240622-150155:a109233906aa90ded01d8ed407b14fc125b2dcaa", "video_id": "P09-20240622-150155", "question": "How much time passed between my first time picking up the saucepan I used to cook potatoes and picking up the potato masher?", "answer_choice_0": "16 minutes and 22 seconds", "answer_choice_1": "4 minutes and 51 seconds", "answer_choice_2": "11 seconds", "answer_choice_3": "13 minutes and 9 seconds", "answer_choice_4": "19 minutes and 39 seconds", "answer_id": 3, "answer": "13 minutes and 9 seconds", "question_type": [ "Temporal Reasoning", "Numerical Reasoning", "Action Recognition", "Object Recognition" ], "objects_of_interest": [ "saucepan-eaaa0de116893699", "frying pan-d510045435eca92b", "bowl-ea981a91dfb4db94", "bag of chicken-fe3ba4b0a41f9b19", "chicken-1eb0ba2f3be85466", "pyrex bowl-7fd62b5c0e617c12", "discarded chicken bag-669e77c13d83a0c6", "towel-06d457c3d18b0910", "towel2-fea06dac4ea4a30f", "potato peeler-c319f80b11a0a592", "cup of bread crumbs-21252de25de2d8ad", "cup of bread crumbs cap-5d84b03bd77bef5f", "scale-f09148f04fa5d168", "bowl2-d2721bd426e5c19c", "bowl3-34c64810cbc69dc4", "bag of flour-0890c068caf37f46", "pouch of rice-c95b6d502de90cdc", "bottle of ground chili-57ec31dc93a4c02b", "teaspoon-e94f9e5636637939", "bottle of garlic powder-ce5db1acdb4c63a1", "bottle of ground black pepper-ee2a4084493bb086", "bottle of salt-0434f4abf51cc5a5", "tupperware-f062676dc1eb2055", "bowl4-9d7df688cdef6926", "egg-5deb0b311b0b6c63", "egg2-e293348c62e7f37f", "electric kettle-91e63c2cb8515de3", "pot cover-3fda277021cf3664", "saucepan2-85b54cc17ec1671a", "red saucepan-31701be2c3ca6617", "pot cover2-a532e882a3a733be", "bottle of washing up liquid-747a3f68e8427a9c", "sponge-8090a791330975b2", "spoon-a5bc07287813f070", "potatoes-bf87bed0ccee3f9d", "ginger-68836b8eddf285c9", "bag of ginger-d91eb0cfc44f83bd", "pack of dried hibiscus flowers-dc02bcfa55ddf1de", "saucepan cover-ea642f32c4e6490d", "some dried hibiscus flowers-47389b49cd317eb9", "knife-c88b7b52fc975faf", "chunks of sliced potato-8b06f30ac9601bc9", "potato bits-918cf9c5f49a8dff", "potato-0121203fec36c708", "more potato bits-8beb52e430b0bc78", "more potato bits2-11e03b271b1e0134", "chicken breast-04e0d6d6b2971042", "chicken breast2-3c4cc2e9007379e1", "chicken chunk-fcd07ee885138d63", "chicken strip-661cea196e0d16e3", "chicken strip2-02f73767db97f73b", "chicken strip3-ea82baf3e87b6299", "chicken strip4-7a84a44acc9fac15", "chicken strips5-8b4dde3cb0e45483", "spoon 1-c162f4c69a135827", "plastic chopping board-aff6cf3571b5e746", "potato peels-a129023c2e975289", "slice of potato peel-a7c2d019073e057e", "egg shell-f37d5171d244b593", "pair of tongs-b2cb60919ca4589b", "bottle of oil-429f4383d7550753", "table spoon-1e1514d81bd95cbf", "fork-0ccf59091cbd3221", "fork2-16f46ef8e1f52cc8", "potato masher-8ecaad0bee4a2482", "bag of cheese-05b11d84f980233b", "bowl5-4a909ddf142486d7", "fork3-acee27d6717a322b", "serving spoon-db3fc3c2d7b899f4", "foil-efd9a8b5e899fe1e", "packet-a1bb131de6a58410", "potato chicken ball-a396ef3f39417275", "potato chicken ball2-fd1ff80272e8d015", "potato chicken ball3-36f01a64517663bb", "potato chicken ball4-4ce990da51f12cbf", "potato chicken ball5-b9b830a0af24d7a3", "potato chicken ball6-8b9a9e03e054d476", "potato chicken ball7-ef7ef274207eec42" ], "reasoning": "1. I watched the video and looked for when the user cooks potatoes (potatoes-bf87bed0ccee3f9d, chunks of sliced potato-8b06f30ac9601bc9, potato-0121203fec36c708). From 13:46-14:55 I saw the user chop 3 potatoes (potatoes-bf87bed0ccee3f9d, chunks of sliced potato-8b06f30ac9601bc9, potato-0121203fec36c708) and add the pieces to a small saucepan (saucepan2-85b54cc17ec1671a). I saw the small saucepan (saucepan2-85b54cc17ec1671a) on the stove until the user removed it at 20:14. \n2. I went back in the video and looked for the first time the user picked up the small saucepan (saucepan2-85b54cc17ec1671a). This occurred at 06:53. \n3. I looked for when the user picked up the potato masher (potato masher-8ecaad0bee4a2482) and found that this occurred at 20:02. \n4. I took the difference between picking up the small saucepan (saucepan2-85b54cc17ec1671a) and the potato masher (potato masher-8ecaad0bee4a2482): 20:02-06:53=13:09, for a difference of 13 minutes and 9 seconds.", "referenced_object_masks": [ "saucepan2-85b54cc17ec1671a", "potato masher-8ecaad0bee4a2482", "potatoes-bf87bed0ccee3f9d", "chunks of sliced potato-8b06f30ac9601bc9", "potato-0121203fec36c708" ] }, { "key": "P09-20240622-150155:e391c2aa46a4fa2518564163a698209e78d8de94", "video_id": "P09-20240622-150155", "question": "When I whisked the eggs, approximately how many steps would I take, and in which direction, to face the place where I first found the saucepan I used to cook potatoes?", "answer_choice_0": "1-2 steps forward", "answer_choice_1": "1-2 steps right", "answer_choice_2": "1-2 steps left", "answer_choice_3": "5-6 steps right", "answer_choice_4": "5-6 steps left", "answer_id": 1, "answer": "1-2 steps right", "question_type": [ "Egocentric & Spatial Perception", "Action Recognition", "Object Recognition", "Temporal Reasoning", "Counterfactual" ], "objects_of_interest": [ "saucepan-eaaa0de116893699", "frying pan-d510045435eca92b", "bowl-ea981a91dfb4db94", "bag of chicken-fe3ba4b0a41f9b19", "chicken-1eb0ba2f3be85466", "pyrex bowl-7fd62b5c0e617c12", "discarded chicken bag-669e77c13d83a0c6", "towel-06d457c3d18b0910", "towel2-fea06dac4ea4a30f", "potato peeler-c319f80b11a0a592", "cup of bread crumbs-21252de25de2d8ad", "cup of bread crumbs cap-5d84b03bd77bef5f", "scale-f09148f04fa5d168", "bowl2-d2721bd426e5c19c", "bowl3-34c64810cbc69dc4", "bag of flour-0890c068caf37f46", "pouch of rice-c95b6d502de90cdc", "bottle of ground chili-57ec31dc93a4c02b", "teaspoon-e94f9e5636637939", "bottle of garlic powder-ce5db1acdb4c63a1", "bottle of ground black pepper-ee2a4084493bb086", "bottle of salt-0434f4abf51cc5a5", "tupperware-f062676dc1eb2055", "bowl4-9d7df688cdef6926", "egg-5deb0b311b0b6c63", "egg2-e293348c62e7f37f", "electric kettle-91e63c2cb8515de3", "pot cover-3fda277021cf3664", "saucepan2-85b54cc17ec1671a", "red saucepan-31701be2c3ca6617", "pot cover2-a532e882a3a733be", "bottle of washing up liquid-747a3f68e8427a9c", "sponge-8090a791330975b2", "spoon-a5bc07287813f070", "potatoes-bf87bed0ccee3f9d", "ginger-68836b8eddf285c9", "bag of ginger-d91eb0cfc44f83bd", "pack of dried hibiscus flowers-dc02bcfa55ddf1de", "saucepan cover-ea642f32c4e6490d", "some dried hibiscus flowers-47389b49cd317eb9", "knife-c88b7b52fc975faf", "chunks of sliced potato-8b06f30ac9601bc9", "potato bits-918cf9c5f49a8dff", "potato-0121203fec36c708", "more potato bits-8beb52e430b0bc78", "more potato bits2-11e03b271b1e0134", "chicken breast-04e0d6d6b2971042", "chicken breast2-3c4cc2e9007379e1", "chicken chunk-fcd07ee885138d63", "chicken strip-661cea196e0d16e3", "chicken strip2-02f73767db97f73b", "chicken strip3-ea82baf3e87b6299", "chicken strip4-7a84a44acc9fac15", "chicken strips5-8b4dde3cb0e45483", "spoon 1-c162f4c69a135827", "plastic chopping board-aff6cf3571b5e746", "potato peels-a129023c2e975289", "slice of potato peel-a7c2d019073e057e", "egg shell-f37d5171d244b593", "pair of tongs-b2cb60919ca4589b", "bottle of oil-429f4383d7550753", "table spoon-1e1514d81bd95cbf", "fork-0ccf59091cbd3221", "fork2-16f46ef8e1f52cc8", "potato masher-8ecaad0bee4a2482", "bag of cheese-05b11d84f980233b", "bowl5-4a909ddf142486d7", "fork3-acee27d6717a322b", "serving spoon-db3fc3c2d7b899f4", "foil-efd9a8b5e899fe1e", "packet-a1bb131de6a58410", "potato chicken ball-a396ef3f39417275", "potato chicken ball2-fd1ff80272e8d015", "potato chicken ball3-36f01a64517663bb", "potato chicken ball4-4ce990da51f12cbf", "potato chicken ball5-b9b830a0af24d7a3", "potato chicken ball6-8b9a9e03e054d476", "potato chicken ball7-ef7ef274207eec42" ], "reasoning": "1. I watched the video and looked for when the user cooks potatoes (potatoes-bf87bed0ccee3f9d, chunks of sliced potato-8b06f30ac9601bc9, potato-0121203fec36c708). From 13:46-14:55 I saw the user chop 3 potatoes (potatoes-bf87bed0ccee3f9d, chunks of sliced potato-8b06f30ac9601bc9, potato-0121203fec36c708) and add the pieces to a small saucepan (saucepan2-85b54cc17ec1671a). I saw the small saucepan (saucepan2-85b54cc17ec1671a) on the stove until the user removed it at 20:14. \n2. I looked for the first time the user picked up the small saucepan (saucepan2-85b54cc17ec1671a). This occurred at 06:53. \n3. I observed the surroundings at 06:53 and saw that the user was crouched down in front of the oven and an open cabinet. \n4. I looked for when the user whisked a bowl of eggs . From 05:48-05:50, I saw the user crack 2 eggs (egg-5deb0b311b0b6c63, egg2-e293348c62e7f37f) and add the yolks into a bowl (bowl4-9d7df688cdef6926). The user whisked the bowl of eggs from 06:17-06:42. \n5. I observed the surroundings from 06:17-06:42. I noticed that the user was standing in front of the oven, near the front left corner of the oven. \n6. Therefore, when the user whisked the bowl of eggs they were a couple steps to the left of where they found the small saucepan (saucepan2-85b54cc17ec1671a) used to cook the potatoes (potatoes-bf87bed0ccee3f9d, chunks of sliced potato-8b06f30ac9601bc9, potato-0121203fec36c708).", "referenced_object_masks": [ "egg-5deb0b311b0b6c63", "egg2-e293348c62e7f37f", "saucepan2-85b54cc17ec1671a", "bowl4-9d7df688cdef6926", "potatoes-bf87bed0ccee3f9d", "chunks of sliced potato-8b06f30ac9601bc9", "potato-0121203fec36c708" ] }, { "key": "P09-20240622-150155:eb43b1c585f63d65d59278515702898bbedeb058", "video_id": "P09-20240622-150155", "question": "How many times am I able to slice a potato with a potato peeler before the kettle is finished heating up water the first time?", "answer_choice_0": "85 times", "answer_choice_1": "44 times", "answer_choice_2": "55 times", "answer_choice_3": "67 times", "answer_choice_4": "80 times", "answer_id": 3, "answer": "67 times", "question_type": [ "Counting", "Action Recognition", "Object Recognition", "Situational Awareness", "State Changes", "Temporal Reasoning" ], "objects_of_interest": [ "saucepan-eaaa0de116893699", "frying pan-d510045435eca92b", "bowl-ea981a91dfb4db94", "bag of chicken-fe3ba4b0a41f9b19", "chicken-1eb0ba2f3be85466", "pyrex bowl-7fd62b5c0e617c12", "discarded chicken bag-669e77c13d83a0c6", "towel-06d457c3d18b0910", "towel2-fea06dac4ea4a30f", "potato peeler-c319f80b11a0a592", "cup of bread crumbs-21252de25de2d8ad", "cup of bread crumbs cap-5d84b03bd77bef5f", "scale-f09148f04fa5d168", "bowl2-d2721bd426e5c19c", "bowl3-34c64810cbc69dc4", "bag of flour-0890c068caf37f46", "pouch of rice-c95b6d502de90cdc", "bottle of ground chili-57ec31dc93a4c02b", "teaspoon-e94f9e5636637939", "bottle of garlic powder-ce5db1acdb4c63a1", "bottle of ground black pepper-ee2a4084493bb086", "bottle of salt-0434f4abf51cc5a5", "tupperware-f062676dc1eb2055", "bowl4-9d7df688cdef6926", "egg-5deb0b311b0b6c63", "egg2-e293348c62e7f37f", "electric kettle-91e63c2cb8515de3", "pot cover-3fda277021cf3664", "saucepan2-85b54cc17ec1671a", "red saucepan-31701be2c3ca6617", "pot cover2-a532e882a3a733be", "bottle of washing up liquid-747a3f68e8427a9c", "sponge-8090a791330975b2", "spoon-a5bc07287813f070", "potatoes-bf87bed0ccee3f9d", "ginger-68836b8eddf285c9", "bag of ginger-d91eb0cfc44f83bd", "pack of dried hibiscus flowers-dc02bcfa55ddf1de", "saucepan cover-ea642f32c4e6490d", "some dried hibiscus flowers-47389b49cd317eb9", "knife-c88b7b52fc975faf", "chunks of sliced potato-8b06f30ac9601bc9", "potato bits-918cf9c5f49a8dff", "potato-0121203fec36c708", "more potato bits-8beb52e430b0bc78", "more potato bits2-11e03b271b1e0134", "chicken breast-04e0d6d6b2971042", "chicken breast2-3c4cc2e9007379e1", "chicken chunk-fcd07ee885138d63", "chicken strip-661cea196e0d16e3", "chicken strip2-02f73767db97f73b", "chicken strip3-ea82baf3e87b6299", "chicken strip4-7a84a44acc9fac15", "chicken strips5-8b4dde3cb0e45483", "spoon 1-c162f4c69a135827", "plastic chopping board-aff6cf3571b5e746", "potato peels-a129023c2e975289", "slice of potato peel-a7c2d019073e057e", "egg shell-f37d5171d244b593", "pair of tongs-b2cb60919ca4589b", "bottle of oil-429f4383d7550753", "table spoon-1e1514d81bd95cbf", "fork-0ccf59091cbd3221", "fork2-16f46ef8e1f52cc8", "potato masher-8ecaad0bee4a2482", "bag of cheese-05b11d84f980233b", "bowl5-4a909ddf142486d7", "fork3-acee27d6717a322b", "serving spoon-db3fc3c2d7b899f4", "foil-efd9a8b5e899fe1e", "packet-a1bb131de6a58410", "potato chicken ball-a396ef3f39417275", "potato chicken ball2-fd1ff80272e8d015", "potato chicken ball3-36f01a64517663bb", "potato chicken ball4-4ce990da51f12cbf", "potato chicken ball5-b9b830a0af24d7a3", "potato chicken ball6-8b9a9e03e054d476", "potato chicken ball7-ef7ef274207eec42" ], "reasoning": "1. I look for the first time the user places the kettle (electric kettle-91e63c2cb8515de3) on the station to heat it up, finding my time at 06:07, when I hear the click of the kettle (electric kettle-91e63c2cb8515de3) in its station . I look for the time the user takes the kettle (electric kettle-91e63c2cb8515de3) off the station and I find it at 09:40.\n2. I count the number of peels the user is able to perform on a potato with a potato peeler (potato peeler-c319f80b11a0a592) from 06:07-09:40, and notice that the user peels 2 potatoes (potatoes-bf87bed0ccee3f9d, chunks of sliced potato-8b06f30ac9601bc9) from 08:44-9:37.\n3. I count all the peels made during this time, and I come to a total of 67 times.\n4. Therefore, 67 is the answer.", "referenced_object_masks": [ "electric kettle-91e63c2cb8515de3", "potato peeler-c319f80b11a0a592", "potatoes-bf87bed0ccee3f9d", "chunks of sliced potato-8b06f30ac9601bc9" ] }, { "key": "P09-20240622-150155:ec660e796be0718f908438aaeedec1670bcd5498", "video_id": "P09-20240622-150155", "question": "Assuming that a \"dedicated wipe down\" of a counter surface means to wipe down a countertop for longer than 3 seconds, and assuming the stovetop counts as the center of the countertop, how many dedicated wipedowns do I make on the left side of the countertop with a towel?", "answer_choice_0": "3", "answer_choice_1": "5", "answer_choice_2": "1", "answer_choice_3": "4", "answer_choice_4": "6", "answer_id": 0, "answer": "3", "question_type": [ "Action Recognition", "Object Recognition", "Egocentric & Spatial Perception", "Counting" ], "objects_of_interest": [ "saucepan-eaaa0de116893699", "frying pan-d510045435eca92b", "bowl-ea981a91dfb4db94", "bag of chicken-fe3ba4b0a41f9b19", "chicken-1eb0ba2f3be85466", "pyrex bowl-7fd62b5c0e617c12", "discarded chicken bag-669e77c13d83a0c6", "towel-06d457c3d18b0910", "towel2-fea06dac4ea4a30f", "potato peeler-c319f80b11a0a592", "cup of bread crumbs-21252de25de2d8ad", "cup of bread crumbs cap-5d84b03bd77bef5f", "scale-f09148f04fa5d168", "bowl2-d2721bd426e5c19c", "bowl3-34c64810cbc69dc4", "bag of flour-0890c068caf37f46", "pouch of rice-c95b6d502de90cdc", "bottle of ground chili-57ec31dc93a4c02b", "teaspoon-e94f9e5636637939", "bottle of garlic powder-ce5db1acdb4c63a1", "bottle of ground black pepper-ee2a4084493bb086", "bottle of salt-0434f4abf51cc5a5", "tupperware-f062676dc1eb2055", "bowl4-9d7df688cdef6926", "egg-5deb0b311b0b6c63", "egg2-e293348c62e7f37f", "electric kettle-91e63c2cb8515de3", "pot cover-3fda277021cf3664", "saucepan2-85b54cc17ec1671a", "red saucepan-31701be2c3ca6617", "pot cover2-a532e882a3a733be", "bottle of washing up liquid-747a3f68e8427a9c", "sponge-8090a791330975b2", "spoon-a5bc07287813f070", "potatoes-bf87bed0ccee3f9d", "ginger-68836b8eddf285c9", "bag of ginger-d91eb0cfc44f83bd", "pack of dried hibiscus flowers-dc02bcfa55ddf1de", "saucepan cover-ea642f32c4e6490d", "some dried hibiscus flowers-47389b49cd317eb9", "knife-c88b7b52fc975faf", "chunks of sliced potato-8b06f30ac9601bc9", "potato bits-918cf9c5f49a8dff", "potato-0121203fec36c708", "more potato bits-8beb52e430b0bc78", "more potato bits2-11e03b271b1e0134", "chicken breast-04e0d6d6b2971042", "chicken breast2-3c4cc2e9007379e1", "chicken chunk-fcd07ee885138d63", "chicken strip-661cea196e0d16e3", "chicken strip2-02f73767db97f73b", "chicken strip3-ea82baf3e87b6299", "chicken strip4-7a84a44acc9fac15", "chicken strips5-8b4dde3cb0e45483", "spoon 1-c162f4c69a135827", "plastic chopping board-aff6cf3571b5e746", "potato peels-a129023c2e975289", "slice of potato peel-a7c2d019073e057e", "egg shell-f37d5171d244b593", "pair of tongs-b2cb60919ca4589b", "bottle of oil-429f4383d7550753", "table spoon-1e1514d81bd95cbf", "fork-0ccf59091cbd3221", "fork2-16f46ef8e1f52cc8", "potato masher-8ecaad0bee4a2482", "bag of cheese-05b11d84f980233b", "bowl5-4a909ddf142486d7", "fork3-acee27d6717a322b", "serving spoon-db3fc3c2d7b899f4", "foil-efd9a8b5e899fe1e", "packet-a1bb131de6a58410", "potato chicken ball-a396ef3f39417275", "potato chicken ball2-fd1ff80272e8d015", "potato chicken ball3-36f01a64517663bb", "potato chicken ball4-4ce990da51f12cbf", "potato chicken ball5-b9b830a0af24d7a3", "potato chicken ball6-8b9a9e03e054d476", "potato chicken ball7-ef7ef274207eec42" ], "reasoning": "1. The question has 3 criteria: engaging in a \"dedicated wipe down\" that lasts longer than 3 seconds, wiping the left side of the countertop (relative to the stove), and using a towel (towel-06d457c3d18b0910). I look for actions that meet all 3 of these criteria.\n2. From 16:39-16:42, the user wipes down the right countertop with a towel (towel-06d457c3d18b0910). This action meets 1 criterion for satisfying the requirements of the question.\n3. From 18:26-18:34, the user wipes down the left countertop, but they do so with a sponge (sponge-8090a791330975b2). This action meets 2 criteria.\n4. From 18:35-18:44, the user wipes down the right countertop with a sponge (sponge-8090a791330975b2). This action meets 1 criteria.\n5. From 18:52-18:57, the user wipes down the left countertop with a towel (towel-06d457c3d18b0910). This action meets all 3 criteria, and so I count it.\n6. From 19:13-19:17, the user wipes down the left side of the countertop with a towel (towel-06d457c3d18b0910). This action meets all 3 criteria, so I count this, too.\n7. From 30:17-30:19, the user wipes down the left side of the countertop with a towel (towel-06d457c3d18b0910). Because the time is too short, however, only 2 criteria are met.\n8. From 35:17-35:22, the user wipes down the left side of the countertop with a towel (towel-06d457c3d18b0910). This action also satisfies all 3 criteria, so I count this too.\n9. I count up each action that meets all 3 criteria for satisfying the question and I get 3.\n10. Therefore, 3 is my answer.", "referenced_object_masks": [ "sponge-8090a791330975b2", "towel-06d457c3d18b0910" ] }, { "key": "P09-20240622-154652:00a8804849fa67a0efdb4ccc9a10e59b51f0b681", "video_id": "P09-20240622-154652", "question": "Where is the bag of flour in relation to the eggs?", "answer_choice_0": "Above the eggs", "answer_choice_1": "Under the eggs", "answer_choice_2": "Behind the eggs", "answer_choice_3": "Across from the eggs", "answer_choice_4": "Parallel to the eggs", "answer_id": 2, "answer": "Behind the eggs", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Event Occurence" ], "objects_of_interest": [ "container of breadcrumbs-1ca5d7faea2143ab", "lid of breadcrumb container-7821e3e46af97604", "grey bowl of flour-4f92493f372ef2d9", "bag of flour-f53a942c365f1235", "tupperware of flour-8376069d5622b22d", "spoon-f01ca7db4d1a8d55", "potato chicken ball-3f563c8cfc15dc94", "potato chicken ball 2-68520902beddb4f8", "saucepan-c45048443d03d04b", "tea towel-2ec6f93537e363f0", "potato chicken ball 3-2f525d07cca4e842", "white bowl-8dff37e4645d324d", "sponge-47db642131832813", "chopping board-cd87f1c1a2a14a31", "sponge2-6439f51fef587e5e", "bottle of washing up liquid-1b71ddff683760c4", "potato masher-6873a2cefdbe0d3f", "fork-852d85a45a1117cf", "fork2-31d71a3d5f2178a5", "bowl2-6ca6188ab1f058f0", "bowl3-bd3d80b4e0ba289d", "frying pan-69278a59cbad1aac", "bowl4-e723007dc410a0b9", "tupperware lid-383734a289219590", "smaller saucepan-543453297facc4c3", "spoon2-97a6f8cfd79b3f63", "slotted spoon-a72a37432b66ea10", "bottle of oil-e4e32e3f8a3f56ee", "jars of spice-4d2172268057fb22", "salt-13d614d230b7605f", "plastic basket-607f320e83dbeae4", "spice jar black-d20dbea818c9c292", "spice jar green-aafc73fca4cf16cc", "spice jar orange-bd102a5f4343bb05", "plastic cheese wrapper-625ead5742a0a5e6", "tea towel2-9b372184c5ee0acc", "weighing scales-4a99d80128421547", "tea towel3-b93f08d24b37687d", "bowl5-b80e7f34fa60478e", "box of foil-89a3ed7d96b37421", "box of cling film-ff41883e2f9d3a47", "roll of cling film-db6f84eb0c14c35b", "piece of cling film-de91e8ab1f601461", "potato chicken ball 4-58fd7f35bb585cbc", "potato chicken ball 5-50e897b8807e33cc", "potato chicken ball 6-e7f6af52fc542ffc", "potato chicken ball 7-256e44b7921846c2", "potato chicken ball 8-cc00da1d7aea099c", "potato chicken ball 9-46d2442193a00aa6", "potato chicken ball 10-aa26b99054c556e0", "piece of foil-e8688684e720979e", "wrapped potato chicken balls-b5482033008f84f7", "potato chicken balls-e994509f661fa800" ], "reasoning": "1. I watched the video from the beginning to locate the bag of flour (bag of flour-f53a942c365f1235), which I found at 00:42.\n2. At the 00:42 mark, I watched the user pull out the bag of flour (bag of flour-f53a942c365f1235) from the kitchen cabinet. The bag (bag of flour-f53a942c365f1235) was hiding behind a green container , which is also behind a carton of eggs . \n3. Therefore, I concluded that the bag of flour (bag of flour-f53a942c365f1235) is behind the carton of eggs .", "referenced_object_masks": [ "bag of flour-f53a942c365f1235" ] }, { "key": "P09-20240622-154652:0113a591f51f13afefb2a4c7064194bf5e300913", "video_id": "P09-20240622-154652", "question": "If I were to stand facing my dishwasher, and that direction were due north, in what approximate cardinal direction is my trash can?", "answer_choice_0": "Northwest", "answer_choice_1": "Southwest", "answer_choice_2": "Northeast", "answer_choice_3": "West", "answer_choice_4": "South", "answer_id": 0, "answer": "Northwest", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition" ], "objects_of_interest": [ "container of breadcrumbs-1ca5d7faea2143ab", "lid of breadcrumb container-7821e3e46af97604", "grey bowl of flour-4f92493f372ef2d9", "bag of flour-f53a942c365f1235", "tupperware of flour-8376069d5622b22d", "spoon-f01ca7db4d1a8d55", "potato chicken ball-3f563c8cfc15dc94", "potato chicken ball 2-68520902beddb4f8", "saucepan-c45048443d03d04b", "tea towel-2ec6f93537e363f0", "potato chicken ball 3-2f525d07cca4e842", "white bowl-8dff37e4645d324d", "sponge-47db642131832813", "chopping board-cd87f1c1a2a14a31", "sponge2-6439f51fef587e5e", "bottle of washing up liquid-1b71ddff683760c4", "potato masher-6873a2cefdbe0d3f", "fork-852d85a45a1117cf", "fork2-31d71a3d5f2178a5", "bowl2-6ca6188ab1f058f0", "bowl3-bd3d80b4e0ba289d", "frying pan-69278a59cbad1aac", "bowl4-e723007dc410a0b9", "tupperware lid-383734a289219590", "smaller saucepan-543453297facc4c3", "spoon2-97a6f8cfd79b3f63", "slotted spoon-a72a37432b66ea10", "bottle of oil-e4e32e3f8a3f56ee", "jars of spice-4d2172268057fb22", "salt-13d614d230b7605f", "plastic basket-607f320e83dbeae4", "spice jar black-d20dbea818c9c292", "spice jar green-aafc73fca4cf16cc", "spice jar orange-bd102a5f4343bb05", "plastic cheese wrapper-625ead5742a0a5e6", "tea towel2-9b372184c5ee0acc", "weighing scales-4a99d80128421547", "tea towel3-b93f08d24b37687d", "bowl5-b80e7f34fa60478e", "box of foil-89a3ed7d96b37421", "box of cling film-ff41883e2f9d3a47", "roll of cling film-db6f84eb0c14c35b", "piece of cling film-de91e8ab1f601461", "potato chicken ball 4-58fd7f35bb585cbc", "potato chicken ball 5-50e897b8807e33cc", "potato chicken ball 6-e7f6af52fc542ffc", "potato chicken ball 7-256e44b7921846c2", "potato chicken ball 8-cc00da1d7aea099c", "potato chicken ball 9-46d2442193a00aa6", "potato chicken ball 10-aa26b99054c556e0", "piece of foil-e8688684e720979e", "wrapped potato chicken balls-b5482033008f84f7", "potato chicken balls-e994509f661fa800" ], "reasoning": "1. I watched the video to find the user's position when they were directly facing the dishwasher and saw this position at 05:20, which would be considered north.\n2. I then watched the video to find out the location of the trash can relative to the dishwasher and found the trash can at 09:31, which is slightly up and to the left of the dishwasher .\n3. Therefore, the trash can can be described as being northwest of the dishwasher .", "referenced_object_masks": [] }, { "key": "P09-20240622-154652:2a32b843269d6bab4e46e3ea1ef032bc2f3dc17a", "video_id": "P09-20240622-154652", "question": "If I'm standing in front of the sink, in which direction should I move to exit the kitchen through the sliding doors?", "answer_choice_0": "Turn left 180 degrees, walk forward, then turn left", "answer_choice_1": "Walk straight ahead", "answer_choice_2": "Walk right, then turn left", "answer_choice_3": "Turn 90 degrees left, walk forward, and turn right", "answer_choice_4": "Walk backwards, turn 90 degrees right, and walk left", "answer_id": 3, "answer": "Turn 90 degrees left, walk forward, and turn right", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Situational Awareness" ], "objects_of_interest": [ "container of breadcrumbs-1ca5d7faea2143ab", "lid of breadcrumb container-7821e3e46af97604", "grey bowl of flour-4f92493f372ef2d9", "bag of flour-f53a942c365f1235", "tupperware of flour-8376069d5622b22d", "spoon-f01ca7db4d1a8d55", "potato chicken ball-3f563c8cfc15dc94", "potato chicken ball 2-68520902beddb4f8", "saucepan-c45048443d03d04b", "tea towel-2ec6f93537e363f0", "potato chicken ball 3-2f525d07cca4e842", "white bowl-8dff37e4645d324d", "sponge-47db642131832813", "chopping board-cd87f1c1a2a14a31", "sponge2-6439f51fef587e5e", "bottle of washing up liquid-1b71ddff683760c4", "potato masher-6873a2cefdbe0d3f", "fork-852d85a45a1117cf", "fork2-31d71a3d5f2178a5", "bowl2-6ca6188ab1f058f0", "bowl3-bd3d80b4e0ba289d", "frying pan-69278a59cbad1aac", "bowl4-e723007dc410a0b9", "tupperware lid-383734a289219590", "smaller saucepan-543453297facc4c3", "spoon2-97a6f8cfd79b3f63", "slotted spoon-a72a37432b66ea10", "bottle of oil-e4e32e3f8a3f56ee", "jars of spice-4d2172268057fb22", "salt-13d614d230b7605f", "plastic basket-607f320e83dbeae4", "spice jar black-d20dbea818c9c292", "spice jar green-aafc73fca4cf16cc", "spice jar orange-bd102a5f4343bb05", "plastic cheese wrapper-625ead5742a0a5e6", "tea towel2-9b372184c5ee0acc", "weighing scales-4a99d80128421547", "tea towel3-b93f08d24b37687d", "bowl5-b80e7f34fa60478e", "box of foil-89a3ed7d96b37421", "box of cling film-ff41883e2f9d3a47", "roll of cling film-db6f84eb0c14c35b", "piece of cling film-de91e8ab1f601461", "potato chicken ball 4-58fd7f35bb585cbc", "potato chicken ball 5-50e897b8807e33cc", "potato chicken ball 6-e7f6af52fc542ffc", "potato chicken ball 7-256e44b7921846c2", "potato chicken ball 8-cc00da1d7aea099c", "potato chicken ball 9-46d2442193a00aa6", "potato chicken ball 10-aa26b99054c556e0", "piece of foil-e8688684e720979e", "wrapped potato chicken balls-b5482033008f84f7", "potato chicken balls-e994509f661fa800" ], "reasoning": "1. I watched the video from the beginning to locate the sliding doors in relation to the sink. \n2. At 00:00, the user enters the kitchen. The sliding door is visible from the entrance, since it's straight ahead on the opposite side. The kitchen is to the right, off a hallway that leads from the entrance to the sliding door. \n3. At 02:52 the user stands in front of the sink , with the kitchen cabinet to its right, and the hallway to its left. \n4. Therefore, if the user wants to exit the kitchen through the sliding door , the user needs to turn 90 degrees left, walk forward, then turn right towards the sliding door .", "referenced_object_masks": [] }, { "key": "P09-20240622-154652:3c136bb945764647f4fe9830a1b117c6ecc1768b", "video_id": "P09-20240622-154652", "question": "What is the ratio of potato balls fried to potato balls stored?", "answer_choice_0": "1:9", "answer_choice_1": "4:6", "answer_choice_2": "8:2", "answer_choice_3": "2:8", "answer_choice_4": "3:7", "answer_id": 3, "answer": "2:8", "question_type": [ "Object Recognition", "Action Recognition", "Counting", "Event Occurence", "Numerical Reasoning" ], "objects_of_interest": [ "container of breadcrumbs-1ca5d7faea2143ab", "lid of breadcrumb container-7821e3e46af97604", "grey bowl of flour-4f92493f372ef2d9", "bag of flour-f53a942c365f1235", "tupperware of flour-8376069d5622b22d", "spoon-f01ca7db4d1a8d55", "potato chicken ball-3f563c8cfc15dc94", "potato chicken ball 2-68520902beddb4f8", "saucepan-c45048443d03d04b", "tea towel-2ec6f93537e363f0", "potato chicken ball 3-2f525d07cca4e842", "white bowl-8dff37e4645d324d", "sponge-47db642131832813", "chopping board-cd87f1c1a2a14a31", "sponge2-6439f51fef587e5e", "bottle of washing up liquid-1b71ddff683760c4", "potato masher-6873a2cefdbe0d3f", "fork-852d85a45a1117cf", "fork2-31d71a3d5f2178a5", "bowl2-6ca6188ab1f058f0", "bowl3-bd3d80b4e0ba289d", "frying pan-69278a59cbad1aac", "bowl4-e723007dc410a0b9", "tupperware lid-383734a289219590", "smaller saucepan-543453297facc4c3", "spoon2-97a6f8cfd79b3f63", "slotted spoon-a72a37432b66ea10", "bottle of oil-e4e32e3f8a3f56ee", "jars of spice-4d2172268057fb22", "salt-13d614d230b7605f", "plastic basket-607f320e83dbeae4", "spice jar black-d20dbea818c9c292", "spice jar green-aafc73fca4cf16cc", "spice jar orange-bd102a5f4343bb05", "plastic cheese wrapper-625ead5742a0a5e6", "tea towel2-9b372184c5ee0acc", "weighing scales-4a99d80128421547", "tea towel3-b93f08d24b37687d", "bowl5-b80e7f34fa60478e", "box of foil-89a3ed7d96b37421", "box of cling film-ff41883e2f9d3a47", "roll of cling film-db6f84eb0c14c35b", "piece of cling film-de91e8ab1f601461", "potato chicken ball 4-58fd7f35bb585cbc", "potato chicken ball 5-50e897b8807e33cc", "potato chicken ball 6-e7f6af52fc542ffc", "potato chicken ball 7-256e44b7921846c2", "potato chicken ball 8-cc00da1d7aea099c", "potato chicken ball 9-46d2442193a00aa6", "potato chicken ball 10-aa26b99054c556e0", "piece of foil-e8688684e720979e", "wrapped potato chicken balls-b5482033008f84f7", "potato chicken balls-e994509f661fa800" ], "reasoning": "1. I watched the video from the beginning to locate the moment the user starts to cook the 10 potato balls (potato chicken ball-3f563c8cfc15dc94, potato chicken ball 2-68520902beddb4f8, potato chicken ball 3-2f525d07cca4e842, potato chicken ball 4-58fd7f35bb585cbc, potato chicken ball 5-50e897b8807e33cc, potato chicken ball 6-e7f6af52fc542ffc, potato chicken ball 7-256e44b7921846c2, potato chicken ball 8-cc00da1d7aea099c, potato chicken ball 9-46d2442193a00aa6, potato chicken ball 10-aa26b99054c556e0) \nthey prepared, which happens at 10:44, where the user puts the first potato ball (potato chicken ball-3f563c8cfc15dc94) in the pan (smaller saucepan-543453297facc4c3) to fry. \n2. While the first potato ball (potato chicken ball-3f563c8cfc15dc94) was frying, the user pulled a piece of plastic wrap (piece of cling film-de91e8ab1f601461) at 11:50 to wrap 3 potato balls (potato chicken ball 4-58fd7f35bb585cbc, potato chicken ball 5-50e897b8807e33cc, potato chicken ball 6-e7f6af52fc542ffc) from 12:16 to 12:42. \n3. At 13:53, the user cuts another piece of plastic wrap (piece of cling film-de91e8ab1f601461) to wrap 4 more potato balls (potato chicken ball 7-256e44b7921846c2, potato chicken ball 8-cc00da1d7aea099c, potato chicken ball 9-46d2442193a00aa6, potato chicken ball 10-aa26b99054c556e0).\n4. At 14:08, the user removes the first potato ball (potato chicken ball-3f563c8cfc15dc94) from the hot oil (smaller saucepan-543453297facc4c3) and fries the second potato ball (potato chicken ball 2-68520902beddb4f8) at 14:18. \n5. At 15:19, the user wraps the last uncooked potato ball (potato chicken ball 10-aa26b99054c556e0) in plastic wrap (piece of cling film-de91e8ab1f601461).\n6. In conclusion, the user cooked two potato balls (potato chicken balls-e994509f661fa800) and stored 8. Therefore, the ratio is 2:8.", "referenced_object_masks": [ "potato chicken ball-3f563c8cfc15dc94", "potato chicken ball 2-68520902beddb4f8", "potato chicken ball 3-2f525d07cca4e842", "potato chicken ball 4-58fd7f35bb585cbc", "potato chicken ball 5-50e897b8807e33cc", "potato chicken ball 6-e7f6af52fc542ffc", "potato chicken ball 7-256e44b7921846c2", "potato chicken ball 8-cc00da1d7aea099c", "potato chicken ball 9-46d2442193a00aa6", "potato chicken ball 10-aa26b99054c556e0", "potato chicken balls-e994509f661fa800", "smaller saucepan-543453297facc4c3", "piece of cling film-de91e8ab1f601461" ] }, { "key": "P09-20240622-154652:59740a3ee68305da06c5df69ab6af1711c5af628", "video_id": "P09-20240622-154652", "question": "When I am standing at the fridge, what appliance is directly behind me using clock position?", "answer_choice_0": "The stove", "answer_choice_1": "The dishwasher", "answer_choice_2": "The coffee machine", "answer_choice_3": "The toaster", "answer_choice_4": "The microwave", "answer_id": 1, "answer": "The dishwasher", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition" ], "objects_of_interest": [ "container of breadcrumbs-1ca5d7faea2143ab", "lid of breadcrumb container-7821e3e46af97604", "grey bowl of flour-4f92493f372ef2d9", "bag of flour-f53a942c365f1235", "tupperware of flour-8376069d5622b22d", "spoon-f01ca7db4d1a8d55", "potato chicken ball-3f563c8cfc15dc94", "potato chicken ball 2-68520902beddb4f8", "saucepan-c45048443d03d04b", "tea towel-2ec6f93537e363f0", "potato chicken ball 3-2f525d07cca4e842", "white bowl-8dff37e4645d324d", "sponge-47db642131832813", "chopping board-cd87f1c1a2a14a31", "sponge2-6439f51fef587e5e", "bottle of washing up liquid-1b71ddff683760c4", "potato masher-6873a2cefdbe0d3f", "fork-852d85a45a1117cf", "fork2-31d71a3d5f2178a5", "bowl2-6ca6188ab1f058f0", "bowl3-bd3d80b4e0ba289d", "frying pan-69278a59cbad1aac", "bowl4-e723007dc410a0b9", "tupperware lid-383734a289219590", "smaller saucepan-543453297facc4c3", "spoon2-97a6f8cfd79b3f63", "slotted spoon-a72a37432b66ea10", "bottle of oil-e4e32e3f8a3f56ee", "jars of spice-4d2172268057fb22", "salt-13d614d230b7605f", "plastic basket-607f320e83dbeae4", "spice jar black-d20dbea818c9c292", "spice jar green-aafc73fca4cf16cc", "spice jar orange-bd102a5f4343bb05", "plastic cheese wrapper-625ead5742a0a5e6", "tea towel2-9b372184c5ee0acc", "weighing scales-4a99d80128421547", "tea towel3-b93f08d24b37687d", "bowl5-b80e7f34fa60478e", "box of foil-89a3ed7d96b37421", "box of cling film-ff41883e2f9d3a47", "roll of cling film-db6f84eb0c14c35b", "piece of cling film-de91e8ab1f601461", "potato chicken ball 4-58fd7f35bb585cbc", "potato chicken ball 5-50e897b8807e33cc", "potato chicken ball 6-e7f6af52fc542ffc", "potato chicken ball 7-256e44b7921846c2", "potato chicken ball 8-cc00da1d7aea099c", "potato chicken ball 9-46d2442193a00aa6", "potato chicken ball 10-aa26b99054c556e0", "piece of foil-e8688684e720979e", "wrapped potato chicken balls-b5482033008f84f7", "potato chicken balls-e994509f661fa800" ], "reasoning": "1. I watched the video to find the user's position when they were directly facing the fridge and saw this position at 17:31.\n2. I then watched the video to observe what is directly behind the user when they are facing the fridge and saw the dishwasher at the beginning of the video at 00:00.\n3. The dishwasher appears to be parallel to the fridge , so this is what would be directly behind the user when they are standing at the fridge .", "referenced_object_masks": [] }, { "key": "P09-20240622-154652:65218498d7ef58ebd206b621254dfd44c8169b44", "video_id": "P09-20240622-154652", "question": "Not counting the sponge, how many total items did I wash in the sink?", "answer_choice_0": "11 dishes", "answer_choice_1": "13 dishes", "answer_choice_2": "17 dishes", "answer_choice_3": "16 dishes", "answer_choice_4": "14 dishes", "answer_id": 4, "answer": "14 dishes", "question_type": [ "Counting", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "container of breadcrumbs-1ca5d7faea2143ab", "lid of breadcrumb container-7821e3e46af97604", "grey bowl of flour-4f92493f372ef2d9", "bag of flour-f53a942c365f1235", "tupperware of flour-8376069d5622b22d", "spoon-f01ca7db4d1a8d55", "potato chicken ball-3f563c8cfc15dc94", "potato chicken ball 2-68520902beddb4f8", "saucepan-c45048443d03d04b", "tea towel-2ec6f93537e363f0", "potato chicken ball 3-2f525d07cca4e842", "white bowl-8dff37e4645d324d", "sponge-47db642131832813", "chopping board-cd87f1c1a2a14a31", "sponge2-6439f51fef587e5e", "bottle of washing up liquid-1b71ddff683760c4", "potato masher-6873a2cefdbe0d3f", "fork-852d85a45a1117cf", "fork2-31d71a3d5f2178a5", "bowl2-6ca6188ab1f058f0", "bowl3-bd3d80b4e0ba289d", "frying pan-69278a59cbad1aac", "bowl4-e723007dc410a0b9", "tupperware lid-383734a289219590", "smaller saucepan-543453297facc4c3", "spoon2-97a6f8cfd79b3f63", "slotted spoon-a72a37432b66ea10", "bottle of oil-e4e32e3f8a3f56ee", "jars of spice-4d2172268057fb22", "salt-13d614d230b7605f", "plastic basket-607f320e83dbeae4", "spice jar black-d20dbea818c9c292", "spice jar green-aafc73fca4cf16cc", "spice jar orange-bd102a5f4343bb05", "plastic cheese wrapper-625ead5742a0a5e6", "tea towel2-9b372184c5ee0acc", "weighing scales-4a99d80128421547", "tea towel3-b93f08d24b37687d", "bowl5-b80e7f34fa60478e", "box of foil-89a3ed7d96b37421", "box of cling film-ff41883e2f9d3a47", "roll of cling film-db6f84eb0c14c35b", "piece of cling film-de91e8ab1f601461", "potato chicken ball 4-58fd7f35bb585cbc", "potato chicken ball 5-50e897b8807e33cc", "potato chicken ball 6-e7f6af52fc542ffc", "potato chicken ball 7-256e44b7921846c2", "potato chicken ball 8-cc00da1d7aea099c", "potato chicken ball 9-46d2442193a00aa6", "potato chicken ball 10-aa26b99054c556e0", "piece of foil-e8688684e720979e", "wrapped potato chicken balls-b5482033008f84f7", "potato chicken balls-e994509f661fa800" ], "reasoning": "1. I watched the video to find when the user started washing items and saw this happen at 04:47 when the user washed a large spoon (spoon-f01ca7db4d1a8d55) using water and a sponge (sponge-47db642131832813). For the final total, I did not count the sponge (sponge-47db642131832813).\n2. I then observed the user washing a potato masher (potato masher-6873a2cefdbe0d3f) at 04:54.\n3. I then observed the user washing a fork (fork-852d85a45a1117cf) at 04:58.\n4. I then observed the user washing another fork (fork2-31d71a3d5f2178a5) at 05:02.\n5. I then observed the user washing a small spoon at 05:05.\n6. I then observed the user washing a bowl (bowl2-6ca6188ab1f058f0) at 05:26.\n7. I then observed the user washing another bowl (bowl3-bd3d80b4e0ba289d) at 05:35.\n8. I then observed the user washing another bowl (bowl4-e723007dc410a0b9) at 05:59.\n9. I then observed the user washing another bowl (white bowl-8dff37e4645d324d) at 06:09.\n10. I then observed the user washing a tupperware lid (tupperware lid-383734a289219590) at 06:27.\n11. I then observed the user washing a chopping board (chopping board-cd87f1c1a2a14a31) at 06:39.\n12. I then observed the user washing a small saucepan (smaller saucepan-543453297facc4c3) at 06:58.\n13. I then observed the user washing a larger saucepan (saucepan-c45048443d03d04b) at 07:09.\n14. I then observed the user washing a spoon (spoon2-97a6f8cfd79b3f63) at 07:39.\n15. Altogether, the user washed 14 items, not counting the sponge (sponge-47db642131832813).", "referenced_object_masks": [ "sponge-47db642131832813", "white bowl-8dff37e4645d324d", "saucepan-c45048443d03d04b", "bowl2-6ca6188ab1f058f0", "chopping board-cd87f1c1a2a14a31", "spoon-f01ca7db4d1a8d55", "fork2-31d71a3d5f2178a5", "tupperware lid-383734a289219590", "potato masher-6873a2cefdbe0d3f", "smaller saucepan-543453297facc4c3", "fork-852d85a45a1117cf", "spoon2-97a6f8cfd79b3f63", "bowl3-bd3d80b4e0ba289d", "bowl4-e723007dc410a0b9" ] }, { "key": "P09-20240622-154652:c29ccedc10e490d495e01130cc95680ad731fb6d", "video_id": "P09-20240622-154652", "question": "How many bowls did I use for culinary purposes and what answer would I get if I multiplied this number by the number of potato chicken balls I was making?", "answer_choice_0": "30", "answer_choice_1": "90", "answer_choice_2": "40", "answer_choice_3": "100", "answer_choice_4": "70", "answer_id": 2, "answer": "40", "question_type": [ "Counting", "Object Recognition", "Numerical Reasoning" ], "objects_of_interest": [ "container of breadcrumbs-1ca5d7faea2143ab", "lid of breadcrumb container-7821e3e46af97604", "grey bowl of flour-4f92493f372ef2d9", "bag of flour-f53a942c365f1235", "tupperware of flour-8376069d5622b22d", "spoon-f01ca7db4d1a8d55", "potato chicken ball-3f563c8cfc15dc94", "potato chicken ball 2-68520902beddb4f8", "saucepan-c45048443d03d04b", "tea towel-2ec6f93537e363f0", "potato chicken ball 3-2f525d07cca4e842", "white bowl-8dff37e4645d324d", "sponge-47db642131832813", "chopping board-cd87f1c1a2a14a31", "sponge2-6439f51fef587e5e", "bottle of washing up liquid-1b71ddff683760c4", "potato masher-6873a2cefdbe0d3f", "fork-852d85a45a1117cf", "fork2-31d71a3d5f2178a5", "bowl2-6ca6188ab1f058f0", "bowl3-bd3d80b4e0ba289d", "frying pan-69278a59cbad1aac", "bowl4-e723007dc410a0b9", "tupperware lid-383734a289219590", "smaller saucepan-543453297facc4c3", "spoon2-97a6f8cfd79b3f63", "slotted spoon-a72a37432b66ea10", "bottle of oil-e4e32e3f8a3f56ee", "jars of spice-4d2172268057fb22", "salt-13d614d230b7605f", "plastic basket-607f320e83dbeae4", "spice jar black-d20dbea818c9c292", "spice jar green-aafc73fca4cf16cc", "spice jar orange-bd102a5f4343bb05", "plastic cheese wrapper-625ead5742a0a5e6", "tea towel2-9b372184c5ee0acc", "weighing scales-4a99d80128421547", "tea towel3-b93f08d24b37687d", "bowl5-b80e7f34fa60478e", "box of foil-89a3ed7d96b37421", "box of cling film-ff41883e2f9d3a47", "roll of cling film-db6f84eb0c14c35b", "piece of cling film-de91e8ab1f601461", "potato chicken ball 4-58fd7f35bb585cbc", "potato chicken ball 5-50e897b8807e33cc", "potato chicken ball 6-e7f6af52fc542ffc", "potato chicken ball 7-256e44b7921846c2", "potato chicken ball 8-cc00da1d7aea099c", "potato chicken ball 9-46d2442193a00aa6", "potato chicken ball 10-aa26b99054c556e0", "piece of foil-e8688684e720979e", "wrapped potato chicken balls-b5482033008f84f7", "potato chicken balls-e994509f661fa800" ], "reasoning": "1. I watched the video to observe how many bowls were used for \"culinary purposes\" and saw 3 bowls (grey bowl of flour-4f92493f372ef2d9, white bowl-8dff37e4645d324d) with various ingredients in them at 00:29.\n2. I then watched the video to see if any more bowls were used and saw the user take out another bowl (bowl5-b80e7f34fa60478e) at 09:54 to put the finished potato chicken balls in starting at 14:07.\n3. Therefore, the amount of bowls the user used for culinary purposes is 4.\n4. I then watched the video to see how many potato chicken balls the user was making and saw the full amount of 10 at 10:41.\n5. 4x10=40", "referenced_object_masks": [ "bowl5-b80e7f34fa60478e", "grey bowl of flour-4f92493f372ef2d9", "white bowl-8dff37e4645d324d" ] }, { "key": "P09-20240622-154652:c707c4fa681fb1b4e498e1aefd138b8af92b8732", "video_id": "P09-20240622-154652", "question": "What is the product of the number of potato balls I fried and the dishes I washed and placed on the drying rack from 04:45 to 07:29?", "answer_choice_0": "24", "answer_choice_1": "16", "answer_choice_2": "12", "answer_choice_3": "17", "answer_choice_4": "32", "answer_id": 0, "answer": "24", "question_type": [ "Object Recognition", "Action Recognition", "Counting", "Numerical Reasoning" ], "objects_of_interest": [ "container of breadcrumbs-1ca5d7faea2143ab", "lid of breadcrumb container-7821e3e46af97604", "grey bowl of flour-4f92493f372ef2d9", "bag of flour-f53a942c365f1235", "tupperware of flour-8376069d5622b22d", "spoon-f01ca7db4d1a8d55", "potato chicken ball-3f563c8cfc15dc94", "potato chicken ball 2-68520902beddb4f8", "saucepan-c45048443d03d04b", "tea towel-2ec6f93537e363f0", "potato chicken ball 3-2f525d07cca4e842", "white bowl-8dff37e4645d324d", "sponge-47db642131832813", "chopping board-cd87f1c1a2a14a31", "sponge2-6439f51fef587e5e", "bottle of washing up liquid-1b71ddff683760c4", "potato masher-6873a2cefdbe0d3f", "fork-852d85a45a1117cf", "fork2-31d71a3d5f2178a5", "bowl2-6ca6188ab1f058f0", "bowl3-bd3d80b4e0ba289d", "frying pan-69278a59cbad1aac", "bowl4-e723007dc410a0b9", "tupperware lid-383734a289219590", "smaller saucepan-543453297facc4c3", "spoon2-97a6f8cfd79b3f63", "slotted spoon-a72a37432b66ea10", "bottle of oil-e4e32e3f8a3f56ee", "jars of spice-4d2172268057fb22", "salt-13d614d230b7605f", "plastic basket-607f320e83dbeae4", "spice jar black-d20dbea818c9c292", "spice jar green-aafc73fca4cf16cc", "spice jar orange-bd102a5f4343bb05", "plastic cheese wrapper-625ead5742a0a5e6", "tea towel2-9b372184c5ee0acc", "weighing scales-4a99d80128421547", "tea towel3-b93f08d24b37687d", "bowl5-b80e7f34fa60478e", "box of foil-89a3ed7d96b37421", "box of cling film-ff41883e2f9d3a47", "roll of cling film-db6f84eb0c14c35b", "piece of cling film-de91e8ab1f601461", "potato chicken ball 4-58fd7f35bb585cbc", "potato chicken ball 5-50e897b8807e33cc", "potato chicken ball 6-e7f6af52fc542ffc", "potato chicken ball 7-256e44b7921846c2", "potato chicken ball 8-cc00da1d7aea099c", "potato chicken ball 9-46d2442193a00aa6", "potato chicken ball 10-aa26b99054c556e0", "piece of foil-e8688684e720979e", "wrapped potato chicken balls-b5482033008f84f7", "potato chicken balls-e994509f661fa800" ], "reasoning": "1. I first located the 04:45 to 07:29 timestamp to calculate how many dishes were washed and put in the drying rack in this segment. I counted 13 dishes washed in total, with the small saucepan (smaller saucepan-543453297facc4c3) as the only item placed outside of the drying rack . The dishes were washed as follows: a serving spoon (spoon-f01ca7db4d1a8d55) at 04:46, a potato masher (potato masher-6873a2cefdbe0d3f) at 04:54, a fork (fork-852d85a45a1117cf) at 04:58, a second fork (fork2-31d71a3d5f2178a5) at 05:02, a spoon at 05:04, a white bowl (bowl2-6ca6188ab1f058f0) at 05:26, (bowl2-6ca6188ab1f058f0) at 05:26, a teal bowl (bowl3-bd3d80b4e0ba289d) at 05:34, a clear bowl (bowl4-e723007dc410a0b9) at 05:59, a striped bowl (white bowl-8dff37e4645d324d) at 06:09, a tupperware lid (tupperware lid-383734a289219590) at 06:26, a cutting board (chopping board-cd87f1c1a2a14a31) at 06:39, a small saucepan (smaller saucepan-543453297facc4c3) at 06:58, and a red saucepan (saucepan-c45048443d03d04b) at 07:09. \n2. I looked for the amount of potato balls (potato chicken balls-e994509f661fa800) the user fried in the sauce pan (smaller saucepan-543453297facc4c3). I found two instances, which happened at 10:44, where the user puts the first potato ball (potato chicken ball-3f563c8cfc15dc94) in the pan (smaller saucepan-543453297facc4c3) to fry, and a second one (potato chicken ball 2-68520902beddb4f8) at 14:18.\n3. In conclusion, the product of the 12 dishes washed and put on the drying rack, and the 2 potato balls (potato chicken ball-3f563c8cfc15dc94) fried equals 24.", "referenced_object_masks": [ "saucepan-c45048443d03d04b", "white bowl-8dff37e4645d324d", "bowl2-6ca6188ab1f058f0", "chopping board-cd87f1c1a2a14a31", "spoon-f01ca7db4d1a8d55", "potato chicken balls-e994509f661fa800", "fork2-31d71a3d5f2178a5", "tupperware lid-383734a289219590", "potato chicken ball 2-68520902beddb4f8", "smaller saucepan-543453297facc4c3", "potato masher-6873a2cefdbe0d3f", "potato chicken ball-3f563c8cfc15dc94", "fork-852d85a45a1117cf", "bowl3-bd3d80b4e0ba289d", "bowl4-e723007dc410a0b9" ] }, { "key": "P09-20240622-154652:e55bc7ace7f8546871abae76c1c1cb58a3ae70c4", "video_id": "P09-20240622-154652", "question": "When I finish washing the dishes, approximately how many degrees clockwise do I turn to face the potato chicken balls?", "answer_choice_0": "125 degrees clockwise", "answer_choice_1": "50 degrees clockwise", "answer_choice_2": "90 degrees clockwise", "answer_choice_3": "180 degrees clockwise", "answer_choice_4": "25 degrees clockwise", "answer_id": 2, "answer": "90 degrees clockwise", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition" ], "objects_of_interest": [ "container of breadcrumbs-1ca5d7faea2143ab", "lid of breadcrumb container-7821e3e46af97604", "grey bowl of flour-4f92493f372ef2d9", "bag of flour-f53a942c365f1235", "tupperware of flour-8376069d5622b22d", "spoon-f01ca7db4d1a8d55", "potato chicken ball-3f563c8cfc15dc94", "potato chicken ball 2-68520902beddb4f8", "saucepan-c45048443d03d04b", "tea towel-2ec6f93537e363f0", "potato chicken ball 3-2f525d07cca4e842", "white bowl-8dff37e4645d324d", "sponge-47db642131832813", "chopping board-cd87f1c1a2a14a31", "sponge2-6439f51fef587e5e", "bottle of washing up liquid-1b71ddff683760c4", "potato masher-6873a2cefdbe0d3f", "fork-852d85a45a1117cf", "fork2-31d71a3d5f2178a5", "bowl2-6ca6188ab1f058f0", "bowl3-bd3d80b4e0ba289d", "frying pan-69278a59cbad1aac", "bowl4-e723007dc410a0b9", "tupperware lid-383734a289219590", "smaller saucepan-543453297facc4c3", "spoon2-97a6f8cfd79b3f63", "slotted spoon-a72a37432b66ea10", "bottle of oil-e4e32e3f8a3f56ee", "jars of spice-4d2172268057fb22", "salt-13d614d230b7605f", "plastic basket-607f320e83dbeae4", "spice jar black-d20dbea818c9c292", "spice jar green-aafc73fca4cf16cc", "spice jar orange-bd102a5f4343bb05", "plastic cheese wrapper-625ead5742a0a5e6", "tea towel2-9b372184c5ee0acc", "weighing scales-4a99d80128421547", "tea towel3-b93f08d24b37687d", "bowl5-b80e7f34fa60478e", "box of foil-89a3ed7d96b37421", "box of cling film-ff41883e2f9d3a47", "roll of cling film-db6f84eb0c14c35b", "piece of cling film-de91e8ab1f601461", "potato chicken ball 4-58fd7f35bb585cbc", "potato chicken ball 5-50e897b8807e33cc", "potato chicken ball 6-e7f6af52fc542ffc", "potato chicken ball 7-256e44b7921846c2", "potato chicken ball 8-cc00da1d7aea099c", "potato chicken ball 9-46d2442193a00aa6", "potato chicken ball 10-aa26b99054c556e0", "piece of foil-e8688684e720979e", "wrapped potato chicken balls-b5482033008f84f7", "potato chicken balls-e994509f661fa800" ], "reasoning": "1. I watched the video to find the user's position when they were directly facing the kitchen sink and saw this position at 05:20.\n2. I then watched the video to see the positioning of the user when they turn after they finish washing the dishes and saw this position at 07:56, where it appears they turned all the way to the right once to face the potato chicken balls .\n3. Therefore, the user turned 90 degrees clockwise when they finished washing the dishes.", "referenced_object_masks": [] }, { "key": "P09-20240622-154652:e853cee75bda67eb577ab6b51572281a6b055f9b", "video_id": "P09-20240622-154652", "question": "What is the 6th thing I do with my right hand after pulling the can of bread crumbs from the cabinet?", "answer_choice_0": "Remove the breadcrumps lid from the scale", "answer_choice_1": "Grab the container to remove the lid", "answer_choice_2": "Grab the breadcrumps container from the cabinet", "answer_choice_3": "Turn the scale on", "answer_choice_4": "Remove the lid from the breadcrumps container", "answer_id": 0, "answer": "Remove the breadcrumps lid from the scale", "question_type": [ "Object Recognition", "Action Recognition", "Counting", "Event Occurence" ], "objects_of_interest": [ "container of breadcrumbs-1ca5d7faea2143ab", "lid of breadcrumb container-7821e3e46af97604", "grey bowl of flour-4f92493f372ef2d9", "bag of flour-f53a942c365f1235", "tupperware of flour-8376069d5622b22d", "spoon-f01ca7db4d1a8d55", "potato chicken ball-3f563c8cfc15dc94", "potato chicken ball 2-68520902beddb4f8", "saucepan-c45048443d03d04b", "tea towel-2ec6f93537e363f0", "potato chicken ball 3-2f525d07cca4e842", "white bowl-8dff37e4645d324d", "sponge-47db642131832813", "chopping board-cd87f1c1a2a14a31", "sponge2-6439f51fef587e5e", "bottle of washing up liquid-1b71ddff683760c4", "potato masher-6873a2cefdbe0d3f", "fork-852d85a45a1117cf", "fork2-31d71a3d5f2178a5", "bowl2-6ca6188ab1f058f0", "bowl3-bd3d80b4e0ba289d", "frying pan-69278a59cbad1aac", "bowl4-e723007dc410a0b9", "tupperware lid-383734a289219590", "smaller saucepan-543453297facc4c3", "spoon2-97a6f8cfd79b3f63", "slotted spoon-a72a37432b66ea10", "bottle of oil-e4e32e3f8a3f56ee", "jars of spice-4d2172268057fb22", "salt-13d614d230b7605f", "plastic basket-607f320e83dbeae4", "spice jar black-d20dbea818c9c292", "spice jar green-aafc73fca4cf16cc", "spice jar orange-bd102a5f4343bb05", "plastic cheese wrapper-625ead5742a0a5e6", "tea towel2-9b372184c5ee0acc", "weighing scales-4a99d80128421547", "tea towel3-b93f08d24b37687d", "bowl5-b80e7f34fa60478e", "box of foil-89a3ed7d96b37421", "box of cling film-ff41883e2f9d3a47", "roll of cling film-db6f84eb0c14c35b", "piece of cling film-de91e8ab1f601461", "potato chicken ball 4-58fd7f35bb585cbc", "potato chicken ball 5-50e897b8807e33cc", "potato chicken ball 6-e7f6af52fc542ffc", "potato chicken ball 7-256e44b7921846c2", "potato chicken ball 8-cc00da1d7aea099c", "potato chicken ball 9-46d2442193a00aa6", "potato chicken ball 10-aa26b99054c556e0", "piece of foil-e8688684e720979e", "wrapped potato chicken balls-b5482033008f84f7", "potato chicken balls-e994509f661fa800" ], "reasoning": "1. I located the moment the user grabs the can of breadcrumbs (container of breadcrumbs-1ca5d7faea2143ab) from the cabinet at 00:07. \n2. From that moment on, I counted the number of actions the user performed with the right hand until I reached the 5th action. \n3. The first action the user performs with the right hand after grabbing the breadcrumps (container of breadcrumbs-1ca5d7faea2143ab) happens at 00:08, when the user, still holding on to the breadcrumps container (container of breadcrumbs-1ca5d7faea2143ab) turns on the digital food scale (weighing scales-4a99d80128421547) with their right index finger. \n4. The second action happens at 00:09, when the user releases the breadcrumps container (container of breadcrumbs-1ca5d7faea2143ab) to press the scale (weighing scales-4a99d80128421547) buttons again. \n5. The third action happens at 00:11, when the user grabs the breadcrumps container (container of breadcrumbs-1ca5d7faea2143ab) again with the right hand to open the lid (lid of breadcrumb container-7821e3e46af97604) with the left hand. \n6. The fourth action happens at 00:13, when the user releases the breadcrumps container (container of breadcrumbs-1ca5d7faea2143ab) again to press the scale (weighing scales-4a99d80128421547) buttons one more time.\n7. The fifth action happens at 00:15, when the user grabs the breadcrumps container (container of breadcrumbs-1ca5d7faea2143ab) once again. \n8. Finally, the sixth action happens at 00:16, when the user removes the breadcrumps container lid (lid of breadcrumb container-7821e3e46af97604) from the scale (weighing scales-4a99d80128421547) and places it on the counter. Therefore, the correct answer is: removes the breadcrumps lid (lid of breadcrumb container-7821e3e46af97604) from the scale (weighing scales-4a99d80128421547) .", "referenced_object_masks": [ "lid of breadcrumb container-7821e3e46af97604", "container of breadcrumbs-1ca5d7faea2143ab", "weighing scales-4a99d80128421547" ] }, { "key": "P09-20240622-162302:1db8eeccfaef72e7c634ee4bde4828a711359433", "video_id": "P09-20240622-162302", "question": "If I take the number of dirty items I clean in the sink after flipping my egg and multiply it by the number of times I take sugar off the sugar jar lid with a spoon, then subtract the desired weight of the sugar and sugar jar lid, what number do I get?", "answer_choice_0": "24", "answer_choice_1": "8", "answer_choice_2": "12", "answer_choice_3": "3", "answer_choice_4": "17", "answer_id": 2, "answer": "12", "question_type": [ "Action Recognition", "Counting", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "saucepan-909e681eb4fdd646", "kitchen rag-299becd1209b2b21", "bowl-7227396ef916d3b0", "lid of saucepan-f527258b20dd020d", "new serving spoon-9a33cf0f22051860", "sponge-f601fc1be05ad5cb", "washing up liquid-0d20e7d7ec9bbd50", "slotted spoon-904ae21c08131332", "glass pyrex bowl-de35551755b3e88e", "plastic bottle-a874fbc3589d207e", "jar of sugar-a20fdf44d55edf06", "measuring scale-9cca96148a5b25c5", "electric kettle-85ddc3132e12319c", "hand towel-ee49c41803e77403", "bag of sugar-df89eea1ea74ef17", "lid of sugar jar-d70d69ccfebdaed1", "bowl2-f04f64f4bf0548dc", "tablespoon-2242a9fd1edd6210", "plastic bag-9bef70736f343cee", "beaker-8dc2db9e886bf2d1", "slotted spoon2-a3ce16843bdeea34", "saucepan2-20ff412c86929a78", "pan support3-1413dfccf7fc08af", "pan support-14aa42ce74c03542", "second pan support-fd984f90b9e94d90", "bowl3-8419653396076e38", "food bin-69fb56fbcded9428", "hand towel rack-ccae7b6f11bab0df", "rag2-f82a2a190f8ef4c7", "pan supports-dec1eacef86a4661", "saucepan3-788be97009d408cb", "frying pan-6a96247d28c538eb", "cooking oil bottle-94e14a7bfff57e11", "fork-7f404485fbc02abf", "bowl4-4d02638eba204b05", "glass jar1-576e45fd52e46563", "glass jar2-c4121a19994dd451", "jars caps-f7c2b2a7c948673d", "frying pan2-afba7b615bae2db0", "bowl5-3e1bb55a1455b933", "bowl6-a88e8e2e4f011cf0", "bowl7-4f37992a9767350b", "new spoon-8f1c86d775666a13", "teaspoon-a7b18c3f978549f8", "bowl8-350bb67effaed309", "rag3-f2d434cb72261999", "cup of yogurt-539327b73dcd975c" ], "reasoning": "1. I watched the whole video and saw the user flip their egg at 12:37. \n2. At 12:52, the user starts washing dirty, slotted spoons . The user washes the first spoon (slotted spoon-904ae21c08131332) at 12:51, the second slotted spoon (slotted spoon2-a3ce16843bdeea34) at 12:57, and the third slotted spoon at 13:02. The user does not clean any other dirty items, making the total number of dirty items cleaned 3. \n3. At 04:12, the user starts taking off small amounts of sugar off the sugar jar cap (lid of sugar jar-d70d69ccfebdaed1). I saw them do this a total of 9 times at 04:12-04:15, 04:17-04:20, 04:22-04:26, 04:31-04:32, 04:37-04:39, 04:41, 04:43-04:44, 04:48-04:50, and 04:53-04:55.\n4. Then, I saw at 04:59 that the desired weight of the sugar jar cap (lid of sugar jar-d70d69ccfebdaed1) and the sugar is 15 g since this is the final weight.\n5. Finally, I took the number of dirty items cleaned in the sink which was 3, multiplied it by 9, the number of times the user takes sugar off the sugar jar cap (lid of sugar jar-d70d69ccfebdaed1), and then subtracted the desired weight of the sugar and sugar jar cap (lid of sugar jar-d70d69ccfebdaed1), which was 15: 3*9-15= 12.", "referenced_object_masks": [ "lid of sugar jar-d70d69ccfebdaed1", "slotted spoon2-a3ce16843bdeea34", "slotted spoon-904ae21c08131332" ] }, { "key": "P09-20240622-162302:4ed5dcdb9f430aa9aa9732a84af06917cb47b3ea", "video_id": "P09-20240622-162302", "question": "How many times does the igniter on the stove go off in total?", "answer_choice_0": "35 times", "answer_choice_1": "15 times", "answer_choice_2": "24 times", "answer_choice_3": "29 times", "answer_choice_4": "10 times", "answer_id": 2, "answer": "24 times", "question_type": [ "Counting", "Object Recognition", "Audio Reasoning" ], "objects_of_interest": [ "saucepan-909e681eb4fdd646", "kitchen rag-299becd1209b2b21", "bowl-7227396ef916d3b0", "lid of saucepan-f527258b20dd020d", "new serving spoon-9a33cf0f22051860", "sponge-f601fc1be05ad5cb", "washing up liquid-0d20e7d7ec9bbd50", "slotted spoon-904ae21c08131332", "glass pyrex bowl-de35551755b3e88e", "plastic bottle-a874fbc3589d207e", "jar of sugar-a20fdf44d55edf06", "measuring scale-9cca96148a5b25c5", "electric kettle-85ddc3132e12319c", "hand towel-ee49c41803e77403", "bag of sugar-df89eea1ea74ef17", "lid of sugar jar-d70d69ccfebdaed1", "bowl2-f04f64f4bf0548dc", "tablespoon-2242a9fd1edd6210", "plastic bag-9bef70736f343cee", "beaker-8dc2db9e886bf2d1", "slotted spoon2-a3ce16843bdeea34", "saucepan2-20ff412c86929a78", "pan support3-1413dfccf7fc08af", "pan support-14aa42ce74c03542", "second pan support-fd984f90b9e94d90", "bowl3-8419653396076e38", "food bin-69fb56fbcded9428", "hand towel rack-ccae7b6f11bab0df", "rag2-f82a2a190f8ef4c7", "pan supports-dec1eacef86a4661", "saucepan3-788be97009d408cb", "frying pan-6a96247d28c538eb", "cooking oil bottle-94e14a7bfff57e11", "fork-7f404485fbc02abf", "bowl4-4d02638eba204b05", "glass jar1-576e45fd52e46563", "glass jar2-c4121a19994dd451", "jars caps-f7c2b2a7c948673d", "frying pan2-afba7b615bae2db0", "bowl5-3e1bb55a1455b933", "bowl6-a88e8e2e4f011cf0", "bowl7-4f37992a9767350b", "new spoon-8f1c86d775666a13", "teaspoon-a7b18c3f978549f8", "bowl8-350bb67effaed309", "rag3-f2d434cb72261999", "cup of yogurt-539327b73dcd975c" ], "reasoning": "1. I watched the video to listen for when the stove igniter went off for the first time and heard this happen between 07:31-07:32 3 times.\n2. I then heard the stove igniter go off 12 times between 11:05-11:08.\n3. I then heard the stove igniter go off 9 times between 11:25-11:27.\n4. Therefore, the stove igniter went off 24 times.", "referenced_object_masks": [] }, { "key": "P09-20240622-162302:88dd90b28d3bc4c62b2561dca6e8d0fe83109c56", "video_id": "P09-20240622-162302", "question": "How many times do I use my finger to shovel sugar onto the spoon?", "answer_choice_0": "15 times", "answer_choice_1": "10 times", "answer_choice_2": "24 times", "answer_choice_3": "17 times", "answer_choice_4": "8 times", "answer_id": 3, "answer": "17 times", "question_type": [ "Action Recognition", "Object Recognition", "Counting" ], "objects_of_interest": [ "saucepan-909e681eb4fdd646", "kitchen rag-299becd1209b2b21", "bowl-7227396ef916d3b0", "lid of saucepan-f527258b20dd020d", "new serving spoon-9a33cf0f22051860", "sponge-f601fc1be05ad5cb", "washing up liquid-0d20e7d7ec9bbd50", "slotted spoon-904ae21c08131332", "glass pyrex bowl-de35551755b3e88e", "plastic bottle-a874fbc3589d207e", "jar of sugar-a20fdf44d55edf06", "measuring scale-9cca96148a5b25c5", "electric kettle-85ddc3132e12319c", "hand towel-ee49c41803e77403", "bag of sugar-df89eea1ea74ef17", "lid of sugar jar-d70d69ccfebdaed1", "bowl2-f04f64f4bf0548dc", "tablespoon-2242a9fd1edd6210", "plastic bag-9bef70736f343cee", "beaker-8dc2db9e886bf2d1", "slotted spoon2-a3ce16843bdeea34", "saucepan2-20ff412c86929a78", "pan support3-1413dfccf7fc08af", "pan support-14aa42ce74c03542", "second pan support-fd984f90b9e94d90", "bowl3-8419653396076e38", "food bin-69fb56fbcded9428", "hand towel rack-ccae7b6f11bab0df", "rag2-f82a2a190f8ef4c7", "pan supports-dec1eacef86a4661", "saucepan3-788be97009d408cb", "frying pan-6a96247d28c538eb", "cooking oil bottle-94e14a7bfff57e11", "fork-7f404485fbc02abf", "bowl4-4d02638eba204b05", "glass jar1-576e45fd52e46563", "glass jar2-c4121a19994dd451", "jars caps-f7c2b2a7c948673d", "frying pan2-afba7b615bae2db0", "bowl5-3e1bb55a1455b933", "bowl6-a88e8e2e4f011cf0", "bowl7-4f37992a9767350b", "new spoon-8f1c86d775666a13", "teaspoon-a7b18c3f978549f8", "bowl8-350bb67effaed309", "rag3-f2d434cb72261999", "cup of yogurt-539327b73dcd975c" ], "reasoning": "1. I watched the video to see when the user shoveled sugar onto the spoon (tablespoon-2242a9fd1edd6210) using their finger for the first time and saw this happen 2 times at 04:15.\n2. I then observed the user shoveling sugar onto the spoon (tablespoon-2242a9fd1edd6210) using their finger 4 times between 4:17-04:19.\n3. I then observed the user shoveling sugar onto the spoon (tablespoon-2242a9fd1edd6210) using their finger 5 times between 4:22-04:25.\n4. I then observed the user shoveling sugar onto the spoon (tablespoon-2242a9fd1edd6210) using their finger 1 time at 4:31.\n5. I then observed the user shoveling sugar onto the spoon (tablespoon-2242a9fd1edd6210) using their finger 1 time at 4:39.\n6. I then observed the user shoveling sugar onto the spoon (tablespoon-2242a9fd1edd6210) using their finger 1 time at 4:41.\n7. I then observed the user shoveling sugar onto the spoon (tablespoon-2242a9fd1edd6210) using their finger 1 time at 4:43.\n8. I then observed the user shoveling sugar onto the spoon (tablespoon-2242a9fd1edd6210) using their finger 1 time at 4:47.\n9. I then observed the user shoveling sugar onto the spoon (tablespoon-2242a9fd1edd6210) using their finger 1 time at 4:54.\n10. Therefore, the number of times the user did this comes to 17.", "referenced_object_masks": [ "tablespoon-2242a9fd1edd6210" ] }, { "key": "P09-20240622-162302:8efecfc51ec6aa5cf813b15b4ed90f2eeb862dd9", "video_id": "P09-20240622-162302", "question": "If I was standing at the sink, turned 180 degrees to the right and started walking forward, what would I come in contact with first?", "answer_choice_0": "Refrigerator", "answer_choice_1": "Sliding glass door", "answer_choice_2": "Counter with microwave", "answer_choice_3": "Living room", "answer_choice_4": "Stove", "answer_id": 2, "answer": "Counter with microwave", "question_type": [ "Action Recognition", "Counterfactual", "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "saucepan-909e681eb4fdd646", "kitchen rag-299becd1209b2b21", "bowl-7227396ef916d3b0", "lid of saucepan-f527258b20dd020d", "new serving spoon-9a33cf0f22051860", "sponge-f601fc1be05ad5cb", "washing up liquid-0d20e7d7ec9bbd50", "slotted spoon-904ae21c08131332", "glass pyrex bowl-de35551755b3e88e", "plastic bottle-a874fbc3589d207e", "jar of sugar-a20fdf44d55edf06", "measuring scale-9cca96148a5b25c5", "electric kettle-85ddc3132e12319c", "hand towel-ee49c41803e77403", "bag of sugar-df89eea1ea74ef17", "lid of sugar jar-d70d69ccfebdaed1", "bowl2-f04f64f4bf0548dc", "tablespoon-2242a9fd1edd6210", "plastic bag-9bef70736f343cee", "beaker-8dc2db9e886bf2d1", "slotted spoon2-a3ce16843bdeea34", "saucepan2-20ff412c86929a78", "pan support3-1413dfccf7fc08af", "pan support-14aa42ce74c03542", "second pan support-fd984f90b9e94d90", "bowl3-8419653396076e38", "food bin-69fb56fbcded9428", "hand towel rack-ccae7b6f11bab0df", "rag2-f82a2a190f8ef4c7", "pan supports-dec1eacef86a4661", "saucepan3-788be97009d408cb", "frying pan-6a96247d28c538eb", "cooking oil bottle-94e14a7bfff57e11", "fork-7f404485fbc02abf", "bowl4-4d02638eba204b05", "glass jar1-576e45fd52e46563", "glass jar2-c4121a19994dd451", "jars caps-f7c2b2a7c948673d", "frying pan2-afba7b615bae2db0", "bowl5-3e1bb55a1455b933", "bowl6-a88e8e2e4f011cf0", "bowl7-4f37992a9767350b", "new spoon-8f1c86d775666a13", "teaspoon-a7b18c3f978549f8", "bowl8-350bb67effaed309", "rag3-f2d434cb72261999", "cup of yogurt-539327b73dcd975c" ], "reasoning": "1. At 02:54, I saw the user at the sink washing a plastic water bottle (plastic bottle-a874fbc3589d207e).\n2. At 03:10, I saw the user turn 180 degrees from the sink and saw that the counter with the microwave was directly behind them.\n3. Reasoning about the scene, I realized that the first object the user would come in contact with if they turned 180 degrees to the right and walked forward would be the counter with the microwave .", "referenced_object_masks": [ "plastic bottle-a874fbc3589d207e" ] }, { "key": "P09-20240622-162302:97ef842510c2e37107a3209d6b94fd4abfd8ec80", "video_id": "P09-20240622-162302", "question": "If I was standing facing the kitchen sink, what directions would I have to take to reach my hamper?", "answer_choice_0": "Turn around, walk forward, turn left, walk forward", "answer_choice_1": "Turn left, walk forward, turn left, walk forward", "answer_choice_2": "Turn left, walk forward, turn right, walk forward", "answer_choice_3": "Turn right, walk forward, turn left, walk forward", "answer_choice_4": "Turn around, walk forward", "answer_id": 2, "answer": "Turn left, walk forward, turn right, walk forward", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Counterfactual" ], "objects_of_interest": [ "saucepan-909e681eb4fdd646", "kitchen rag-299becd1209b2b21", "bowl-7227396ef916d3b0", "lid of saucepan-f527258b20dd020d", "new serving spoon-9a33cf0f22051860", "sponge-f601fc1be05ad5cb", "washing up liquid-0d20e7d7ec9bbd50", "slotted spoon-904ae21c08131332", "glass pyrex bowl-de35551755b3e88e", "plastic bottle-a874fbc3589d207e", "jar of sugar-a20fdf44d55edf06", "measuring scale-9cca96148a5b25c5", "electric kettle-85ddc3132e12319c", "hand towel-ee49c41803e77403", "bag of sugar-df89eea1ea74ef17", "lid of sugar jar-d70d69ccfebdaed1", "bowl2-f04f64f4bf0548dc", "tablespoon-2242a9fd1edd6210", "plastic bag-9bef70736f343cee", "beaker-8dc2db9e886bf2d1", "slotted spoon2-a3ce16843bdeea34", "saucepan2-20ff412c86929a78", "pan support3-1413dfccf7fc08af", "pan support-14aa42ce74c03542", "second pan support-fd984f90b9e94d90", "bowl3-8419653396076e38", "food bin-69fb56fbcded9428", "hand towel rack-ccae7b6f11bab0df", "rag2-f82a2a190f8ef4c7", "pan supports-dec1eacef86a4661", "saucepan3-788be97009d408cb", "frying pan-6a96247d28c538eb", "cooking oil bottle-94e14a7bfff57e11", "fork-7f404485fbc02abf", "bowl4-4d02638eba204b05", "glass jar1-576e45fd52e46563", "glass jar2-c4121a19994dd451", "jars caps-f7c2b2a7c948673d", "frying pan2-afba7b615bae2db0", "bowl5-3e1bb55a1455b933", "bowl6-a88e8e2e4f011cf0", "bowl7-4f37992a9767350b", "new spoon-8f1c86d775666a13", "teaspoon-a7b18c3f978549f8", "bowl8-350bb67effaed309", "rag3-f2d434cb72261999", "cup of yogurt-539327b73dcd975c" ], "reasoning": "1. I watched the video to see the positioning of the user when they were standing facing the kitchen sink and saw this position at 00:24.\n2. I then observed the video to observe the location of the hamper relative to the sink and saw the hamper at 09:11.\n3. I saw that between 09:08-09:11, the hamper appears to be around the corner from the sink on the user's left side, so the directions the user would have to take would be turn left, walk forward, turn right, walk forward.", "referenced_object_masks": [] }, { "key": "P09-20240622-162302:b9bccbf0bc8f7101215c3366e2647ebaea945851", "video_id": "P09-20240622-162302", "question": "How many times did the red liquid switch containers?", "answer_choice_0": "Once", "answer_choice_1": "5 times", "answer_choice_2": "Twice", "answer_choice_3": "4 times", "answer_choice_4": "Thrice", "answer_id": 4, "answer": "Thrice", "question_type": [ "Action Recognition", "Object Recognition", "Counting" ], "objects_of_interest": [ "saucepan-909e681eb4fdd646", "kitchen rag-299becd1209b2b21", "bowl-7227396ef916d3b0", "lid of saucepan-f527258b20dd020d", "new serving spoon-9a33cf0f22051860", "sponge-f601fc1be05ad5cb", "washing up liquid-0d20e7d7ec9bbd50", "slotted spoon-904ae21c08131332", "glass pyrex bowl-de35551755b3e88e", "plastic bottle-a874fbc3589d207e", "jar of sugar-a20fdf44d55edf06", "measuring scale-9cca96148a5b25c5", "electric kettle-85ddc3132e12319c", "hand towel-ee49c41803e77403", "bag of sugar-df89eea1ea74ef17", "lid of sugar jar-d70d69ccfebdaed1", "bowl2-f04f64f4bf0548dc", "tablespoon-2242a9fd1edd6210", "plastic bag-9bef70736f343cee", "beaker-8dc2db9e886bf2d1", "slotted spoon2-a3ce16843bdeea34", "saucepan2-20ff412c86929a78", "pan support3-1413dfccf7fc08af", "pan support-14aa42ce74c03542", "second pan support-fd984f90b9e94d90", "bowl3-8419653396076e38", "food bin-69fb56fbcded9428", "hand towel rack-ccae7b6f11bab0df", "rag2-f82a2a190f8ef4c7", "pan supports-dec1eacef86a4661", "saucepan3-788be97009d408cb", "frying pan-6a96247d28c538eb", "cooking oil bottle-94e14a7bfff57e11", "fork-7f404485fbc02abf", "bowl4-4d02638eba204b05", "glass jar1-576e45fd52e46563", "glass jar2-c4121a19994dd451", "jars caps-f7c2b2a7c948673d", "frying pan2-afba7b615bae2db0", "bowl5-3e1bb55a1455b933", "bowl6-a88e8e2e4f011cf0", "bowl7-4f37992a9767350b", "new spoon-8f1c86d775666a13", "teaspoon-a7b18c3f978549f8", "bowl8-350bb67effaed309", "rag3-f2d434cb72261999", "cup of yogurt-539327b73dcd975c" ], "reasoning": "1. I watched the video to see the first time the user poured out the red liquid and saw this happen at 01:25 when the user poured it into a bowl (glass pyrex bowl-de35551755b3e88e).\n2. I then observed the user pouring the red liquid from the bowl (glass pyrex bowl-de35551755b3e88e) into the measuring cup (beaker-8dc2db9e886bf2d1) at 06:45.\n3. I then observed the user pouring the red liquid from the measuring cup (beaker-8dc2db9e886bf2d1) into the water bottle (plastic bottle-a874fbc3589d207e) at 06:52.\n4. Therefore, the red liquid switched containers 3 times.", "referenced_object_masks": [ "glass pyrex bowl-de35551755b3e88e", "plastic bottle-a874fbc3589d207e", "beaker-8dc2db9e886bf2d1" ] }, { "key": "P09-20240622-162302:d7b0bb906d7f6d019217589daa311192bf9e913c", "video_id": "P09-20240622-162302", "question": "How many times do I shake the water bottle?", "answer_choice_0": "18 times", "answer_choice_1": "10 times", "answer_choice_2": "36 times", "answer_choice_3": "24 times", "answer_choice_4": "14 times", "answer_id": 3, "answer": "24 times", "question_type": [ "Action Recognition", "Object Recognition", "Counting" ], "objects_of_interest": [ "saucepan-909e681eb4fdd646", "kitchen rag-299becd1209b2b21", "bowl-7227396ef916d3b0", "lid of saucepan-f527258b20dd020d", "new serving spoon-9a33cf0f22051860", "sponge-f601fc1be05ad5cb", "washing up liquid-0d20e7d7ec9bbd50", "slotted spoon-904ae21c08131332", "glass pyrex bowl-de35551755b3e88e", "plastic bottle-a874fbc3589d207e", "jar of sugar-a20fdf44d55edf06", "measuring scale-9cca96148a5b25c5", "electric kettle-85ddc3132e12319c", "hand towel-ee49c41803e77403", "bag of sugar-df89eea1ea74ef17", "lid of sugar jar-d70d69ccfebdaed1", "bowl2-f04f64f4bf0548dc", "tablespoon-2242a9fd1edd6210", "plastic bag-9bef70736f343cee", "beaker-8dc2db9e886bf2d1", "slotted spoon2-a3ce16843bdeea34", "saucepan2-20ff412c86929a78", "pan support3-1413dfccf7fc08af", "pan support-14aa42ce74c03542", "second pan support-fd984f90b9e94d90", "bowl3-8419653396076e38", "food bin-69fb56fbcded9428", "hand towel rack-ccae7b6f11bab0df", "rag2-f82a2a190f8ef4c7", "pan supports-dec1eacef86a4661", "saucepan3-788be97009d408cb", "frying pan-6a96247d28c538eb", "cooking oil bottle-94e14a7bfff57e11", "fork-7f404485fbc02abf", "bowl4-4d02638eba204b05", "glass jar1-576e45fd52e46563", "glass jar2-c4121a19994dd451", "jars caps-f7c2b2a7c948673d", "frying pan2-afba7b615bae2db0", "bowl5-3e1bb55a1455b933", "bowl6-a88e8e2e4f011cf0", "bowl7-4f37992a9767350b", "new spoon-8f1c86d775666a13", "teaspoon-a7b18c3f978549f8", "bowl8-350bb67effaed309", "rag3-f2d434cb72261999", "cup of yogurt-539327b73dcd975c" ], "reasoning": "1. I watched the video to see when the user shook the water bottle (plastic bottle-a874fbc3589d207e) for the first time and saw this happen 2 times at 02:25.\n2. I then observed the user shaking the water bottle (plastic bottle-a874fbc3589d207e) 17 times between 02:41-02:46.\n3. I then observed the user shaking the water bottle (plastic bottle-a874fbc3589d207e) 2 times at 02:57.\n4. I then observed the user shaking the water bottle (plastic bottle-a874fbc3589d207e) 3 times at 03:04.\n5. Therefore, the user shook the water bottle 24 times.", "referenced_object_masks": [ "plastic bottle-a874fbc3589d207e" ] }, { "key": "P09-20240622-162302:f137d1ba431a07cf0ae4ec8839cd492b5d681bcb", "video_id": "P09-20240622-162302", "question": "About how many degrees did I move the red pan's handle counterclockwise in order to make room for the frying pan?", "answer_choice_0": "150 degrees", "answer_choice_1": "180 degrees", "answer_choice_2": "90 degrees", "answer_choice_3": "100 degrees", "answer_choice_4": "45 degrees", "answer_id": 0, "answer": "150 degrees", "question_type": [ "Action Recognition", "Egocentric & Spatial Perception", "Object Recognition" ], "objects_of_interest": [ "saucepan-909e681eb4fdd646", "kitchen rag-299becd1209b2b21", "bowl-7227396ef916d3b0", "lid of saucepan-f527258b20dd020d", "new serving spoon-9a33cf0f22051860", "sponge-f601fc1be05ad5cb", "washing up liquid-0d20e7d7ec9bbd50", "slotted spoon-904ae21c08131332", "glass pyrex bowl-de35551755b3e88e", "plastic bottle-a874fbc3589d207e", "jar of sugar-a20fdf44d55edf06", "measuring scale-9cca96148a5b25c5", "electric kettle-85ddc3132e12319c", "hand towel-ee49c41803e77403", "bag of sugar-df89eea1ea74ef17", "lid of sugar jar-d70d69ccfebdaed1", "bowl2-f04f64f4bf0548dc", "tablespoon-2242a9fd1edd6210", "plastic bag-9bef70736f343cee", "beaker-8dc2db9e886bf2d1", "slotted spoon2-a3ce16843bdeea34", "saucepan2-20ff412c86929a78", "pan support3-1413dfccf7fc08af", "pan support-14aa42ce74c03542", "second pan support-fd984f90b9e94d90", "bowl3-8419653396076e38", "food bin-69fb56fbcded9428", "hand towel rack-ccae7b6f11bab0df", "rag2-f82a2a190f8ef4c7", "pan supports-dec1eacef86a4661", "saucepan3-788be97009d408cb", "frying pan-6a96247d28c538eb", "cooking oil bottle-94e14a7bfff57e11", "fork-7f404485fbc02abf", "bowl4-4d02638eba204b05", "glass jar1-576e45fd52e46563", "glass jar2-c4121a19994dd451", "jars caps-f7c2b2a7c948673d", "frying pan2-afba7b615bae2db0", "bowl5-3e1bb55a1455b933", "bowl6-a88e8e2e4f011cf0", "bowl7-4f37992a9767350b", "new spoon-8f1c86d775666a13", "teaspoon-a7b18c3f978549f8", "bowl8-350bb67effaed309", "rag3-f2d434cb72261999", "cup of yogurt-539327b73dcd975c" ], "reasoning": "1. I watched the whole video and saw the user place the red pan (saucepan3-788be97009d408cb) on the stove at 11:03.\n2. At 11:15, the user went to the drying rack to get a frying pan (frying pan-6a96247d28c538eb).\n3. To make room for the frying pan (frying pan-6a96247d28c538eb), the user moved the handle of the red pan (saucepan3-788be97009d408cb) at 11:19.\n4. Using the unit circle, the user moved the handle about 150 degrees counterclockwise.", "referenced_object_masks": [ "frying pan-6a96247d28c538eb", "saucepan3-788be97009d408cb" ] }, { "key": "P09-20240622-162302:f6fc5105736b243b17d365a78e297beffa36547c", "video_id": "P09-20240622-162302", "question": "If I take the number of seconds it takes for the user to pour the fruit liquid from the measuring cup to the plastic bottle, and then add the number of times I picked up the white tea kettle during the video, what number would I get?", "answer_choice_0": "19", "answer_choice_1": "15", "answer_choice_2": "13", "answer_choice_3": "11", "answer_choice_4": "16", "answer_id": 2, "answer": "13", "question_type": [ "Action Recognition", "Counting", "Numerical Reasoning", "Object Recognition" ], "objects_of_interest": [ "saucepan-909e681eb4fdd646", "kitchen rag-299becd1209b2b21", "bowl-7227396ef916d3b0", "lid of saucepan-f527258b20dd020d", "new serving spoon-9a33cf0f22051860", "sponge-f601fc1be05ad5cb", "washing up liquid-0d20e7d7ec9bbd50", "slotted spoon-904ae21c08131332", "glass pyrex bowl-de35551755b3e88e", "plastic bottle-a874fbc3589d207e", "jar of sugar-a20fdf44d55edf06", "measuring scale-9cca96148a5b25c5", "electric kettle-85ddc3132e12319c", "hand towel-ee49c41803e77403", "bag of sugar-df89eea1ea74ef17", "lid of sugar jar-d70d69ccfebdaed1", "bowl2-f04f64f4bf0548dc", "tablespoon-2242a9fd1edd6210", "plastic bag-9bef70736f343cee", "beaker-8dc2db9e886bf2d1", "slotted spoon2-a3ce16843bdeea34", "saucepan2-20ff412c86929a78", "pan support3-1413dfccf7fc08af", "pan support-14aa42ce74c03542", "second pan support-fd984f90b9e94d90", "bowl3-8419653396076e38", "food bin-69fb56fbcded9428", "hand towel rack-ccae7b6f11bab0df", "rag2-f82a2a190f8ef4c7", "pan supports-dec1eacef86a4661", "saucepan3-788be97009d408cb", "frying pan-6a96247d28c538eb", "cooking oil bottle-94e14a7bfff57e11", "fork-7f404485fbc02abf", "bowl4-4d02638eba204b05", "glass jar1-576e45fd52e46563", "glass jar2-c4121a19994dd451", "jars caps-f7c2b2a7c948673d", "frying pan2-afba7b615bae2db0", "bowl5-3e1bb55a1455b933", "bowl6-a88e8e2e4f011cf0", "bowl7-4f37992a9767350b", "new spoon-8f1c86d775666a13", "teaspoon-a7b18c3f978549f8", "bowl8-350bb67effaed309", "rag3-f2d434cb72261999", "cup of yogurt-539327b73dcd975c" ], "reasoning": "1. I watched the whole video and saw the user start pouring the fruit liquid into the plastic bottle (plastic bottle-a874fbc3589d207e) from 06:51-07:02, making the entire pouring time 11 seconds.\n2. I saw the user pick up the white tea kettle (electric kettle-85ddc3132e12319c) at 02:12 to fill it with water .\n4. I saw that the second time the user picks up the white tea kettle (electric kettle-85ddc3132e12319c) is at 05:27, making the total number of times user picks up the white tea kettle (electric kettle-85ddc3132e12319c) 2.\n5. I then used the following equation to arrive at the answer: 11+2=13", "referenced_object_masks": [ "electric kettle-85ddc3132e12319c", "plastic bottle-a874fbc3589d207e" ] }, { "key": "P09-20240622-162302:fa612f0a5e54decf80b305c23e8f13ba6f4c9dda", "video_id": "P09-20240622-162302", "question": "How many bowls did I interact with?", "answer_choice_0": "2", "answer_choice_1": "3", "answer_choice_2": "5", "answer_choice_3": "4", "answer_choice_4": "1", "answer_id": 3, "answer": "4", "question_type": [ "Action Recognition", "Counting", "Object Recognition" ], "objects_of_interest": [ "saucepan-909e681eb4fdd646", "kitchen rag-299becd1209b2b21", "bowl-7227396ef916d3b0", "lid of saucepan-f527258b20dd020d", "new serving spoon-9a33cf0f22051860", "sponge-f601fc1be05ad5cb", "washing up liquid-0d20e7d7ec9bbd50", "slotted spoon-904ae21c08131332", "glass pyrex bowl-de35551755b3e88e", "plastic bottle-a874fbc3589d207e", "jar of sugar-a20fdf44d55edf06", "measuring scale-9cca96148a5b25c5", "electric kettle-85ddc3132e12319c", "hand towel-ee49c41803e77403", "bag of sugar-df89eea1ea74ef17", "lid of sugar jar-d70d69ccfebdaed1", "bowl2-f04f64f4bf0548dc", "tablespoon-2242a9fd1edd6210", "plastic bag-9bef70736f343cee", "beaker-8dc2db9e886bf2d1", "slotted spoon2-a3ce16843bdeea34", "saucepan2-20ff412c86929a78", "pan support3-1413dfccf7fc08af", "pan support-14aa42ce74c03542", "second pan support-fd984f90b9e94d90", "bowl3-8419653396076e38", "food bin-69fb56fbcded9428", "hand towel rack-ccae7b6f11bab0df", "rag2-f82a2a190f8ef4c7", "pan supports-dec1eacef86a4661", "saucepan3-788be97009d408cb", "frying pan-6a96247d28c538eb", "cooking oil bottle-94e14a7bfff57e11", "fork-7f404485fbc02abf", "bowl4-4d02638eba204b05", "glass jar1-576e45fd52e46563", "glass jar2-c4121a19994dd451", "jars caps-f7c2b2a7c948673d", "frying pan2-afba7b615bae2db0", "bowl5-3e1bb55a1455b933", "bowl6-a88e8e2e4f011cf0", "bowl7-4f37992a9767350b", "new spoon-8f1c86d775666a13", "teaspoon-a7b18c3f978549f8", "bowl8-350bb67effaed309", "rag3-f2d434cb72261999", "cup of yogurt-539327b73dcd975c" ], "reasoning": "1. I watched the whole video and saw at 00:04, there were already 2 bowls (bowl-7227396ef916d3b0, bowl3-8419653396076e38) being used on the counter. One had a whisked egg in it (bowl-7227396ef916d3b0) and the other had a white powder substance in it (bowl3-8419653396076e38).\n2. The user pours the egg from the first bowl (bowl-7227396ef916d3b0) into the frying pan (frying pan-6a96247d28c538eb) at 12:00.\n3. At 08:05, the user picks up the bowl (bowl3-8419653396076e38) with the white substance and carries it (bowl3-8419653396076e38) to the food bin (food bin-69fb56fbcded9428) and pours the white substance in.\n4. At 01:15, the user got a clear, glass bowl (glass pyrex bowl-de35551755b3e88e) from the drying rack to the left of the sink .\n5. At 03:55, the user went back to the drying rack to get a blue bowl (bowl2-f04f64f4bf0548dc).\n6. So the total number of bowls the user interacts with in the video is 4.", "referenced_object_masks": [ "bowl-7227396ef916d3b0", "bowl3-8419653396076e38", "frying pan-6a96247d28c538eb", "bowl2-f04f64f4bf0548dc", "glass pyrex bowl-de35551755b3e88e", "food bin-69fb56fbcded9428" ] }, { "key": "P09-20240622-194642:0bf99556fdc7e584a227b5c8f6fc347f736a20a8", "video_id": "P09-20240622-194642", "question": "How much time passes between when I turn on the burner and when I put food into the pan on that burner?", "answer_choice_0": "01:12", "answer_choice_1": "02:00", "answer_choice_2": "01:48", "answer_choice_3": "02:02", "answer_choice_4": "01:36", "answer_id": 2, "answer": "01:48", "question_type": [ "Temporal Reasoning", "Event Occurence", "Action Recognition", "Object Recognition" ], "objects_of_interest": [ "container of leftovers-83d0741827ea30a2", "saucepan of used oil-f67d1a8060cc921b", "cling film-e931a1a869c7cf37", "chicken ball1-11ba7393bc900c9c", "measuring beaker-e23b59493bda3cf4", "red saucepan-851a91b556b78af9", "white mug-42d0331a34ee0727", "green plate-ce549c275b7fd411", "tupperware-40641ac742ec814b", "white bowl-4ee92122147bd085", "slotted spoon-d37c4b8ae192514f", "slotted spoon2-bb49f34c59fcd526", "saucepan cover-e6c38033502eb676", "red saucepan2-8534a0503ec919c7", "pan lid-6af2198668d34786", "fork-17d688525e399be9", "glass pyrex bowl-ad2873b58971b199", "patterned bowl-f2ce6a31ed6946f0", "patterned bowl2-d1450dd5c9b1b477", "patterned bowl3-5eb9767c6da03f10", "patterned bowl4-347990598ced8b12", "item-2f28be82441abc05", "items2-e3411dec8d26ddb8", "container of yogurt-8b935296835bd904", "bottle of hibiscus drink-7a070b5a82db0f51", "discarded cling film wrap-ddcf34944c61ad5f", "sponge-df62d82d5f2d52e6", "ice cube tray-eb0c43cc4f5fef08", "bottle of washing up liquid-dc2ac6fa79e333d9", "ice cube tray2-8acbc1d3a433d458", "freezer item-e77780e43a9f5f5b", "package of frozen meat-5faba3c1d60b2439", "black cup-1919699a8953d38a", "tea towel-af1842aaead2b351", "drying rack-7553d825526a865e", "mug-76deac82f3351dde" ], "reasoning": "1. I watched the video to first identify when the burner is turned on, noting that the user turns the igniter from 00:13-00:17. Since 00:17 marked the end of the time the user turned on the burner, I used 00:17 to calculate lapsed time.\n2. I then continued the video to see that the user puts a small food item (chicken ball1-11ba7393bc900c9c) into the pan (saucepan of used oil-f67d1a8060cc921b) at 02:05.\n3. I then did the calculation of 02:05 - 00:17 to find that 01:48 passed between these two actions.", "referenced_object_masks": [ "chicken ball1-11ba7393bc900c9c", "saucepan of used oil-f67d1a8060cc921b" ] }, { "key": "P09-20240622-194642:272dd52fd596dd5a10df0ab8e816e60ff3756a60", "video_id": "P09-20240622-194642", "question": "How much time passes between when I remove the leftovers from the freezer and when I put them back?", "answer_choice_0": "01:21", "answer_choice_1": "01:51", "answer_choice_2": "01:56", "answer_choice_3": "02:01", "answer_choice_4": "02:31", "answer_id": 4, "answer": "02:31", "question_type": [ "Action Recognition", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "container of leftovers-83d0741827ea30a2", "saucepan of used oil-f67d1a8060cc921b", "cling film-e931a1a869c7cf37", "chicken ball1-11ba7393bc900c9c", "measuring beaker-e23b59493bda3cf4", "red saucepan-851a91b556b78af9", "white mug-42d0331a34ee0727", "green plate-ce549c275b7fd411", "tupperware-40641ac742ec814b", "white bowl-4ee92122147bd085", "slotted spoon-d37c4b8ae192514f", "slotted spoon2-bb49f34c59fcd526", "saucepan cover-e6c38033502eb676", "red saucepan2-8534a0503ec919c7", "pan lid-6af2198668d34786", "fork-17d688525e399be9", "glass pyrex bowl-ad2873b58971b199", "patterned bowl-f2ce6a31ed6946f0", "patterned bowl2-d1450dd5c9b1b477", "patterned bowl3-5eb9767c6da03f10", "patterned bowl4-347990598ced8b12", "item-2f28be82441abc05", "items2-e3411dec8d26ddb8", "container of yogurt-8b935296835bd904", "bottle of hibiscus drink-7a070b5a82db0f51", "discarded cling film wrap-ddcf34944c61ad5f", "sponge-df62d82d5f2d52e6", "ice cube tray-eb0c43cc4f5fef08", "bottle of washing up liquid-dc2ac6fa79e333d9", "ice cube tray2-8acbc1d3a433d458", "freezer item-e77780e43a9f5f5b", "package of frozen meat-5faba3c1d60b2439", "black cup-1919699a8953d38a", "tea towel-af1842aaead2b351", "drying rack-7553d825526a865e", "mug-76deac82f3351dde" ], "reasoning": "1. I began watching the video and saw the user remove the leftovers (container of leftovers-83d0741827ea30a2) from the freezer from 00:06-00:08. \n2. I continued watching the video until 02:37-02:39, when I observed the user open the freezer and return the leftovers (container of leftovers-83d0741827ea30a2).\n3. Lastly, I did the math and determined that 02:31 passed using the equation: 02:39 - 00:08 = 02:31).", "referenced_object_masks": [ "container of leftovers-83d0741827ea30a2" ] }, { "key": "P09-20240622-194642:3118424982686fb24b70778b3193722f7f0e1f5d", "video_id": "P09-20240622-194642", "question": "How many unique items do I touch with my left hand in the first 30 seconds of the video?", "answer_choice_0": "2", "answer_choice_1": "8", "answer_choice_2": "5", "answer_choice_3": "0", "answer_choice_4": "7", "answer_id": 2, "answer": "5", "question_type": [ "Action Recognition", "Object Recognition", "Temporal Reasoning", "Counting" ], "objects_of_interest": [ "container of leftovers-83d0741827ea30a2", "saucepan of used oil-f67d1a8060cc921b", "cling film-e931a1a869c7cf37", "chicken ball1-11ba7393bc900c9c", "measuring beaker-e23b59493bda3cf4", "red saucepan-851a91b556b78af9", "white mug-42d0331a34ee0727", "green plate-ce549c275b7fd411", "tupperware-40641ac742ec814b", "white bowl-4ee92122147bd085", "slotted spoon-d37c4b8ae192514f", "slotted spoon2-bb49f34c59fcd526", "saucepan cover-e6c38033502eb676", "red saucepan2-8534a0503ec919c7", "pan lid-6af2198668d34786", "fork-17d688525e399be9", "glass pyrex bowl-ad2873b58971b199", "patterned bowl-f2ce6a31ed6946f0", "patterned bowl2-d1450dd5c9b1b477", "patterned bowl3-5eb9767c6da03f10", "patterned bowl4-347990598ced8b12", "item-2f28be82441abc05", "items2-e3411dec8d26ddb8", "container of yogurt-8b935296835bd904", "bottle of hibiscus drink-7a070b5a82db0f51", "discarded cling film wrap-ddcf34944c61ad5f", "sponge-df62d82d5f2d52e6", "ice cube tray-eb0c43cc4f5fef08", "bottle of washing up liquid-dc2ac6fa79e333d9", "ice cube tray2-8acbc1d3a433d458", "freezer item-e77780e43a9f5f5b", "package of frozen meat-5faba3c1d60b2439", "black cup-1919699a8953d38a", "tea towel-af1842aaead2b351", "drying rack-7553d825526a865e", "mug-76deac82f3351dde" ], "reasoning": "1. First, I observed the user open the freezer door with their right hand before transitioning the door to their left hand from 00:03-00:05. This is the first item touched with the left hand.\n2. I continued to watch as the user picked up the cling film-wrapped leftovers (container of leftovers-83d0741827ea30a2) from the freezer with their right hand at 00:07. The user places the leftovers (container of leftovers-83d0741827ea30a2) on the counter with their left hand at 00:12. The switch from right to left hand occurred off camera from 00:10-00:12. This makes the cling film (cling film-e931a1a869c7cf37) the second item touched with the left hand, since it is the outermost layer of the leftovers (container of leftovers-83d0741827ea30a2). \n3. Next, the user approached the stovetop and turned the igniter knob with their left hand from 00:13-00:17. This marks the third item touched with the left hand in the first 30 seconds.\n4. Then, at 00:21, I saw the user unwrap the leftovers (container of leftovers-83d0741827ea30a2) with both hands, but I did not count the cling wrap (cling film-e931a1a869c7cf37) a second time since the user had already touched it. \n5. At 00:25, I saw the user clearly touch the foil wrapping of the leftovers with their left hand, making this the fourth unique object they touch. \n6. Finally, at 00:28, I saw the user touch a chicken ball inside the wrapping with their left hand, making it the fifth and final unique object touched in the first 30 seconds.", "referenced_object_masks": [ "container of leftovers-83d0741827ea30a2", "cling film-e931a1a869c7cf37" ] }, { "key": "P09-20240622-194642:4a9f555cd4e93034d2bbeaab3c96eaacebc237c5", "video_id": "P09-20240622-194642", "question": "When I put the plate away, approximately how many degrees would I have to turn to the left to face the sink?", "answer_choice_0": "30", "answer_choice_1": "150", "answer_choice_2": "90", "answer_choice_3": "180", "answer_choice_4": "270", "answer_id": 3, "answer": "180", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness", "Counterfactual" ], "objects_of_interest": [ "container of leftovers-83d0741827ea30a2", "saucepan of used oil-f67d1a8060cc921b", "cling film-e931a1a869c7cf37", "chicken ball1-11ba7393bc900c9c", "measuring beaker-e23b59493bda3cf4", "red saucepan-851a91b556b78af9", "white mug-42d0331a34ee0727", "green plate-ce549c275b7fd411", "tupperware-40641ac742ec814b", "white bowl-4ee92122147bd085", "slotted spoon-d37c4b8ae192514f", "slotted spoon2-bb49f34c59fcd526", "saucepan cover-e6c38033502eb676", "red saucepan2-8534a0503ec919c7", "pan lid-6af2198668d34786", "fork-17d688525e399be9", "glass pyrex bowl-ad2873b58971b199", "patterned bowl-f2ce6a31ed6946f0", "patterned bowl2-d1450dd5c9b1b477", "patterned bowl3-5eb9767c6da03f10", "patterned bowl4-347990598ced8b12", "item-2f28be82441abc05", "items2-e3411dec8d26ddb8", "container of yogurt-8b935296835bd904", "bottle of hibiscus drink-7a070b5a82db0f51", "discarded cling film wrap-ddcf34944c61ad5f", "sponge-df62d82d5f2d52e6", "ice cube tray-eb0c43cc4f5fef08", "bottle of washing up liquid-dc2ac6fa79e333d9", "ice cube tray2-8acbc1d3a433d458", "freezer item-e77780e43a9f5f5b", "package of frozen meat-5faba3c1d60b2439", "black cup-1919699a8953d38a", "tea towel-af1842aaead2b351", "drying rack-7553d825526a865e", "mug-76deac82f3351dde" ], "reasoning": "1. I watched the video and at 00:00 noted the approximate layout of the left side of the kitchen. It is a half square with the sink to the far right, and countertop between the sink and the stovetop .\n2. The user then walks forward, and at 00:02, a view of the right side of the sink is presented. To the right of the stove is more countertop, and then the microwave and the refrigerator . I took note of this. \n3. I then looked for the user to put away a plate (green plate-ce549c275b7fd411) and found that they do this at 02:21 when they put the plate (green plate-ce549c275b7fd411) in a cabinet beneath the microwave . \n4. Reasoning about the scene and realizing that the microwave is directly across from the sink , I realized that the user would need to turn approximately 180 degrees if they wanted to face the sink after putting the plate (green plate-ce549c275b7fd411) away.", "referenced_object_masks": [ "green plate-ce549c275b7fd411" ] }, { "key": "P09-20240622-194642:657c699b88398e0c04d0025ab3445afe82e82c92", "video_id": "P09-20240622-194642", "question": "How much time lapses between when I open and close the microwave?", "answer_choice_0": "00:57", "answer_choice_1": "00:52", "answer_choice_2": "00:45", "answer_choice_3": "01:45", "answer_choice_4": "01:10", "answer_id": 4, "answer": "01:10", "question_type": [ "Audio Reasoning", "Action Recognition", "Object Recognition", "Numerical Reasoning", "Temporal Reasoning" ], "objects_of_interest": [ "container of leftovers-83d0741827ea30a2", "saucepan of used oil-f67d1a8060cc921b", "cling film-e931a1a869c7cf37", "chicken ball1-11ba7393bc900c9c", "measuring beaker-e23b59493bda3cf4", "red saucepan-851a91b556b78af9", "white mug-42d0331a34ee0727", "green plate-ce549c275b7fd411", "tupperware-40641ac742ec814b", "white bowl-4ee92122147bd085", "slotted spoon-d37c4b8ae192514f", "slotted spoon2-bb49f34c59fcd526", "saucepan cover-e6c38033502eb676", "red saucepan2-8534a0503ec919c7", "pan lid-6af2198668d34786", "fork-17d688525e399be9", "glass pyrex bowl-ad2873b58971b199", "patterned bowl-f2ce6a31ed6946f0", "patterned bowl2-d1450dd5c9b1b477", "patterned bowl3-5eb9767c6da03f10", "patterned bowl4-347990598ced8b12", "item-2f28be82441abc05", "items2-e3411dec8d26ddb8", "container of yogurt-8b935296835bd904", "bottle of hibiscus drink-7a070b5a82db0f51", "discarded cling film wrap-ddcf34944c61ad5f", "sponge-df62d82d5f2d52e6", "ice cube tray-eb0c43cc4f5fef08", "bottle of washing up liquid-dc2ac6fa79e333d9", "ice cube tray2-8acbc1d3a433d458", "freezer item-e77780e43a9f5f5b", "package of frozen meat-5faba3c1d60b2439", "black cup-1919699a8953d38a", "tea towel-af1842aaead2b351", "drying rack-7553d825526a865e", "mug-76deac82f3351dde" ], "reasoning": "1. I watched the video until 02:10-02:11, when the user opened the microwave door .\n2. I continued watching until 03:21, when the user stopped at the counter with the microwave to their right, and at that time I heard the sound of the microwave door closing. Therefore, the user closed the microwave door at 03:21.\n3. Lastly, I calculated the time lapsed by using the equation: 03:21 - 02:11 = 01:10.", "referenced_object_masks": [] }, { "key": "P09-20240623-120359:25ada86affb38145e0524daaa1050564c9dd037b", "video_id": "P09-20240623-120359", "question": "In relation to me, where is the pair of cabinets under which I place the digital timer when I first grab the blender's lid?", "answer_choice_0": "Behind", "answer_choice_1": "Above and to the right", "answer_choice_2": "Directly in front", "answer_choice_3": "Under", "answer_choice_4": "Above and to the left", "answer_id": 4, "answer": "Above and to the left", "question_type": [ "Object Recognition", "Action Recognition", "Egocentric & Spatial Perception", "Temporal Reasoning" ], "objects_of_interest": [ "timer-d59705f7e130d64f", "bag-a8f088817e83b390", "knife-7ae1058802a2d588", "meat box-d91f02b6bc817f2e", "blender's lid-ead6db223b9bf936", "measuring jar-7b545b4808a5e2f8", "gray bowl-6283254f7c3bc941", "bowl2-9d8c45fa1153c18d", "blender-d7460356ec5e6cc6", "blender motor-7109bf56cf56e1f4", "spice jar-1210f4f4f01203dc", "new spice jar-fc7dc1d8c0698064", "spice jar3-aae56f76916dd0b8", "black peppercorn jar-0350a00775f69ec8", "cutlery tray-99f50b55d208bcf6", "spoon-23f9f68d6ce78389", "lid of black peppercorn jar-3435d15339f1c0fa", "some cardamom pods-e27758271fae927d", "spice jar4-9ac0bf6dc46e6e9c", "spice jar5-d691f402e8abb3e1", "spice jar 6-036b9138ef646ef1", "spice jar2-39906f235058e2df", "basket of spices-3484989d3f9fe435", "square thing-cc3937d3288c7f82", "unknown-c3ea67948e063c3f", "saucepan-6b9b99253a0b61ff", "saucepan2-504f3ab26b14b992", "saucepan's lid-44580101d11520f3", "saucepan3-134e94a24056636b", "saucepan's lid2-b41902901bf6f171", "new saucepan lid-98917c2e0603a074", "saucepan lid3-7edbaa6e0793bcef", "measuring jug-88db1439d0007727", "scale-b7d9f96064a4db58", "discarded food bag-59b06ad7f9160918", "napkin-9290099fe5f3abfe", "chunk of minced beef-03af9553115a3e68", "chunk of minced beef2-3b3d80a3984a6778", "chunk of minced beef3-c7f77d21787839b6", "roll of freezer bags-74f87c4fcdf12574", "freezer bag-5d2ee93d607dc65d", "chunk of minced beef4-67f9a475b5322cbf", "unknown pan-c5c854c7b96f3f55", "saucepan4-f8e6af174f8eb4ec", "white chopping board-1ccc6d2a79cbf260", "onion-02f77477ea73f067", "frying pan-1bc42a94f478b754", "kettle-10d40f103af00530", "large knife-13694cd71b173c51", "onion half-81df5fc5f4c0b1e0", "chunk of onion-5c7e44e5b09f8036", "bottle of oil-10eabb9489570f13", "beaker-b6a72cd8c007215c", "spoon2-704c9544d0891940", "cup of yogurt-dacfe1669a009a0d", "more beef chunks3-4a57fdb60490e20b", "cylinder-98f44bec0655509b", "lid of cylinder-0460af4b56499233", "spoon3-d868f2dc11d1ff55", "basket of vegetables-c7ff66dfa3c475ff", "chilies1-47dac01a35eb0bf4", "chilies2-c1bc4f5a7809c072", "chilies3-904d4642c6d5ef23", "chilies4-d397bcf81955676c", "chilies5-f943964fa952abb5", "chilies6-c99d47b7832653d5", "dishwashing soap bottle-f2a6ea13b967db73", "onion discards-92baec7b71ff0e68", "mango-3465b5c1c76c08cb", "spoon4-c6834aeeb9aa09ba", "spoon5-3b9d0e02641f5a60", "spoon6-07fff6fd13685278", "measuring cup-9e30c441522f457b", "salt bottle-8ea80fb62146aeb7", "container of ghee-e0f51ba4c6e11825", "tin of coconut milk-11c18e6cab2eea47", "jar of honey-601698a7a3974747", "tin opener-ecf647a59ea52388", "yogurt cup's lid-5c6fc7bf723998c8", "knife2-61b5d4aabcfc0487", "knife3-08689fcfc427dd6d", "mango half-5abf142d04650ff3", "slice of mango-bfb8b077b31742e0", "piece of mango skin-2570b5854d0d884c", "some mango skin-d31f98bf1c22a9a2", "green chopping board-baeaab8d8a1fda7f", "mango half2-4a7d9e9059a95014", "mango skin3-2c23c881f90b2f91", "mango skin4-1b772c0349a051f9", "slice of mango2-a1033f9793f846fc", "slice of mango3-684b425b8c6b961a", "slice of mango4-b7bb1ea6cfdabaa5", "slice of mango5-e30cb431d9b99c89", "slice of mango6-80e3a4286b99f8c8", "spice bottle lid-c3b07b6092751f18", "lid of tin of ghee-39c8e10fb9cfd1f8", "lid of jar of honey-7655edc34bdd4b5b", "bottle of cold water-73a965086c608ef5", "ice tray-72f2ae2c39573d8b", "ice tray's lid-18dffed19e5ec1d2", "blender's lid cover-24b79e6b19a9e7b5", "box of roll of cling film-b2e0f0d79762a311", "roll of cling film-91152383b6ba9bd6", "some cling film-fa70d51acb22f70d", "sponge-a6622eeb55f4279a", "some cling film2-0ad0a05f9f39c8fe", "spice jar unknown-94669f4c633d4c41", "mango bits-2eaf9d32478aa5f1", "mango bits2-399a236d3b6fa4fd", "mango bits3-9676de887a501e40", "mango bits4-df124e6244d30d27", "mango bits5-f354b376120ceaa7", "bowl3-15779f54e0f55b31", "tupperware-6d16462da19a13d7", "tumbler-d025677181f57aec", "lunchbox-28afac02a8f495b1", "fork-7159a8a251a8acab", "tumbler2-a41fd448fd7e87c4", "cable tie-f39490b6dbcbd0aa", "foil-83d861b69e06102c", "obj2-10d1a1392a7fbb45", "packet of mold-88f11f7b514e8f7a", "roll of foodwaste bag-e041ee160b0dc821", "bin bag-eb6175eaac49d67b", "mango skinx-6c861f5630621c3f", "keep cup-d8746624eef5b59d", "spoon-unknown-21fef58f55c972d6", "stack of keep cups-6e4f59c0ff648504", "keep cup cover-c5e8d4c663219532", "keep cup2-677ff477d4d26c13", "tupperware lid-43adb08620439946", "whisk-e333f78f343bc1ed" ], "reasoning": "1. I locate the digital timer (timer-d59705f7e130d64f) at 00:02 in the user's hands.\n2. At 00:18, I observe the user put the digital timer (timer-d59705f7e130d64f) under the pair of cabinets above and to the right of the stove .\n3. At 00:36, I observe the user grab the blender's lid (blender's lid-ead6db223b9bf936) for the first time.\n4. At 00:36, I determine the pair of cabinets is above and to the left of the user.", "referenced_object_masks": [ "timer-d59705f7e130d64f", "blender's lid-ead6db223b9bf936" ] }, { "key": "P09-20240623-120359:347f2cf19fae41ac0b5ed1bf9ff215ec90770571", "video_id": "P09-20240623-120359", "question": "Where would I have to look to see the basket of spices when I reach for the dishwashing soap bottle?", "answer_choice_0": "To the left", "answer_choice_1": "Up and to the right", "answer_choice_2": "Down and to the right", "answer_choice_3": "Backwards and up", "answer_choice_4": "To the right", "answer_id": 3, "answer": "Backwards and up", "question_type": [ "Object Recognition", "Action Recognition", "Temporal Reasoning", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "timer-d59705f7e130d64f", "bag-a8f088817e83b390", "knife-7ae1058802a2d588", "meat box-d91f02b6bc817f2e", "blender's lid-ead6db223b9bf936", "measuring jar-7b545b4808a5e2f8", "gray bowl-6283254f7c3bc941", "bowl2-9d8c45fa1153c18d", "blender-d7460356ec5e6cc6", "blender motor-7109bf56cf56e1f4", "spice jar-1210f4f4f01203dc", "new spice jar-fc7dc1d8c0698064", "spice jar3-aae56f76916dd0b8", "black peppercorn jar-0350a00775f69ec8", "cutlery tray-99f50b55d208bcf6", "spoon-23f9f68d6ce78389", "lid of black peppercorn jar-3435d15339f1c0fa", "some cardamom pods-e27758271fae927d", "spice jar4-9ac0bf6dc46e6e9c", "spice jar5-d691f402e8abb3e1", "spice jar 6-036b9138ef646ef1", "spice jar2-39906f235058e2df", "basket of spices-3484989d3f9fe435", "square thing-cc3937d3288c7f82", "unknown-c3ea67948e063c3f", "saucepan-6b9b99253a0b61ff", "saucepan2-504f3ab26b14b992", "saucepan's lid-44580101d11520f3", "saucepan3-134e94a24056636b", "saucepan's lid2-b41902901bf6f171", "new saucepan lid-98917c2e0603a074", "saucepan lid3-7edbaa6e0793bcef", "measuring jug-88db1439d0007727", "scale-b7d9f96064a4db58", "discarded food bag-59b06ad7f9160918", "napkin-9290099fe5f3abfe", "chunk of minced beef-03af9553115a3e68", "chunk of minced beef2-3b3d80a3984a6778", "chunk of minced beef3-c7f77d21787839b6", "roll of freezer bags-74f87c4fcdf12574", "freezer bag-5d2ee93d607dc65d", "chunk of minced beef4-67f9a475b5322cbf", "unknown pan-c5c854c7b96f3f55", "saucepan4-f8e6af174f8eb4ec", "white chopping board-1ccc6d2a79cbf260", "onion-02f77477ea73f067", "frying pan-1bc42a94f478b754", "kettle-10d40f103af00530", "large knife-13694cd71b173c51", "onion half-81df5fc5f4c0b1e0", "chunk of onion-5c7e44e5b09f8036", "bottle of oil-10eabb9489570f13", "beaker-b6a72cd8c007215c", "spoon2-704c9544d0891940", "cup of yogurt-dacfe1669a009a0d", "more beef chunks3-4a57fdb60490e20b", "cylinder-98f44bec0655509b", "lid of cylinder-0460af4b56499233", "spoon3-d868f2dc11d1ff55", "basket of vegetables-c7ff66dfa3c475ff", "chilies1-47dac01a35eb0bf4", "chilies2-c1bc4f5a7809c072", "chilies3-904d4642c6d5ef23", "chilies4-d397bcf81955676c", "chilies5-f943964fa952abb5", "chilies6-c99d47b7832653d5", "dishwashing soap bottle-f2a6ea13b967db73", "onion discards-92baec7b71ff0e68", "mango-3465b5c1c76c08cb", "spoon4-c6834aeeb9aa09ba", "spoon5-3b9d0e02641f5a60", "spoon6-07fff6fd13685278", "measuring cup-9e30c441522f457b", "salt bottle-8ea80fb62146aeb7", "container of ghee-e0f51ba4c6e11825", "tin of coconut milk-11c18e6cab2eea47", "jar of honey-601698a7a3974747", "tin opener-ecf647a59ea52388", "yogurt cup's lid-5c6fc7bf723998c8", "knife2-61b5d4aabcfc0487", "knife3-08689fcfc427dd6d", "mango half-5abf142d04650ff3", "slice of mango-bfb8b077b31742e0", "piece of mango skin-2570b5854d0d884c", "some mango skin-d31f98bf1c22a9a2", "green chopping board-baeaab8d8a1fda7f", "mango half2-4a7d9e9059a95014", "mango skin3-2c23c881f90b2f91", "mango skin4-1b772c0349a051f9", "slice of mango2-a1033f9793f846fc", "slice of mango3-684b425b8c6b961a", "slice of mango4-b7bb1ea6cfdabaa5", "slice of mango5-e30cb431d9b99c89", "slice of mango6-80e3a4286b99f8c8", "spice bottle lid-c3b07b6092751f18", "lid of tin of ghee-39c8e10fb9cfd1f8", "lid of jar of honey-7655edc34bdd4b5b", "bottle of cold water-73a965086c608ef5", "ice tray-72f2ae2c39573d8b", "ice tray's lid-18dffed19e5ec1d2", "blender's lid cover-24b79e6b19a9e7b5", "box of roll of cling film-b2e0f0d79762a311", "roll of cling film-91152383b6ba9bd6", "some cling film-fa70d51acb22f70d", "sponge-a6622eeb55f4279a", "some cling film2-0ad0a05f9f39c8fe", "spice jar unknown-94669f4c633d4c41", "mango bits-2eaf9d32478aa5f1", "mango bits2-399a236d3b6fa4fd", "mango bits3-9676de887a501e40", "mango bits4-df124e6244d30d27", "mango bits5-f354b376120ceaa7", "bowl3-15779f54e0f55b31", "tupperware-6d16462da19a13d7", "tumbler-d025677181f57aec", "lunchbox-28afac02a8f495b1", "fork-7159a8a251a8acab", "tumbler2-a41fd448fd7e87c4", "cable tie-f39490b6dbcbd0aa", "foil-83d861b69e06102c", "obj2-10d1a1392a7fbb45", "packet of mold-88f11f7b514e8f7a", "roll of foodwaste bag-e041ee160b0dc821", "bin bag-eb6175eaac49d67b", "mango skinx-6c861f5630621c3f", "keep cup-d8746624eef5b59d", "spoon-unknown-21fef58f55c972d6", "stack of keep cups-6e4f59c0ff648504", "keep cup cover-c5e8d4c663219532", "keep cup2-677ff477d4d26c13", "tupperware lid-43adb08620439946", "whisk-e333f78f343bc1ed" ], "reasoning": "1. I identify the sink across the room from the user to the left at 00:00.\n2. I identify the fridge to the right of the user at 00:03.\n3. I observe the user reach above the top row of cabinets to grab the basket of spices (basket of spices-3484989d3f9fe435) at 02:54 to the left of the fridge .\n4. I observe the user place the basket of spices (basket of spices-3484989d3f9fe435) back above the top row of cabinets at 03:19.\n5. I observe the user reach for the dishwasher soap (dishwashing soap bottle-f2a6ea13b967db73) at 16:44 while at the sink .\n6. I determine that the user would have to look backwards and up to see the basket of spices (basket of spices-3484989d3f9fe435).", "referenced_object_masks": [ "basket of spices-3484989d3f9fe435", "dishwashing soap bottle-f2a6ea13b967db73" ] }, { "key": "P09-20240623-120359:3bb4e0555175c479d5f6849963157242b8e6bab3", "video_id": "P09-20240623-120359", "question": "When I am standing at the stove, at which clock positions are my fridge and dishwasher at?", "answer_choice_0": "4 o'clock and 8 o'clock", "answer_choice_1": "10 o'clock and 9 o'clock", "answer_choice_2": "12 o'clock and 2 o'clock", "answer_choice_3": "1 o'clock and 1 o'clock", "answer_choice_4": "3 o'clock and 6 o'clock", "answer_id": 0, "answer": "4 o'clock and 8 o'clock", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "timer-d59705f7e130d64f", "bag-a8f088817e83b390", "knife-7ae1058802a2d588", "meat box-d91f02b6bc817f2e", "blender's lid-ead6db223b9bf936", "measuring jar-7b545b4808a5e2f8", "gray bowl-6283254f7c3bc941", "bowl2-9d8c45fa1153c18d", "blender-d7460356ec5e6cc6", "blender motor-7109bf56cf56e1f4", "spice jar-1210f4f4f01203dc", "new spice jar-fc7dc1d8c0698064", "spice jar3-aae56f76916dd0b8", "black peppercorn jar-0350a00775f69ec8", "cutlery tray-99f50b55d208bcf6", "spoon-23f9f68d6ce78389", "lid of black peppercorn jar-3435d15339f1c0fa", "some cardamom pods-e27758271fae927d", "spice jar4-9ac0bf6dc46e6e9c", "spice jar5-d691f402e8abb3e1", "spice jar 6-036b9138ef646ef1", "spice jar2-39906f235058e2df", "basket of spices-3484989d3f9fe435", "square thing-cc3937d3288c7f82", "unknown-c3ea67948e063c3f", "saucepan-6b9b99253a0b61ff", "saucepan2-504f3ab26b14b992", "saucepan's lid-44580101d11520f3", "saucepan3-134e94a24056636b", "saucepan's lid2-b41902901bf6f171", "new saucepan lid-98917c2e0603a074", "saucepan lid3-7edbaa6e0793bcef", "measuring jug-88db1439d0007727", "scale-b7d9f96064a4db58", "discarded food bag-59b06ad7f9160918", "napkin-9290099fe5f3abfe", "chunk of minced beef-03af9553115a3e68", "chunk of minced beef2-3b3d80a3984a6778", "chunk of minced beef3-c7f77d21787839b6", "roll of freezer bags-74f87c4fcdf12574", "freezer bag-5d2ee93d607dc65d", "chunk of minced beef4-67f9a475b5322cbf", "unknown pan-c5c854c7b96f3f55", "saucepan4-f8e6af174f8eb4ec", "white chopping board-1ccc6d2a79cbf260", "onion-02f77477ea73f067", "frying pan-1bc42a94f478b754", "kettle-10d40f103af00530", "large knife-13694cd71b173c51", "onion half-81df5fc5f4c0b1e0", "chunk of onion-5c7e44e5b09f8036", "bottle of oil-10eabb9489570f13", "beaker-b6a72cd8c007215c", "spoon2-704c9544d0891940", "cup of yogurt-dacfe1669a009a0d", "more beef chunks3-4a57fdb60490e20b", "cylinder-98f44bec0655509b", "lid of cylinder-0460af4b56499233", "spoon3-d868f2dc11d1ff55", "basket of vegetables-c7ff66dfa3c475ff", "chilies1-47dac01a35eb0bf4", "chilies2-c1bc4f5a7809c072", "chilies3-904d4642c6d5ef23", "chilies4-d397bcf81955676c", "chilies5-f943964fa952abb5", "chilies6-c99d47b7832653d5", "dishwashing soap bottle-f2a6ea13b967db73", "onion discards-92baec7b71ff0e68", "mango-3465b5c1c76c08cb", "spoon4-c6834aeeb9aa09ba", "spoon5-3b9d0e02641f5a60", "spoon6-07fff6fd13685278", "measuring cup-9e30c441522f457b", "salt bottle-8ea80fb62146aeb7", "container of ghee-e0f51ba4c6e11825", "tin of coconut milk-11c18e6cab2eea47", "jar of honey-601698a7a3974747", "tin opener-ecf647a59ea52388", "yogurt cup's lid-5c6fc7bf723998c8", "knife2-61b5d4aabcfc0487", "knife3-08689fcfc427dd6d", "mango half-5abf142d04650ff3", "slice of mango-bfb8b077b31742e0", "piece of mango skin-2570b5854d0d884c", "some mango skin-d31f98bf1c22a9a2", "green chopping board-baeaab8d8a1fda7f", "mango half2-4a7d9e9059a95014", "mango skin3-2c23c881f90b2f91", "mango skin4-1b772c0349a051f9", "slice of mango2-a1033f9793f846fc", "slice of mango3-684b425b8c6b961a", "slice of mango4-b7bb1ea6cfdabaa5", "slice of mango5-e30cb431d9b99c89", "slice of mango6-80e3a4286b99f8c8", "spice bottle lid-c3b07b6092751f18", "lid of tin of ghee-39c8e10fb9cfd1f8", "lid of jar of honey-7655edc34bdd4b5b", "bottle of cold water-73a965086c608ef5", "ice tray-72f2ae2c39573d8b", "ice tray's lid-18dffed19e5ec1d2", "blender's lid cover-24b79e6b19a9e7b5", "box of roll of cling film-b2e0f0d79762a311", "roll of cling film-91152383b6ba9bd6", "some cling film-fa70d51acb22f70d", "sponge-a6622eeb55f4279a", "some cling film2-0ad0a05f9f39c8fe", "spice jar unknown-94669f4c633d4c41", "mango bits-2eaf9d32478aa5f1", "mango bits2-399a236d3b6fa4fd", "mango bits3-9676de887a501e40", "mango bits4-df124e6244d30d27", "mango bits5-f354b376120ceaa7", "bowl3-15779f54e0f55b31", "tupperware-6d16462da19a13d7", "tumbler-d025677181f57aec", "lunchbox-28afac02a8f495b1", "fork-7159a8a251a8acab", "tumbler2-a41fd448fd7e87c4", "cable tie-f39490b6dbcbd0aa", "foil-83d861b69e06102c", "obj2-10d1a1392a7fbb45", "packet of mold-88f11f7b514e8f7a", "roll of foodwaste bag-e041ee160b0dc821", "bin bag-eb6175eaac49d67b", "mango skinx-6c861f5630621c3f", "keep cup-d8746624eef5b59d", "spoon-unknown-21fef58f55c972d6", "stack of keep cups-6e4f59c0ff648504", "keep cup cover-c5e8d4c663219532", "keep cup2-677ff477d4d26c13", "tupperware lid-43adb08620439946", "whisk-e333f78f343bc1ed" ], "reasoning": "1. I watched the video to find the user's position when they were directly facing the stove and saw this position at 12:00.\n2. I then watched the video to observe the positioning of the stove relative to the fridge and saw this position between 05:05-05:06, when the user turns to his right from the stove .\n3. From the user's position at the stove , the fridge is at a diagonal, and this diagonal goes slightly past what is directly on the right of the user, which means that this position can be described as 4 o'clock.\n4. I then watched the video to observe the positioning of the stove relative to the dishwasher and I was able to see this position at the beginning of the video at 00:00.\n5. The dishwasher appears to be parallel to the fridge, which means that it is on a diagonal, and this diagonal goes slightly past what is directly on the left of the user, which means that this position can be described at 8 o'clock.\n6. When the user is standing at the stove , the fridge and dishwasher are at 4 o'clock and 8 o'clock, respectively.", "referenced_object_masks": [] }, { "key": "P09-20240623-120359:40f36748aeb08e19232eb8cbb2ce40fe25ed5614", "video_id": "P09-20240623-120359", "question": "If I was facing the stove, then turned around 180 degrees and walked forward, what would I bump into first?", "answer_choice_0": "The kitchen table", "answer_choice_1": "The fridge", "answer_choice_2": "The kitchen sink", "answer_choice_3": "The dishwasher", "answer_choice_4": "The kitchen door", "answer_id": 0, "answer": "The kitchen table", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Action Recognition", "Situational Awareness" ], "objects_of_interest": [ "timer-d59705f7e130d64f", "bag-a8f088817e83b390", "knife-7ae1058802a2d588", "meat box-d91f02b6bc817f2e", "blender's lid-ead6db223b9bf936", "measuring jar-7b545b4808a5e2f8", "gray bowl-6283254f7c3bc941", "bowl2-9d8c45fa1153c18d", "blender-d7460356ec5e6cc6", "blender motor-7109bf56cf56e1f4", "spice jar-1210f4f4f01203dc", "new spice jar-fc7dc1d8c0698064", "spice jar3-aae56f76916dd0b8", "black peppercorn jar-0350a00775f69ec8", "cutlery tray-99f50b55d208bcf6", "spoon-23f9f68d6ce78389", "lid of black peppercorn jar-3435d15339f1c0fa", "some cardamom pods-e27758271fae927d", "spice jar4-9ac0bf6dc46e6e9c", "spice jar5-d691f402e8abb3e1", "spice jar 6-036b9138ef646ef1", "spice jar2-39906f235058e2df", "basket of spices-3484989d3f9fe435", "square thing-cc3937d3288c7f82", "unknown-c3ea67948e063c3f", "saucepan-6b9b99253a0b61ff", "saucepan2-504f3ab26b14b992", "saucepan's lid-44580101d11520f3", "saucepan3-134e94a24056636b", "saucepan's lid2-b41902901bf6f171", "new saucepan lid-98917c2e0603a074", "saucepan lid3-7edbaa6e0793bcef", "measuring jug-88db1439d0007727", "scale-b7d9f96064a4db58", "discarded food bag-59b06ad7f9160918", "napkin-9290099fe5f3abfe", "chunk of minced beef-03af9553115a3e68", "chunk of minced beef2-3b3d80a3984a6778", "chunk of minced beef3-c7f77d21787839b6", "roll of freezer bags-74f87c4fcdf12574", "freezer bag-5d2ee93d607dc65d", "chunk of minced beef4-67f9a475b5322cbf", "unknown pan-c5c854c7b96f3f55", "saucepan4-f8e6af174f8eb4ec", "white chopping board-1ccc6d2a79cbf260", "onion-02f77477ea73f067", "frying pan-1bc42a94f478b754", "kettle-10d40f103af00530", "large knife-13694cd71b173c51", "onion half-81df5fc5f4c0b1e0", "chunk of onion-5c7e44e5b09f8036", "bottle of oil-10eabb9489570f13", "beaker-b6a72cd8c007215c", "spoon2-704c9544d0891940", "cup of yogurt-dacfe1669a009a0d", "more beef chunks3-4a57fdb60490e20b", "cylinder-98f44bec0655509b", "lid of cylinder-0460af4b56499233", "spoon3-d868f2dc11d1ff55", "basket of vegetables-c7ff66dfa3c475ff", "chilies1-47dac01a35eb0bf4", "chilies2-c1bc4f5a7809c072", "chilies3-904d4642c6d5ef23", "chilies4-d397bcf81955676c", "chilies5-f943964fa952abb5", "chilies6-c99d47b7832653d5", "dishwashing soap bottle-f2a6ea13b967db73", "onion discards-92baec7b71ff0e68", "mango-3465b5c1c76c08cb", "spoon4-c6834aeeb9aa09ba", "spoon5-3b9d0e02641f5a60", "spoon6-07fff6fd13685278", "measuring cup-9e30c441522f457b", "salt bottle-8ea80fb62146aeb7", "container of ghee-e0f51ba4c6e11825", "tin of coconut milk-11c18e6cab2eea47", "jar of honey-601698a7a3974747", "tin opener-ecf647a59ea52388", "yogurt cup's lid-5c6fc7bf723998c8", "knife2-61b5d4aabcfc0487", "knife3-08689fcfc427dd6d", "mango half-5abf142d04650ff3", "slice of mango-bfb8b077b31742e0", "piece of mango skin-2570b5854d0d884c", "some mango skin-d31f98bf1c22a9a2", "green chopping board-baeaab8d8a1fda7f", "mango half2-4a7d9e9059a95014", "mango skin3-2c23c881f90b2f91", "mango skin4-1b772c0349a051f9", "slice of mango2-a1033f9793f846fc", "slice of mango3-684b425b8c6b961a", "slice of mango4-b7bb1ea6cfdabaa5", "slice of mango5-e30cb431d9b99c89", "slice of mango6-80e3a4286b99f8c8", "spice bottle lid-c3b07b6092751f18", "lid of tin of ghee-39c8e10fb9cfd1f8", "lid of jar of honey-7655edc34bdd4b5b", "bottle of cold water-73a965086c608ef5", "ice tray-72f2ae2c39573d8b", "ice tray's lid-18dffed19e5ec1d2", "blender's lid cover-24b79e6b19a9e7b5", "box of roll of cling film-b2e0f0d79762a311", "roll of cling film-91152383b6ba9bd6", "some cling film-fa70d51acb22f70d", "sponge-a6622eeb55f4279a", "some cling film2-0ad0a05f9f39c8fe", "spice jar unknown-94669f4c633d4c41", "mango bits-2eaf9d32478aa5f1", "mango bits2-399a236d3b6fa4fd", "mango bits3-9676de887a501e40", "mango bits4-df124e6244d30d27", "mango bits5-f354b376120ceaa7", "bowl3-15779f54e0f55b31", "tupperware-6d16462da19a13d7", "tumbler-d025677181f57aec", "lunchbox-28afac02a8f495b1", "fork-7159a8a251a8acab", "tumbler2-a41fd448fd7e87c4", "cable tie-f39490b6dbcbd0aa", "foil-83d861b69e06102c", "obj2-10d1a1392a7fbb45", "packet of mold-88f11f7b514e8f7a", "roll of foodwaste bag-e041ee160b0dc821", "bin bag-eb6175eaac49d67b", "mango skinx-6c861f5630621c3f", "keep cup-d8746624eef5b59d", "spoon-unknown-21fef58f55c972d6", "stack of keep cups-6e4f59c0ff648504", "keep cup cover-c5e8d4c663219532", "keep cup2-677ff477d4d26c13", "tupperware lid-43adb08620439946", "whisk-e333f78f343bc1ed" ], "reasoning": "1. I watched the video to find the user's position when they were directly facing the stove and saw this position at 12:00.\n2. I then watched the video to observe what was directly behind the user, and saw the user make a 180 degree turn to the left at 45:12.\n3. If the user walked forward at 45:12, they would bump into their kitchen table.", "referenced_object_masks": [] }, { "key": "P09-20240623-120359:473a698694d703d4510144f3c826707f37052a49", "video_id": "P09-20240623-120359", "question": "After reaching for the bottle of oil for the first time, where is the original location of the blender in relation to where I am standing?", "answer_choice_0": "Immediately to the left", "answer_choice_1": "By my legs", "answer_choice_2": "Immediately to the right", "answer_choice_3": "Behind", "answer_choice_4": "Above", "answer_id": 1, "answer": "By my legs", "question_type": [ "Object Recognition", "Action Recognition", "Temporal Reasoning", "Egocentric & Spatial Perception", "State Changes" ], "objects_of_interest": [ "timer-d59705f7e130d64f", "bag-a8f088817e83b390", "knife-7ae1058802a2d588", "meat box-d91f02b6bc817f2e", "blender's lid-ead6db223b9bf936", "measuring jar-7b545b4808a5e2f8", "gray bowl-6283254f7c3bc941", "bowl2-9d8c45fa1153c18d", "blender-d7460356ec5e6cc6", "blender motor-7109bf56cf56e1f4", "spice jar-1210f4f4f01203dc", "new spice jar-fc7dc1d8c0698064", "spice jar3-aae56f76916dd0b8", "black peppercorn jar-0350a00775f69ec8", "cutlery tray-99f50b55d208bcf6", "spoon-23f9f68d6ce78389", "lid of black peppercorn jar-3435d15339f1c0fa", "some cardamom pods-e27758271fae927d", "spice jar4-9ac0bf6dc46e6e9c", "spice jar5-d691f402e8abb3e1", "spice jar 6-036b9138ef646ef1", "spice jar2-39906f235058e2df", "basket of spices-3484989d3f9fe435", "square thing-cc3937d3288c7f82", "unknown-c3ea67948e063c3f", "saucepan-6b9b99253a0b61ff", "saucepan2-504f3ab26b14b992", "saucepan's lid-44580101d11520f3", "saucepan3-134e94a24056636b", "saucepan's lid2-b41902901bf6f171", "new saucepan lid-98917c2e0603a074", "saucepan lid3-7edbaa6e0793bcef", "measuring jug-88db1439d0007727", "scale-b7d9f96064a4db58", "discarded food bag-59b06ad7f9160918", "napkin-9290099fe5f3abfe", "chunk of minced beef-03af9553115a3e68", "chunk of minced beef2-3b3d80a3984a6778", "chunk of minced beef3-c7f77d21787839b6", "roll of freezer bags-74f87c4fcdf12574", "freezer bag-5d2ee93d607dc65d", "chunk of minced beef4-67f9a475b5322cbf", "unknown pan-c5c854c7b96f3f55", "saucepan4-f8e6af174f8eb4ec", "white chopping board-1ccc6d2a79cbf260", "onion-02f77477ea73f067", "frying pan-1bc42a94f478b754", "kettle-10d40f103af00530", "large knife-13694cd71b173c51", "onion half-81df5fc5f4c0b1e0", "chunk of onion-5c7e44e5b09f8036", "bottle of oil-10eabb9489570f13", "beaker-b6a72cd8c007215c", "spoon2-704c9544d0891940", "cup of yogurt-dacfe1669a009a0d", "more beef chunks3-4a57fdb60490e20b", "cylinder-98f44bec0655509b", "lid of cylinder-0460af4b56499233", "spoon3-d868f2dc11d1ff55", "basket of vegetables-c7ff66dfa3c475ff", "chilies1-47dac01a35eb0bf4", "chilies2-c1bc4f5a7809c072", "chilies3-904d4642c6d5ef23", "chilies4-d397bcf81955676c", "chilies5-f943964fa952abb5", "chilies6-c99d47b7832653d5", "dishwashing soap bottle-f2a6ea13b967db73", "onion discards-92baec7b71ff0e68", "mango-3465b5c1c76c08cb", "spoon4-c6834aeeb9aa09ba", "spoon5-3b9d0e02641f5a60", "spoon6-07fff6fd13685278", "measuring cup-9e30c441522f457b", "salt bottle-8ea80fb62146aeb7", "container of ghee-e0f51ba4c6e11825", "tin of coconut milk-11c18e6cab2eea47", "jar of honey-601698a7a3974747", "tin opener-ecf647a59ea52388", "yogurt cup's lid-5c6fc7bf723998c8", "knife2-61b5d4aabcfc0487", "knife3-08689fcfc427dd6d", "mango half-5abf142d04650ff3", "slice of mango-bfb8b077b31742e0", "piece of mango skin-2570b5854d0d884c", "some mango skin-d31f98bf1c22a9a2", "green chopping board-baeaab8d8a1fda7f", "mango half2-4a7d9e9059a95014", "mango skin3-2c23c881f90b2f91", "mango skin4-1b772c0349a051f9", "slice of mango2-a1033f9793f846fc", "slice of mango3-684b425b8c6b961a", "slice of mango4-b7bb1ea6cfdabaa5", "slice of mango5-e30cb431d9b99c89", "slice of mango6-80e3a4286b99f8c8", "spice bottle lid-c3b07b6092751f18", "lid of tin of ghee-39c8e10fb9cfd1f8", "lid of jar of honey-7655edc34bdd4b5b", "bottle of cold water-73a965086c608ef5", "ice tray-72f2ae2c39573d8b", "ice tray's lid-18dffed19e5ec1d2", "blender's lid cover-24b79e6b19a9e7b5", "box of roll of cling film-b2e0f0d79762a311", "roll of cling film-91152383b6ba9bd6", "some cling film-fa70d51acb22f70d", "sponge-a6622eeb55f4279a", "some cling film2-0ad0a05f9f39c8fe", "spice jar unknown-94669f4c633d4c41", "mango bits-2eaf9d32478aa5f1", "mango bits2-399a236d3b6fa4fd", "mango bits3-9676de887a501e40", "mango bits4-df124e6244d30d27", "mango bits5-f354b376120ceaa7", "bowl3-15779f54e0f55b31", "tupperware-6d16462da19a13d7", "tumbler-d025677181f57aec", "lunchbox-28afac02a8f495b1", "fork-7159a8a251a8acab", "tumbler2-a41fd448fd7e87c4", "cable tie-f39490b6dbcbd0aa", "foil-83d861b69e06102c", "obj2-10d1a1392a7fbb45", "packet of mold-88f11f7b514e8f7a", "roll of foodwaste bag-e041ee160b0dc821", "bin bag-eb6175eaac49d67b", "mango skinx-6c861f5630621c3f", "keep cup-d8746624eef5b59d", "spoon-unknown-21fef58f55c972d6", "stack of keep cups-6e4f59c0ff648504", "keep cup cover-c5e8d4c663219532", "keep cup2-677ff477d4d26c13", "tupperware lid-43adb08620439946", "whisk-e333f78f343bc1ed" ], "reasoning": "1. I observe the user reach for the oil (bottle of oil-10eabb9489570f13) at 09:20 on the left side of the cabinet in front of them.\n2. I rewind to 09:19 to get a better view of the location of the cabinet , which I determine is to the left of the fridge and above the counter .\n3. I identify the first location of the blender (blender-d7460356ec5e6cc6) under the counter to the left of the fridge at 00:47.\n4. I determine that the location of the blender (blender-d7460356ec5e6cc6) is by the user's legs.", "referenced_object_masks": [ "bottle of oil-10eabb9489570f13", "blender-d7460356ec5e6cc6" ] }, { "key": "P09-20240623-120359:804bb67791b5feac00467771b3bca2e3839ab0f1", "video_id": "P09-20240623-120359", "question": "Before putting the first piece of mango in the blender, how many cuts to the mango do I make?", "answer_choice_0": "1", "answer_choice_1": "2", "answer_choice_2": "3", "answer_choice_3": "4", "answer_choice_4": "5", "answer_id": 1, "answer": "2", "question_type": [ "Object Recognition", "Action Recognition", "Counting", "Temporal Reasoning" ], "objects_of_interest": [ "timer-d59705f7e130d64f", "bag-a8f088817e83b390", "knife-7ae1058802a2d588", "meat box-d91f02b6bc817f2e", "blender's lid-ead6db223b9bf936", "measuring jar-7b545b4808a5e2f8", "gray bowl-6283254f7c3bc941", "bowl2-9d8c45fa1153c18d", "blender-d7460356ec5e6cc6", "blender motor-7109bf56cf56e1f4", "spice jar-1210f4f4f01203dc", "new spice jar-fc7dc1d8c0698064", "spice jar3-aae56f76916dd0b8", "black peppercorn jar-0350a00775f69ec8", "cutlery tray-99f50b55d208bcf6", "spoon-23f9f68d6ce78389", "lid of black peppercorn jar-3435d15339f1c0fa", "some cardamom pods-e27758271fae927d", "spice jar4-9ac0bf6dc46e6e9c", "spice jar5-d691f402e8abb3e1", "spice jar 6-036b9138ef646ef1", "spice jar2-39906f235058e2df", "basket of spices-3484989d3f9fe435", "square thing-cc3937d3288c7f82", "unknown-c3ea67948e063c3f", "saucepan-6b9b99253a0b61ff", "saucepan2-504f3ab26b14b992", "saucepan's lid-44580101d11520f3", "saucepan3-134e94a24056636b", "saucepan's lid2-b41902901bf6f171", "new saucepan lid-98917c2e0603a074", "saucepan lid3-7edbaa6e0793bcef", "measuring jug-88db1439d0007727", "scale-b7d9f96064a4db58", "discarded food bag-59b06ad7f9160918", "napkin-9290099fe5f3abfe", "chunk of minced beef-03af9553115a3e68", "chunk of minced beef2-3b3d80a3984a6778", "chunk of minced beef3-c7f77d21787839b6", "roll of freezer bags-74f87c4fcdf12574", "freezer bag-5d2ee93d607dc65d", "chunk of minced beef4-67f9a475b5322cbf", "unknown pan-c5c854c7b96f3f55", "saucepan4-f8e6af174f8eb4ec", "white chopping board-1ccc6d2a79cbf260", "onion-02f77477ea73f067", "frying pan-1bc42a94f478b754", "kettle-10d40f103af00530", "large knife-13694cd71b173c51", "onion half-81df5fc5f4c0b1e0", "chunk of onion-5c7e44e5b09f8036", "bottle of oil-10eabb9489570f13", "beaker-b6a72cd8c007215c", "spoon2-704c9544d0891940", "cup of yogurt-dacfe1669a009a0d", "more beef chunks3-4a57fdb60490e20b", "cylinder-98f44bec0655509b", "lid of cylinder-0460af4b56499233", "spoon3-d868f2dc11d1ff55", "basket of vegetables-c7ff66dfa3c475ff", "chilies1-47dac01a35eb0bf4", "chilies2-c1bc4f5a7809c072", "chilies3-904d4642c6d5ef23", "chilies4-d397bcf81955676c", "chilies5-f943964fa952abb5", "chilies6-c99d47b7832653d5", "dishwashing soap bottle-f2a6ea13b967db73", "onion discards-92baec7b71ff0e68", "mango-3465b5c1c76c08cb", "spoon4-c6834aeeb9aa09ba", "spoon5-3b9d0e02641f5a60", "spoon6-07fff6fd13685278", "measuring cup-9e30c441522f457b", "salt bottle-8ea80fb62146aeb7", "container of ghee-e0f51ba4c6e11825", "tin of coconut milk-11c18e6cab2eea47", "jar of honey-601698a7a3974747", "tin opener-ecf647a59ea52388", "yogurt cup's lid-5c6fc7bf723998c8", "knife2-61b5d4aabcfc0487", "knife3-08689fcfc427dd6d", "mango half-5abf142d04650ff3", "slice of mango-bfb8b077b31742e0", "piece of mango skin-2570b5854d0d884c", "some mango skin-d31f98bf1c22a9a2", "green chopping board-baeaab8d8a1fda7f", "mango half2-4a7d9e9059a95014", "mango skin3-2c23c881f90b2f91", "mango skin4-1b772c0349a051f9", "slice of mango2-a1033f9793f846fc", "slice of mango3-684b425b8c6b961a", "slice of mango4-b7bb1ea6cfdabaa5", "slice of mango5-e30cb431d9b99c89", "slice of mango6-80e3a4286b99f8c8", "spice bottle lid-c3b07b6092751f18", "lid of tin of ghee-39c8e10fb9cfd1f8", "lid of jar of honey-7655edc34bdd4b5b", "bottle of cold water-73a965086c608ef5", "ice tray-72f2ae2c39573d8b", "ice tray's lid-18dffed19e5ec1d2", "blender's lid cover-24b79e6b19a9e7b5", "box of roll of cling film-b2e0f0d79762a311", "roll of cling film-91152383b6ba9bd6", "some cling film-fa70d51acb22f70d", "sponge-a6622eeb55f4279a", "some cling film2-0ad0a05f9f39c8fe", "spice jar unknown-94669f4c633d4c41", "mango bits-2eaf9d32478aa5f1", "mango bits2-399a236d3b6fa4fd", "mango bits3-9676de887a501e40", "mango bits4-df124e6244d30d27", "mango bits5-f354b376120ceaa7", "bowl3-15779f54e0f55b31", "tupperware-6d16462da19a13d7", "tumbler-d025677181f57aec", "lunchbox-28afac02a8f495b1", "fork-7159a8a251a8acab", "tumbler2-a41fd448fd7e87c4", "cable tie-f39490b6dbcbd0aa", "foil-83d861b69e06102c", "obj2-10d1a1392a7fbb45", "packet of mold-88f11f7b514e8f7a", "roll of foodwaste bag-e041ee160b0dc821", "bin bag-eb6175eaac49d67b", "mango skinx-6c861f5630621c3f", "keep cup-d8746624eef5b59d", "spoon-unknown-21fef58f55c972d6", "stack of keep cups-6e4f59c0ff648504", "keep cup cover-c5e8d4c663219532", "keep cup2-677ff477d4d26c13", "tupperware lid-43adb08620439946", "whisk-e333f78f343bc1ed" ], "reasoning": "1. I watched the whole video and the user started cutting up parts of the mango (mango-3465b5c1c76c08cb) at 24:16.\n2. The user makes 2 cuts into the mango before putting the first piece of mango (mango-3465b5c1c76c08cb) into the blender at 24:35.", "referenced_object_masks": [ "mango-3465b5c1c76c08cb" ] }, { "key": "P09-20240623-120359:862a746354213bd43a2af52c8ccb73d3f804f1ff", "video_id": "P09-20240623-120359", "question": "After removing the onion skin, how many times do I cut part of, or the whole onion before putting the knife back down again?", "answer_choice_0": "22", "answer_choice_1": "24", "answer_choice_2": "18", "answer_choice_3": "20", "answer_choice_4": "16", "answer_id": 0, "answer": "22", "question_type": [ "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counting" ], "objects_of_interest": [ "timer-d59705f7e130d64f", "bag-a8f088817e83b390", "knife-7ae1058802a2d588", "meat box-d91f02b6bc817f2e", "blender's lid-ead6db223b9bf936", "measuring jar-7b545b4808a5e2f8", "gray bowl-6283254f7c3bc941", "bowl2-9d8c45fa1153c18d", "blender-d7460356ec5e6cc6", "blender motor-7109bf56cf56e1f4", "spice jar-1210f4f4f01203dc", "new spice jar-fc7dc1d8c0698064", "spice jar3-aae56f76916dd0b8", "black peppercorn jar-0350a00775f69ec8", "cutlery tray-99f50b55d208bcf6", "spoon-23f9f68d6ce78389", "lid of black peppercorn jar-3435d15339f1c0fa", "some cardamom pods-e27758271fae927d", "spice jar4-9ac0bf6dc46e6e9c", "spice jar5-d691f402e8abb3e1", "spice jar 6-036b9138ef646ef1", "spice jar2-39906f235058e2df", "basket of spices-3484989d3f9fe435", "square thing-cc3937d3288c7f82", "unknown-c3ea67948e063c3f", "saucepan-6b9b99253a0b61ff", "saucepan2-504f3ab26b14b992", "saucepan's lid-44580101d11520f3", "saucepan3-134e94a24056636b", "saucepan's lid2-b41902901bf6f171", "new saucepan lid-98917c2e0603a074", "saucepan lid3-7edbaa6e0793bcef", "measuring jug-88db1439d0007727", "scale-b7d9f96064a4db58", "discarded food bag-59b06ad7f9160918", "napkin-9290099fe5f3abfe", "chunk of minced beef-03af9553115a3e68", "chunk of minced beef2-3b3d80a3984a6778", "chunk of minced beef3-c7f77d21787839b6", "roll of freezer bags-74f87c4fcdf12574", "freezer bag-5d2ee93d607dc65d", "chunk of minced beef4-67f9a475b5322cbf", "unknown pan-c5c854c7b96f3f55", "saucepan4-f8e6af174f8eb4ec", "white chopping board-1ccc6d2a79cbf260", "onion-02f77477ea73f067", "frying pan-1bc42a94f478b754", "kettle-10d40f103af00530", "large knife-13694cd71b173c51", "onion half-81df5fc5f4c0b1e0", "chunk of onion-5c7e44e5b09f8036", "bottle of oil-10eabb9489570f13", "beaker-b6a72cd8c007215c", "spoon2-704c9544d0891940", "cup of yogurt-dacfe1669a009a0d", "more beef chunks3-4a57fdb60490e20b", "cylinder-98f44bec0655509b", "lid of cylinder-0460af4b56499233", "spoon3-d868f2dc11d1ff55", "basket of vegetables-c7ff66dfa3c475ff", "chilies1-47dac01a35eb0bf4", "chilies2-c1bc4f5a7809c072", "chilies3-904d4642c6d5ef23", "chilies4-d397bcf81955676c", "chilies5-f943964fa952abb5", "chilies6-c99d47b7832653d5", "dishwashing soap bottle-f2a6ea13b967db73", "onion discards-92baec7b71ff0e68", "mango-3465b5c1c76c08cb", "spoon4-c6834aeeb9aa09ba", "spoon5-3b9d0e02641f5a60", "spoon6-07fff6fd13685278", "measuring cup-9e30c441522f457b", "salt bottle-8ea80fb62146aeb7", "container of ghee-e0f51ba4c6e11825", "tin of coconut milk-11c18e6cab2eea47", "jar of honey-601698a7a3974747", "tin opener-ecf647a59ea52388", "yogurt cup's lid-5c6fc7bf723998c8", "knife2-61b5d4aabcfc0487", "knife3-08689fcfc427dd6d", "mango half-5abf142d04650ff3", "slice of mango-bfb8b077b31742e0", "piece of mango skin-2570b5854d0d884c", "some mango skin-d31f98bf1c22a9a2", "green chopping board-baeaab8d8a1fda7f", "mango half2-4a7d9e9059a95014", "mango skin3-2c23c881f90b2f91", "mango skin4-1b772c0349a051f9", "slice of mango2-a1033f9793f846fc", "slice of mango3-684b425b8c6b961a", "slice of mango4-b7bb1ea6cfdabaa5", "slice of mango5-e30cb431d9b99c89", "slice of mango6-80e3a4286b99f8c8", "spice bottle lid-c3b07b6092751f18", "lid of tin of ghee-39c8e10fb9cfd1f8", "lid of jar of honey-7655edc34bdd4b5b", "bottle of cold water-73a965086c608ef5", "ice tray-72f2ae2c39573d8b", "ice tray's lid-18dffed19e5ec1d2", "blender's lid cover-24b79e6b19a9e7b5", "box of roll of cling film-b2e0f0d79762a311", "roll of cling film-91152383b6ba9bd6", "some cling film-fa70d51acb22f70d", "sponge-a6622eeb55f4279a", "some cling film2-0ad0a05f9f39c8fe", "spice jar unknown-94669f4c633d4c41", "mango bits-2eaf9d32478aa5f1", "mango bits2-399a236d3b6fa4fd", "mango bits3-9676de887a501e40", "mango bits4-df124e6244d30d27", "mango bits5-f354b376120ceaa7", "bowl3-15779f54e0f55b31", "tupperware-6d16462da19a13d7", "tumbler-d025677181f57aec", "lunchbox-28afac02a8f495b1", "fork-7159a8a251a8acab", "tumbler2-a41fd448fd7e87c4", "cable tie-f39490b6dbcbd0aa", "foil-83d861b69e06102c", "obj2-10d1a1392a7fbb45", "packet of mold-88f11f7b514e8f7a", "roll of foodwaste bag-e041ee160b0dc821", "bin bag-eb6175eaac49d67b", "mango skinx-6c861f5630621c3f", "keep cup-d8746624eef5b59d", "spoon-unknown-21fef58f55c972d6", "stack of keep cups-6e4f59c0ff648504", "keep cup cover-c5e8d4c663219532", "keep cup2-677ff477d4d26c13", "tupperware lid-43adb08620439946", "whisk-e333f78f343bc1ed" ], "reasoning": "1. I observe the user remove the skin from the onion (onion-02f77477ea73f067) from 06:29 to 06:45.\n2. I observe the user place the knife (knife-7ae1058802a2d588) down at 06:52.\n3. From 06:45 to 06:52, I observe the user make 22 cuts.", "referenced_object_masks": [ "knife-7ae1058802a2d588", "onion-02f77477ea73f067" ] }, { "key": "P09-20240623-120359:91e69cbc06b05ee4e7e906e7b67f444e88f861c9", "video_id": "P09-20240623-120359", "question": "How many different ingredients did I add to the first bowl I weighed?", "answer_choice_0": "10", "answer_choice_1": "7", "answer_choice_2": "3", "answer_choice_3": "6", "answer_choice_4": "2", "answer_id": 3, "answer": "6", "question_type": [ "Counting", "Object Recognition", "Action Recognition" ], "objects_of_interest": [ "timer-d59705f7e130d64f", "bag-a8f088817e83b390", "knife-7ae1058802a2d588", "meat box-d91f02b6bc817f2e", "blender's lid-ead6db223b9bf936", "measuring jar-7b545b4808a5e2f8", "gray bowl-6283254f7c3bc941", "bowl2-9d8c45fa1153c18d", "blender-d7460356ec5e6cc6", "blender motor-7109bf56cf56e1f4", "spice jar-1210f4f4f01203dc", "new spice jar-fc7dc1d8c0698064", "spice jar3-aae56f76916dd0b8", "black peppercorn jar-0350a00775f69ec8", "cutlery tray-99f50b55d208bcf6", "spoon-23f9f68d6ce78389", "lid of black peppercorn jar-3435d15339f1c0fa", "some cardamom pods-e27758271fae927d", "spice jar4-9ac0bf6dc46e6e9c", "spice jar5-d691f402e8abb3e1", "spice jar 6-036b9138ef646ef1", "spice jar2-39906f235058e2df", "basket of spices-3484989d3f9fe435", "square thing-cc3937d3288c7f82", "unknown-c3ea67948e063c3f", "saucepan-6b9b99253a0b61ff", "saucepan2-504f3ab26b14b992", "saucepan's lid-44580101d11520f3", "saucepan3-134e94a24056636b", "saucepan's lid2-b41902901bf6f171", "new saucepan lid-98917c2e0603a074", "saucepan lid3-7edbaa6e0793bcef", "measuring jug-88db1439d0007727", "scale-b7d9f96064a4db58", "discarded food bag-59b06ad7f9160918", "napkin-9290099fe5f3abfe", "chunk of minced beef-03af9553115a3e68", "chunk of minced beef2-3b3d80a3984a6778", "chunk of minced beef3-c7f77d21787839b6", "roll of freezer bags-74f87c4fcdf12574", "freezer bag-5d2ee93d607dc65d", "chunk of minced beef4-67f9a475b5322cbf", "unknown pan-c5c854c7b96f3f55", "saucepan4-f8e6af174f8eb4ec", "white chopping board-1ccc6d2a79cbf260", "onion-02f77477ea73f067", "frying pan-1bc42a94f478b754", "kettle-10d40f103af00530", "large knife-13694cd71b173c51", "onion half-81df5fc5f4c0b1e0", "chunk of onion-5c7e44e5b09f8036", "bottle of oil-10eabb9489570f13", "beaker-b6a72cd8c007215c", "spoon2-704c9544d0891940", "cup of yogurt-dacfe1669a009a0d", "more beef chunks3-4a57fdb60490e20b", "cylinder-98f44bec0655509b", "lid of cylinder-0460af4b56499233", "spoon3-d868f2dc11d1ff55", "basket of vegetables-c7ff66dfa3c475ff", "chilies1-47dac01a35eb0bf4", "chilies2-c1bc4f5a7809c072", "chilies3-904d4642c6d5ef23", "chilies4-d397bcf81955676c", "chilies5-f943964fa952abb5", "chilies6-c99d47b7832653d5", "dishwashing soap bottle-f2a6ea13b967db73", "onion discards-92baec7b71ff0e68", "mango-3465b5c1c76c08cb", "spoon4-c6834aeeb9aa09ba", "spoon5-3b9d0e02641f5a60", "spoon6-07fff6fd13685278", "measuring cup-9e30c441522f457b", "salt bottle-8ea80fb62146aeb7", "container of ghee-e0f51ba4c6e11825", "tin of coconut milk-11c18e6cab2eea47", "jar of honey-601698a7a3974747", "tin opener-ecf647a59ea52388", "yogurt cup's lid-5c6fc7bf723998c8", "knife2-61b5d4aabcfc0487", "knife3-08689fcfc427dd6d", "mango half-5abf142d04650ff3", "slice of mango-bfb8b077b31742e0", "piece of mango skin-2570b5854d0d884c", "some mango skin-d31f98bf1c22a9a2", "green chopping board-baeaab8d8a1fda7f", "mango half2-4a7d9e9059a95014", "mango skin3-2c23c881f90b2f91", "mango skin4-1b772c0349a051f9", "slice of mango2-a1033f9793f846fc", "slice of mango3-684b425b8c6b961a", "slice of mango4-b7bb1ea6cfdabaa5", "slice of mango5-e30cb431d9b99c89", "slice of mango6-80e3a4286b99f8c8", "spice bottle lid-c3b07b6092751f18", "lid of tin of ghee-39c8e10fb9cfd1f8", "lid of jar of honey-7655edc34bdd4b5b", "bottle of cold water-73a965086c608ef5", "ice tray-72f2ae2c39573d8b", "ice tray's lid-18dffed19e5ec1d2", "blender's lid cover-24b79e6b19a9e7b5", "box of roll of cling film-b2e0f0d79762a311", "roll of cling film-91152383b6ba9bd6", "some cling film-fa70d51acb22f70d", "sponge-a6622eeb55f4279a", "some cling film2-0ad0a05f9f39c8fe", "spice jar unknown-94669f4c633d4c41", "mango bits-2eaf9d32478aa5f1", "mango bits2-399a236d3b6fa4fd", "mango bits3-9676de887a501e40", "mango bits4-df124e6244d30d27", "mango bits5-f354b376120ceaa7", "bowl3-15779f54e0f55b31", "tupperware-6d16462da19a13d7", "tumbler-d025677181f57aec", "lunchbox-28afac02a8f495b1", "fork-7159a8a251a8acab", "tumbler2-a41fd448fd7e87c4", "cable tie-f39490b6dbcbd0aa", "foil-83d861b69e06102c", "obj2-10d1a1392a7fbb45", "packet of mold-88f11f7b514e8f7a", "roll of foodwaste bag-e041ee160b0dc821", "bin bag-eb6175eaac49d67b", "mango skinx-6c861f5630621c3f", "keep cup-d8746624eef5b59d", "spoon-unknown-21fef58f55c972d6", "stack of keep cups-6e4f59c0ff648504", "keep cup cover-c5e8d4c663219532", "keep cup2-677ff477d4d26c13", "tupperware lid-43adb08620439946", "whisk-e333f78f343bc1ed" ], "reasoning": "1. I watched the video to find the first bowl (bowl2-9d8c45fa1153c18d). The user found it and put it on the scale (scale-b7d9f96064a4db58) at 01:09.\n2. I observed the first ingredient (black peppercorn jar-0350a00775f69ec8) being put into the bowl (bowl2-9d8c45fa1153c18d) at 01:42.\n3. I then observed the second ingredient (some cardamom pods-e27758271fae927d) being put in the bowl (bowl2-9d8c45fa1153c18d) at 01:52.\n4. I then observed the third ingredient (spice jar4-9ac0bf6dc46e6e9c) being put in the bowl (bowl2-9d8c45fa1153c18d) at 02:05.\n5. I then observed the fourth ingredient (spice jar2-39906f235058e2df) being put in the bowl (bowl2-9d8c45fa1153c18d) at 02:16.\n6. I then observed the fifth ingredient (spice jar5-d691f402e8abb3e1) being put in the bowl (bowl2-9d8c45fa1153c18d) at 02:25.\n7. I then observed the sixth and final ingredient being put in the bowl (bowl2-9d8c45fa1153c18d) at 02:51. No other ingredients were added after this point.\n8. The total amount of ingredients added to the bowl (bowl2-9d8c45fa1153c18d) comes to 6.", "referenced_object_masks": [ "bowl2-9d8c45fa1153c18d", "spice jar2-39906f235058e2df", "some cardamom pods-e27758271fae927d", "spice jar5-d691f402e8abb3e1", "scale-b7d9f96064a4db58", "black peppercorn jar-0350a00775f69ec8", "spice jar4-9ac0bf6dc46e6e9c" ] }, { "key": "P09-20240623-120359:aeb7d8644da5617fb973c56c63899e71ba6acf33", "video_id": "P09-20240623-120359", "question": "When I am standing at the kitchen sink, what appliance is due south of me?", "answer_choice_0": "The dishwasher", "answer_choice_1": "The microwave", "answer_choice_2": "The kettle", "answer_choice_3": "The coffee maker", "answer_choice_4": "The stove", "answer_id": 1, "answer": "The microwave", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition" ], "objects_of_interest": [ "timer-d59705f7e130d64f", "bag-a8f088817e83b390", "knife-7ae1058802a2d588", "meat box-d91f02b6bc817f2e", "blender's lid-ead6db223b9bf936", "measuring jar-7b545b4808a5e2f8", "gray bowl-6283254f7c3bc941", "bowl2-9d8c45fa1153c18d", "blender-d7460356ec5e6cc6", "blender motor-7109bf56cf56e1f4", "spice jar-1210f4f4f01203dc", "new spice jar-fc7dc1d8c0698064", "spice jar3-aae56f76916dd0b8", "black peppercorn jar-0350a00775f69ec8", "cutlery tray-99f50b55d208bcf6", "spoon-23f9f68d6ce78389", "lid of black peppercorn jar-3435d15339f1c0fa", "some cardamom pods-e27758271fae927d", "spice jar4-9ac0bf6dc46e6e9c", "spice jar5-d691f402e8abb3e1", "spice jar 6-036b9138ef646ef1", "spice jar2-39906f235058e2df", "basket of spices-3484989d3f9fe435", "square thing-cc3937d3288c7f82", "unknown-c3ea67948e063c3f", "saucepan-6b9b99253a0b61ff", "saucepan2-504f3ab26b14b992", "saucepan's lid-44580101d11520f3", "saucepan3-134e94a24056636b", "saucepan's lid2-b41902901bf6f171", "new saucepan lid-98917c2e0603a074", "saucepan lid3-7edbaa6e0793bcef", "measuring jug-88db1439d0007727", "scale-b7d9f96064a4db58", "discarded food bag-59b06ad7f9160918", "napkin-9290099fe5f3abfe", "chunk of minced beef-03af9553115a3e68", "chunk of minced beef2-3b3d80a3984a6778", "chunk of minced beef3-c7f77d21787839b6", "roll of freezer bags-74f87c4fcdf12574", "freezer bag-5d2ee93d607dc65d", "chunk of minced beef4-67f9a475b5322cbf", "unknown pan-c5c854c7b96f3f55", "saucepan4-f8e6af174f8eb4ec", "white chopping board-1ccc6d2a79cbf260", "onion-02f77477ea73f067", "frying pan-1bc42a94f478b754", "kettle-10d40f103af00530", "large knife-13694cd71b173c51", "onion half-81df5fc5f4c0b1e0", "chunk of onion-5c7e44e5b09f8036", "bottle of oil-10eabb9489570f13", "beaker-b6a72cd8c007215c", "spoon2-704c9544d0891940", "cup of yogurt-dacfe1669a009a0d", "more beef chunks3-4a57fdb60490e20b", "cylinder-98f44bec0655509b", "lid of cylinder-0460af4b56499233", "spoon3-d868f2dc11d1ff55", "basket of vegetables-c7ff66dfa3c475ff", "chilies1-47dac01a35eb0bf4", "chilies2-c1bc4f5a7809c072", "chilies3-904d4642c6d5ef23", "chilies4-d397bcf81955676c", "chilies5-f943964fa952abb5", "chilies6-c99d47b7832653d5", "dishwashing soap bottle-f2a6ea13b967db73", "onion discards-92baec7b71ff0e68", "mango-3465b5c1c76c08cb", "spoon4-c6834aeeb9aa09ba", "spoon5-3b9d0e02641f5a60", "spoon6-07fff6fd13685278", "measuring cup-9e30c441522f457b", "salt bottle-8ea80fb62146aeb7", "container of ghee-e0f51ba4c6e11825", "tin of coconut milk-11c18e6cab2eea47", "jar of honey-601698a7a3974747", "tin opener-ecf647a59ea52388", "yogurt cup's lid-5c6fc7bf723998c8", "knife2-61b5d4aabcfc0487", "knife3-08689fcfc427dd6d", "mango half-5abf142d04650ff3", "slice of mango-bfb8b077b31742e0", "piece of mango skin-2570b5854d0d884c", "some mango skin-d31f98bf1c22a9a2", "green chopping board-baeaab8d8a1fda7f", "mango half2-4a7d9e9059a95014", "mango skin3-2c23c881f90b2f91", "mango skin4-1b772c0349a051f9", "slice of mango2-a1033f9793f846fc", "slice of mango3-684b425b8c6b961a", "slice of mango4-b7bb1ea6cfdabaa5", "slice of mango5-e30cb431d9b99c89", "slice of mango6-80e3a4286b99f8c8", "spice bottle lid-c3b07b6092751f18", "lid of tin of ghee-39c8e10fb9cfd1f8", "lid of jar of honey-7655edc34bdd4b5b", "bottle of cold water-73a965086c608ef5", "ice tray-72f2ae2c39573d8b", "ice tray's lid-18dffed19e5ec1d2", "blender's lid cover-24b79e6b19a9e7b5", "box of roll of cling film-b2e0f0d79762a311", "roll of cling film-91152383b6ba9bd6", "some cling film-fa70d51acb22f70d", "sponge-a6622eeb55f4279a", "some cling film2-0ad0a05f9f39c8fe", "spice jar unknown-94669f4c633d4c41", "mango bits-2eaf9d32478aa5f1", "mango bits2-399a236d3b6fa4fd", "mango bits3-9676de887a501e40", "mango bits4-df124e6244d30d27", "mango bits5-f354b376120ceaa7", "bowl3-15779f54e0f55b31", "tupperware-6d16462da19a13d7", "tumbler-d025677181f57aec", "lunchbox-28afac02a8f495b1", "fork-7159a8a251a8acab", "tumbler2-a41fd448fd7e87c4", "cable tie-f39490b6dbcbd0aa", "foil-83d861b69e06102c", "obj2-10d1a1392a7fbb45", "packet of mold-88f11f7b514e8f7a", "roll of foodwaste bag-e041ee160b0dc821", "bin bag-eb6175eaac49d67b", "mango skinx-6c861f5630621c3f", "keep cup-d8746624eef5b59d", "spoon-unknown-21fef58f55c972d6", "stack of keep cups-6e4f59c0ff648504", "keep cup cover-c5e8d4c663219532", "keep cup2-677ff477d4d26c13", "tupperware lid-43adb08620439946", "whisk-e333f78f343bc1ed" ], "reasoning": "1. I watched the video to find the user's position when they were directly facing the sink and saw this position at 05:20.\n2. I then observed the user making a 180 degree turn from the sink between 06:05-06:09, which reveals what is due south of the user while they are facing the sink .\n3. It appears that the microwave on the counter is the appliance that is due south of the user when they are standing at the kitchen sink .", "referenced_object_masks": [] }, { "key": "P09-20240623-120359:e50d3f79f5d01bdb433560993a1783cf1ff94f20", "video_id": "P09-20240623-120359", "question": "What did I do with the leftover meat?", "answer_choice_0": "I set it aside on the counter", "answer_choice_1": "I wrapped it up and put it in the freezer", "answer_choice_2": "I wrapped it up and put it in the fridge", "answer_choice_3": "I added it to a separate dish", "answer_choice_4": "I threw it away", "answer_id": 1, "answer": "I wrapped it up and put it in the freezer", "question_type": [ "Action Recognition", "Object Recognition", "Goal Reasoning" ], "objects_of_interest": [ "timer-d59705f7e130d64f", "bag-a8f088817e83b390", "knife-7ae1058802a2d588", "meat box-d91f02b6bc817f2e", "blender's lid-ead6db223b9bf936", "measuring jar-7b545b4808a5e2f8", "gray bowl-6283254f7c3bc941", "bowl2-9d8c45fa1153c18d", "blender-d7460356ec5e6cc6", "blender motor-7109bf56cf56e1f4", "spice jar-1210f4f4f01203dc", "new spice jar-fc7dc1d8c0698064", "spice jar3-aae56f76916dd0b8", "black peppercorn jar-0350a00775f69ec8", "cutlery tray-99f50b55d208bcf6", "spoon-23f9f68d6ce78389", "lid of black peppercorn jar-3435d15339f1c0fa", "some cardamom pods-e27758271fae927d", "spice jar4-9ac0bf6dc46e6e9c", "spice jar5-d691f402e8abb3e1", "spice jar 6-036b9138ef646ef1", "spice jar2-39906f235058e2df", "basket of spices-3484989d3f9fe435", "square thing-cc3937d3288c7f82", "unknown-c3ea67948e063c3f", "saucepan-6b9b99253a0b61ff", "saucepan2-504f3ab26b14b992", "saucepan's lid-44580101d11520f3", "saucepan3-134e94a24056636b", "saucepan's lid2-b41902901bf6f171", "new saucepan lid-98917c2e0603a074", "saucepan lid3-7edbaa6e0793bcef", "measuring jug-88db1439d0007727", "scale-b7d9f96064a4db58", "discarded food bag-59b06ad7f9160918", "napkin-9290099fe5f3abfe", "chunk of minced beef-03af9553115a3e68", "chunk of minced beef2-3b3d80a3984a6778", "chunk of minced beef3-c7f77d21787839b6", "roll of freezer bags-74f87c4fcdf12574", "freezer bag-5d2ee93d607dc65d", "chunk of minced beef4-67f9a475b5322cbf", "unknown pan-c5c854c7b96f3f55", "saucepan4-f8e6af174f8eb4ec", "white chopping board-1ccc6d2a79cbf260", "onion-02f77477ea73f067", "frying pan-1bc42a94f478b754", "kettle-10d40f103af00530", "large knife-13694cd71b173c51", "onion half-81df5fc5f4c0b1e0", "chunk of onion-5c7e44e5b09f8036", "bottle of oil-10eabb9489570f13", "beaker-b6a72cd8c007215c", "spoon2-704c9544d0891940", "cup of yogurt-dacfe1669a009a0d", "more beef chunks3-4a57fdb60490e20b", "cylinder-98f44bec0655509b", "lid of cylinder-0460af4b56499233", "spoon3-d868f2dc11d1ff55", "basket of vegetables-c7ff66dfa3c475ff", "chilies1-47dac01a35eb0bf4", "chilies2-c1bc4f5a7809c072", "chilies3-904d4642c6d5ef23", "chilies4-d397bcf81955676c", "chilies5-f943964fa952abb5", "chilies6-c99d47b7832653d5", "dishwashing soap bottle-f2a6ea13b967db73", "onion discards-92baec7b71ff0e68", "mango-3465b5c1c76c08cb", "spoon4-c6834aeeb9aa09ba", "spoon5-3b9d0e02641f5a60", "spoon6-07fff6fd13685278", "measuring cup-9e30c441522f457b", "salt bottle-8ea80fb62146aeb7", "container of ghee-e0f51ba4c6e11825", "tin of coconut milk-11c18e6cab2eea47", "jar of honey-601698a7a3974747", "tin opener-ecf647a59ea52388", "yogurt cup's lid-5c6fc7bf723998c8", "knife2-61b5d4aabcfc0487", "knife3-08689fcfc427dd6d", "mango half-5abf142d04650ff3", "slice of mango-bfb8b077b31742e0", "piece of mango skin-2570b5854d0d884c", "some mango skin-d31f98bf1c22a9a2", "green chopping board-baeaab8d8a1fda7f", "mango half2-4a7d9e9059a95014", "mango skin3-2c23c881f90b2f91", "mango skin4-1b772c0349a051f9", "slice of mango2-a1033f9793f846fc", "slice of mango3-684b425b8c6b961a", "slice of mango4-b7bb1ea6cfdabaa5", "slice of mango5-e30cb431d9b99c89", "slice of mango6-80e3a4286b99f8c8", "spice bottle lid-c3b07b6092751f18", "lid of tin of ghee-39c8e10fb9cfd1f8", "lid of jar of honey-7655edc34bdd4b5b", "bottle of cold water-73a965086c608ef5", "ice tray-72f2ae2c39573d8b", "ice tray's lid-18dffed19e5ec1d2", "blender's lid cover-24b79e6b19a9e7b5", "box of roll of cling film-b2e0f0d79762a311", "roll of cling film-91152383b6ba9bd6", "some cling film-fa70d51acb22f70d", "sponge-a6622eeb55f4279a", "some cling film2-0ad0a05f9f39c8fe", "spice jar unknown-94669f4c633d4c41", "mango bits-2eaf9d32478aa5f1", "mango bits2-399a236d3b6fa4fd", "mango bits3-9676de887a501e40", "mango bits4-df124e6244d30d27", "mango bits5-f354b376120ceaa7", "bowl3-15779f54e0f55b31", "tupperware-6d16462da19a13d7", "tumbler-d025677181f57aec", "lunchbox-28afac02a8f495b1", "fork-7159a8a251a8acab", "tumbler2-a41fd448fd7e87c4", "cable tie-f39490b6dbcbd0aa", "foil-83d861b69e06102c", "obj2-10d1a1392a7fbb45", "packet of mold-88f11f7b514e8f7a", "roll of foodwaste bag-e041ee160b0dc821", "bin bag-eb6175eaac49d67b", "mango skinx-6c861f5630621c3f", "keep cup-d8746624eef5b59d", "spoon-unknown-21fef58f55c972d6", "stack of keep cups-6e4f59c0ff648504", "keep cup cover-c5e8d4c663219532", "keep cup2-677ff477d4d26c13", "tupperware lid-43adb08620439946", "whisk-e333f78f343bc1ed" ], "reasoning": "1. I watched to see when the user interacts with meat and saw this happen between 03:52-05:56.\n2. During this time, the user put most of the meat (discarded food bag-59b06ad7f9160918, chunk of minced beef-03af9553115a3e68, chunk of minced beef2-3b3d80a3984a6778) in a glass bowl (measuring jug-88db1439d0007727) between 4:07-4:33, and put the left over meat (chunk of minced beef3-c7f77d21787839b6, chunk of minced beef4-67f9a475b5322cbf) back in the meat box (meat box-d91f02b6bc817f2e) momentarily at 04:33, then later put it all into a freezer bag (freezer bag-5d2ee93d607dc65d) at 05:00.\n3. The user then proceed to put this freezer bag (freezer bag-5d2ee93d607dc65d) full of the leftover meat (chunk of minced beef3-c7f77d21787839b6, chunk of minced beef4-67f9a475b5322cbf) inside of the freezer at 05:11.", "referenced_object_masks": [ "chunk of minced beef3-c7f77d21787839b6", "chunk of minced beef4-67f9a475b5322cbf", "meat box-d91f02b6bc817f2e", "discarded food bag-59b06ad7f9160918", "chunk of minced beef-03af9553115a3e68", "chunk of minced beef2-3b3d80a3984a6778", "freezer bag-5d2ee93d607dc65d", "measuring jug-88db1439d0007727" ] }, { "key": "P09-20240623-153253:17f7fb412a34cc1ac4427fe6f089ef37b71e96a2", "video_id": "P09-20240623-153253", "question": "Approximately how much time passed between me rinsing the gray bowl and the green bowl?", "answer_choice_0": "20 seconds", "answer_choice_1": "10 seconds", "answer_choice_2": "30 seconds", "answer_choice_3": "50 seconds", "answer_choice_4": "40 seconds", "answer_id": 2, "answer": "30 seconds", "question_type": [ "Object Recognition", "Action Recognition", "Temporal Reasoning", "Numerical Reasoning", "State Changes" ], "objects_of_interest": [ "blender's pitcher-3ef0086d49aa3ed0", "blender's pitcher cover-de0ac8fa045c34ca", "measuring cup-43d975016615f932", "sponge-5cf8f04506244465", "washing up liquid-5976fefef460d7db", "white mug-9b8fbcbc1ade1595", "spoon-f4b91b0b9c02e9cf", "white bowl-aef6e1b448d74932", "green bowl-bddf0cba1a2c2173", "grey bowl-d0ec00f072d7e9c6", "mug-efed071e7d3a218b", "keep cup-ffb55a882b5cfa21", "mug1-b9f1c84a189ded1d", "pot cover-ecc19e02a49b5fb4", "saucepan-6869c5467aa5603b", "saucepan2-efa2cea37424e9f7", "kitchen rag-8811d9a9a22fbc42" ], "reasoning": "1. I watched the video and found the gray bowl (grey bowl-d0ec00f072d7e9c6) and the green bowl (green bowl-bddf0cba1a2c2173) at 00:31.\n2. I saw the user rinse the gray bowl (grey bowl-d0ec00f072d7e9c6) at 01:19.\n3. I saw the user rinse the green bowl (green bowl-bddf0cba1a2c2173) at 01:48.\n4. I calculated 01:48-01:19=00:29, which approximates to 30 seconds.\n5. Therefore, approximately 30 seconds passed between the user rinsing the gray bowl (grey bowl-d0ec00f072d7e9c6) and the green bowl (green bowl-bddf0cba1a2c2173).", "referenced_object_masks": [ "grey bowl-d0ec00f072d7e9c6", "green bowl-bddf0cba1a2c2173" ] }, { "key": "P09-20240623-153253:38307811fc477711dd7d6fdbc67db71f1f00f62c", "video_id": "P09-20240623-153253", "question": "What direction would I have turned to look at the dishwasher if it were in the opposite direction while I was at the sink?", "answer_choice_0": "Directly below", "answer_choice_1": "Directly behind", "answer_choice_2": "Directly ahead", "answer_choice_3": "To the right", "answer_choice_4": "To the left", "answer_id": 3, "answer": "To the right", "question_type": [ "Situational Awareness", "Egocentric & Spatial Perception", "Temporal Reasoning", "Counterfactual" ], "objects_of_interest": [ "blender's pitcher-3ef0086d49aa3ed0", "blender's pitcher cover-de0ac8fa045c34ca", "measuring cup-43d975016615f932", "sponge-5cf8f04506244465", "washing up liquid-5976fefef460d7db", "white mug-9b8fbcbc1ade1595", "spoon-f4b91b0b9c02e9cf", "white bowl-aef6e1b448d74932", "green bowl-bddf0cba1a2c2173", "grey bowl-d0ec00f072d7e9c6", "mug-efed071e7d3a218b", "keep cup-ffb55a882b5cfa21", "mug1-b9f1c84a189ded1d", "pot cover-ecc19e02a49b5fb4", "saucepan-6869c5467aa5603b", "saucepan2-efa2cea37424e9f7", "kitchen rag-8811d9a9a22fbc42" ], "reasoning": "1. I watched the video and found the sink at 00:03.\n2. I saw the dishwasher to the left of the user when they were at the sink and turned left to look from the sink to the dishwasher at 01:56-01:58.\n3. The opposite of \"left\" is \"right.\"\n4. Therefore, the user would have turned to the right to look at the dishwasher if it were in the opposite direction while they were at the sink.", "referenced_object_masks": [] }, { "key": "P09-20240623-153253:a7ae3e3e6086c0319b6adbc481d983d6102e94f7", "video_id": "P09-20240623-153253", "question": "What was the 3rd item I cleaned in the sink using the sponge?", "answer_choice_0": "A silver spoon", "answer_choice_1": "A green bowl", "answer_choice_2": "A gray bowl", "answer_choice_3": "A white bowl", "answer_choice_4": "A white mug", "answer_id": 1, "answer": "A green bowl", "question_type": [ "Situational Awareness", "Action Recognition", "Counting", "Object Recognition" ], "objects_of_interest": [ "blender's pitcher-3ef0086d49aa3ed0", "blender's pitcher cover-de0ac8fa045c34ca", "measuring cup-43d975016615f932", "sponge-5cf8f04506244465", "washing up liquid-5976fefef460d7db", "white mug-9b8fbcbc1ade1595", "spoon-f4b91b0b9c02e9cf", "white bowl-aef6e1b448d74932", "green bowl-bddf0cba1a2c2173", "grey bowl-d0ec00f072d7e9c6", "mug-efed071e7d3a218b", "keep cup-ffb55a882b5cfa21", "mug1-b9f1c84a189ded1d", "pot cover-ecc19e02a49b5fb4", "saucepan-6869c5467aa5603b", "saucepan2-efa2cea37424e9f7", "kitchen rag-8811d9a9a22fbc42" ], "reasoning": "1. I watched the video and found the sink at 00:03. From this point, I started to count each item the user cleaned in the sink with the sponge (sponge-5cf8f04506244465).\n2. I saw the user clean the 1st item in the sink with the sponge (sponge-5cf8f04506244465), which was a white bowl (white bowl-aef6e1b448d74932). This occurred from 00:36-00:49.\n3. I saw the user clean the 2nd item in the sink with the sponge (sponge-5cf8f04506244465), which was a green bowl (green bowl-bddf0cba1a2c2173). This occurred from 00:52-01:06.\n4. I saw the user clean the 3rd item in the sink with the sponge (sponge-5cf8f04506244465), which was a gray bowl (grey bowl-d0ec00f072d7e9c6). This occurred from 01:09-01:23.\n5. Therefore, the 3rd item the user cleaned in the sink with the sponge (sponge-5cf8f04506244465) was a gray bowl (grey bowl-d0ec00f072d7e9c6).", "referenced_object_masks": [ "grey bowl-d0ec00f072d7e9c6", "sponge-5cf8f04506244465", "white bowl-aef6e1b448d74932", "green bowl-bddf0cba1a2c2173" ] }, { "key": "P09-20240623-153253:ca9f6d95b318cfdca7ad2307b60f0802201956bc", "video_id": "P09-20240623-153253", "question": "If I doubled the number of spoons that were in the sink by the end of the video, how many would there have been?", "answer_choice_0": "1", "answer_choice_1": "3", "answer_choice_2": "4", "answer_choice_3": "5", "answer_choice_4": "2", "answer_id": 2, "answer": "4", "question_type": [ "Counting", "Situational Awareness", "Object Recognition", "Temporal Reasoning", "Numerical Reasoning", "Counterfactual" ], "objects_of_interest": [ "blender's pitcher-3ef0086d49aa3ed0", "blender's pitcher cover-de0ac8fa045c34ca", "measuring cup-43d975016615f932", "sponge-5cf8f04506244465", "washing up liquid-5976fefef460d7db", "white mug-9b8fbcbc1ade1595", "spoon-f4b91b0b9c02e9cf", "white bowl-aef6e1b448d74932", "green bowl-bddf0cba1a2c2173", "grey bowl-d0ec00f072d7e9c6", "mug-efed071e7d3a218b", "keep cup-ffb55a882b5cfa21", "mug1-b9f1c84a189ded1d", "pot cover-ecc19e02a49b5fb4", "saucepan-6869c5467aa5603b", "saucepan2-efa2cea37424e9f7", "kitchen rag-8811d9a9a22fbc42" ], "reasoning": "1. I watched the video and counted the number of spoons that were in the sink by the end of the video.\n2. At 04:08, I saw a total of 2 spoons (spoon-f4b91b0b9c02e9cf) in a white mug (white mug-9b8fbcbc1ade1595).\n3. Since the user did not add or remove any other spoons to or from the sink, there were a total of 2 spoons (spoon-f4b91b0b9c02e9cf) in it by the end of the video.\n4. I calculated 2*2=4.\n5. Therefore, if the user doubled the number of spoons that were in the sink by the end of the video, there would have been 4.", "referenced_object_masks": [ "spoon-f4b91b0b9c02e9cf", "white mug-9b8fbcbc1ade1595" ] }, { "key": "P09-20240623-153253:e828b30b98a14d34a5b7428c3a00ca3dcec626da", "video_id": "P09-20240623-153253", "question": "Where did I put the green bowl after rinsing it?", "answer_choice_0": "In the kitchen sink", "answer_choice_1": "On a kitchen countertop", "answer_choice_2": "On a kitchen tabletop", "answer_choice_3": "On the kitchen stove", "answer_choice_4": "In the dishwasher", "answer_id": 1, "answer": "On a kitchen countertop", "question_type": [ "Object Recognition", "Action Recognition", "Temporal Reasoning", "Situational Awareness", "State Changes" ], "objects_of_interest": [ "blender's pitcher-3ef0086d49aa3ed0", "blender's pitcher cover-de0ac8fa045c34ca", "measuring cup-43d975016615f932", "sponge-5cf8f04506244465", "washing up liquid-5976fefef460d7db", "white mug-9b8fbcbc1ade1595", "spoon-f4b91b0b9c02e9cf", "white bowl-aef6e1b448d74932", "green bowl-bddf0cba1a2c2173", "grey bowl-d0ec00f072d7e9c6", "mug-efed071e7d3a218b", "keep cup-ffb55a882b5cfa21", "mug1-b9f1c84a189ded1d", "pot cover-ecc19e02a49b5fb4", "saucepan-6869c5467aa5603b", "saucepan2-efa2cea37424e9f7", "kitchen rag-8811d9a9a22fbc42" ], "reasoning": "1. I watched the video and found the green bowl (green bowl-bddf0cba1a2c2173) at 00:31.\n2. I saw the user rinse the green bowl (green bowl-bddf0cba1a2c2173) at 01:48-01:50.\n3. After the user rinsed the green bowl (green bowl-bddf0cba1a2c2173) at 01:48-01:50, I saw them put it on a kitchen countertop at 02:01.\n4. Therefore, the user put the green (green bowl-bddf0cba1a2c2173) on a kitchen countertop after rinsing it.", "referenced_object_masks": [ "green bowl-bddf0cba1a2c2173" ] }, { "key": "P09-20240623-153748:18d335edeb1882094fcd6ad5df389a5330254743", "video_id": "P09-20240623-153748", "question": "What was the second item I rinsed in the sink?", "answer_choice_0": "A red pot", "answer_choice_1": "A black pan", "answer_choice_2": "A silver spoon", "answer_choice_3": "A gray mug", "answer_choice_4": "A white bowl", "answer_id": 0, "answer": "A red pot", "question_type": [ "Situational Awareness", "Counting", "Action Recognition", "Object Recognition" ], "objects_of_interest": [ "saucepan-e540f83d3e04eb2b", "red saucepan-09e767bdfb1fae10", "white mug-79625405dc911808", "spoon-dd0e5d6b86c2bf73", "fork-bd41be5dd3a82249", "sponge-4223e49fee7378e0", "cloth-1cb52fe7bfea5880", "stove metal cover right-a80e4da07c4ebc47", "stove metal covers-62fdd2cff355bbc5", "left stove metal cover-08d447c30215f8a4", "green bowl-7c04b644763eebc2", "washing up liquid-0fa0d3ec75c85506", "kettle-91481c7c7c83bea8", "coffee maker-5e8829c89c36dd2f", "orange box-fadff2ed6e85c617", "chocolate machine's wires-d50c71bda6d2d6d1", "drawer contents-2f768a9d5a0fbf80", "chocolate machine-d971fd86a06402b2", "fruit basket-e5b7b121eaf1cf69", "white packet-fe2d389047daadb5", "tomato box-d6b9488ea03848a4", "bag of tea-2ede2b8dcb1affba", "tea bag-76ab2020c197e3c8", "jar of sugar-ddea7e0ac34f4435", "bunch of forks-9c16138bc80ad5bf", "fork in group-d6d8e4222e7f840e", "box-67ba27f9716f2567", "teaspoon-05eae1067ad38931" ], "reasoning": "1. I watched the video and found the sink at 00:01.\n2. I counted each item the user rinsed in the sink.\n3. I saw the user rinse the first item in the sink at 00:04, which was a black pot (saucepan-e540f83d3e04eb2b).\n4. I saw the user rinse the second item in the sink at 00:10, which was a red pot (red saucepan-09e767bdfb1fae10).\n5. Therefore, the second item the user rinsed in the sink was a red pot (red saucepan-09e767bdfb1fae10).", "referenced_object_masks": [ "saucepan-e540f83d3e04eb2b", "red saucepan-09e767bdfb1fae10" ] }, { "key": "P09-20240623-153748:46723f0a33e84254fd544a73924c26d642fae823", "video_id": "P09-20240623-153748", "question": "If I added the number of pieces of cutlery I either took out or put in the cutlery drawer to the number of grates I took off the stove, what would the total be?", "answer_choice_0": "5", "answer_choice_1": "4", "answer_choice_2": "3", "answer_choice_3": "2", "answer_choice_4": "1", "answer_id": 0, "answer": "5", "question_type": [ "Counting", "Object Recognition", "Action Recognition", "Numerical Reasoning", "Counterfactual" ], "objects_of_interest": [ "saucepan-e540f83d3e04eb2b", "red saucepan-09e767bdfb1fae10", "white mug-79625405dc911808", "spoon-dd0e5d6b86c2bf73", "fork-bd41be5dd3a82249", "sponge-4223e49fee7378e0", "cloth-1cb52fe7bfea5880", "stove metal cover right-a80e4da07c4ebc47", "stove metal covers-62fdd2cff355bbc5", "left stove metal cover-08d447c30215f8a4", "green bowl-7c04b644763eebc2", "washing up liquid-0fa0d3ec75c85506", "kettle-91481c7c7c83bea8", "coffee maker-5e8829c89c36dd2f", "orange box-fadff2ed6e85c617", "chocolate machine's wires-d50c71bda6d2d6d1", "drawer contents-2f768a9d5a0fbf80", "chocolate machine-d971fd86a06402b2", "fruit basket-e5b7b121eaf1cf69", "white packet-fe2d389047daadb5", "tomato box-d6b9488ea03848a4", "bag of tea-2ede2b8dcb1affba", "tea bag-76ab2020c197e3c8", "jar of sugar-ddea7e0ac34f4435", "bunch of forks-9c16138bc80ad5bf", "fork in group-d6d8e4222e7f840e", "box-67ba27f9716f2567", "teaspoon-05eae1067ad38931" ], "reasoning": "1. At 00:36, I saw the user place 1 spoon (spoon-dd0e5d6b86c2bf73) and 1 fork (fork-bd41be5dd3a82249) inside the cutlery drawer.\n2. At 03:14, I saw the user open the cutlery drawer again. After rifling through a few items, the user takes a wire tie (chocolate machine's wires-d50c71bda6d2d6d1) out with their right hand at 03:20 and then transfers that item to their left hand. I did not count this instance since it did not involve cutlery. At 03:26, the user takes out a ring of biscut cutters (drawer contents-2f768a9d5a0fbf80) that were jamming the drawer before placing them back inside the drawer and shutting it, which I also didn't count since it didn't involve cutlery. At 03:40, the user puts a small piece of wire back into the drawer, which I did not count. Finally, at 04:41 the user takes a spoon (teaspoon-05eae1067ad38931) out of the cutlery drawer which was the last piece of cutlery they added or removed from the drawer. \n3. I added the total number of times the user takes cutlery out or puts it in the drawer for a total of 3. \n4. Next, I looked for the user to remove the grates from the stove and at 01:44, I saw the user remove 2 grates (stove metal covers-62fdd2cff355bbc5, left stove metal cover-08d447c30215f8a4) from the stove in order to clean it. \n5. Finally, I added the total number of cutlery items (spoon-dd0e5d6b86c2bf73, fork-bd41be5dd3a82249, teaspoon-05eae1067ad38931) added or removed from the cutlery drawer (3) to the total number of grates (stove metal covers-62fdd2cff355bbc5, left stove metal cover-08d447c30215f8a4) removed from the stove (2) for a total of 5.", "referenced_object_masks": [ "spoon-dd0e5d6b86c2bf73", "chocolate machine's wires-d50c71bda6d2d6d1", "drawer contents-2f768a9d5a0fbf80", "stove metal covers-62fdd2cff355bbc5", "left stove metal cover-08d447c30215f8a4", "fork-bd41be5dd3a82249", "teaspoon-05eae1067ad38931" ] }, { "key": "P09-20240623-153748:52ee5c014c9d4de709f8d62edf6a1bac2a10a6b1", "video_id": "P09-20240623-153748", "question": "Approximately how much time passed after I removed the stovetop grates and before I started to put them back into place?", "answer_choice_0": "45 seconds", "answer_choice_1": "60 seconds", "answer_choice_2": "0 seconds", "answer_choice_3": "30 seconds", "answer_choice_4": "15 seconds", "answer_id": 0, "answer": "45 seconds", "question_type": [ "Object Recognition", "Action Recognition", "State Changes", "Temporal Reasoning", "Numerical Reasoning" ], "objects_of_interest": [ "saucepan-e540f83d3e04eb2b", "red saucepan-09e767bdfb1fae10", "white mug-79625405dc911808", "spoon-dd0e5d6b86c2bf73", "fork-bd41be5dd3a82249", "sponge-4223e49fee7378e0", "cloth-1cb52fe7bfea5880", "stove metal cover right-a80e4da07c4ebc47", "stove metal covers-62fdd2cff355bbc5", "left stove metal cover-08d447c30215f8a4", "green bowl-7c04b644763eebc2", "washing up liquid-0fa0d3ec75c85506", "kettle-91481c7c7c83bea8", "coffee maker-5e8829c89c36dd2f", "orange box-fadff2ed6e85c617", "chocolate machine's wires-d50c71bda6d2d6d1", "drawer contents-2f768a9d5a0fbf80", "chocolate machine-d971fd86a06402b2", "fruit basket-e5b7b121eaf1cf69", "white packet-fe2d389047daadb5", "tomato box-d6b9488ea03848a4", "bag of tea-2ede2b8dcb1affba", "tea bag-76ab2020c197e3c8", "jar of sugar-ddea7e0ac34f4435", "bunch of forks-9c16138bc80ad5bf", "fork in group-d6d8e4222e7f840e", "box-67ba27f9716f2567", "teaspoon-05eae1067ad38931" ], "reasoning": "1. I watched the video and found the stovetop grates (stove metal covers-62fdd2cff355bbc5) at 01:23.\n2. I saw the user remove the stovetop grates (stove metal covers-62fdd2cff355bbc5) from the stove at approximately 01:45.\n3. I saw the user start to put the stovetop grates (stove metal covers-62fdd2cff355bbc5) back into place on the stove at approximately 02:30.\n4. I calculated 02:30-01:45=00:45.\n5. Therefore, approximately 45 seconds passed after the user removed the stovetop grates (stove metal covers-62fdd2cff355bbc5) and before they started to put them back into place.", "referenced_object_masks": [ "stove metal covers-62fdd2cff355bbc5" ] }, { "key": "P09-20240623-153748:623376147b580375c3d5f77dcb51fcb17c2341b3", "video_id": "P09-20240623-153748", "question": "What direction would I have to turn to look at the stove if it were on the opposite side of the room while I was filling the white kettle with water?", "answer_choice_0": "45 degrees to the right", "answer_choice_1": "90 degrees to the right", "answer_choice_2": "180 degrees to the right or left", "answer_choice_3": "90 degrees to the left", "answer_choice_4": "45 degrees to the left", "answer_id": 3, "answer": "90 degrees to the left", "question_type": [ "Object Recognition", "Action Recognition", "Situational Awareness", "Egocentric & Spatial Perception", "Cause and Effect", "Temporal Reasoning", "Counterfactual" ], "objects_of_interest": [ "saucepan-e540f83d3e04eb2b", "red saucepan-09e767bdfb1fae10", "white mug-79625405dc911808", "spoon-dd0e5d6b86c2bf73", "fork-bd41be5dd3a82249", "sponge-4223e49fee7378e0", "cloth-1cb52fe7bfea5880", "stove metal cover right-a80e4da07c4ebc47", "stove metal covers-62fdd2cff355bbc5", "left stove metal cover-08d447c30215f8a4", "green bowl-7c04b644763eebc2", "washing up liquid-0fa0d3ec75c85506", "kettle-91481c7c7c83bea8", "coffee maker-5e8829c89c36dd2f", "orange box-fadff2ed6e85c617", "chocolate machine's wires-d50c71bda6d2d6d1", "drawer contents-2f768a9d5a0fbf80", "chocolate machine-d971fd86a06402b2", "fruit basket-e5b7b121eaf1cf69", "white packet-fe2d389047daadb5", "tomato box-d6b9488ea03848a4", "bag of tea-2ede2b8dcb1affba", "tea bag-76ab2020c197e3c8", "jar of sugar-ddea7e0ac34f4435", "bunch of forks-9c16138bc80ad5bf", "fork in group-d6d8e4222e7f840e", "box-67ba27f9716f2567", "teaspoon-05eae1067ad38931" ], "reasoning": "1. I watched the video and found the white kettle (kettle-91481c7c7c83bea8) at 03:01.\n2. I saw the user filling the kettle (kettle-91481c7c7c83bea8) with water at the sink at 03:04-03:08.\n3. I saw the stove at 03:08 when the user turned 90 degrees to the right to see it.\n4. If the stove were on the opposite side of the room, the user would have to turn 90 degrees left to see it.\n6. Therefore, while the user was filling the white kettle (kettle-91481c7c7c83bea8) with water, the user would have to turn 90 degrees to the left to look to find the stove if it were on the opposite side of the room.", "referenced_object_masks": [ "kettle-91481c7c7c83bea8" ] }, { "key": "P09-20240623-153748:c9b3691c4ca7c1485c2c402a7ac0655bb2b3c9d0", "video_id": "P09-20240623-153748", "question": "Where did I put the white mug after washing it?", "answer_choice_0": "On a countertop", "answer_choice_1": "In the dishwasher", "answer_choice_2": "On the stove", "answer_choice_3": "In the sink", "answer_choice_4": "On a drying rack", "answer_id": 4, "answer": "On a drying rack", "question_type": [ "Object Recognition", "Action Recognition", "Temporal Reasoning", "State Changes", "Situational Awareness" ], "objects_of_interest": [ "saucepan-e540f83d3e04eb2b", "red saucepan-09e767bdfb1fae10", "white mug-79625405dc911808", "spoon-dd0e5d6b86c2bf73", "fork-bd41be5dd3a82249", "sponge-4223e49fee7378e0", "cloth-1cb52fe7bfea5880", "stove metal cover right-a80e4da07c4ebc47", "stove metal covers-62fdd2cff355bbc5", "left stove metal cover-08d447c30215f8a4", "green bowl-7c04b644763eebc2", "washing up liquid-0fa0d3ec75c85506", "kettle-91481c7c7c83bea8", "coffee maker-5e8829c89c36dd2f", "orange box-fadff2ed6e85c617", "chocolate machine's wires-d50c71bda6d2d6d1", "drawer contents-2f768a9d5a0fbf80", "chocolate machine-d971fd86a06402b2", "fruit basket-e5b7b121eaf1cf69", "white packet-fe2d389047daadb5", "tomato box-d6b9488ea03848a4", "bag of tea-2ede2b8dcb1affba", "tea bag-76ab2020c197e3c8", "jar of sugar-ddea7e0ac34f4435", "bunch of forks-9c16138bc80ad5bf", "fork in group-d6d8e4222e7f840e", "box-67ba27f9716f2567", "teaspoon-05eae1067ad38931" ], "reasoning": "1. I watched the video and found the white mug (white mug-79625405dc911808) at 00:20.\n2. I saw the user wash the white mug (white mug-79625405dc911808) at 00:20-00:30.\n3. I saw the user move the white mug (white mug-79625405dc911808) from the sink to a drying rack at 00:30.\n4. Therefore, the user put the white mug (white mug-79625405dc911808) on a drying rack after washing it.", "referenced_object_masks": [ "white mug-79625405dc911808" ] }, { "key": "P09-20240623-154547:4816a7422430d5395b71f43356f7d5b7058915c4", "video_id": "P09-20240623-154547", "question": "If I multiplied the amount of time it takes me to pour water into the mug (counting from when the water is first visible) by the amount of time the skylight is visible, what would the resulting amount of time be?", "answer_choice_0": "18 seconds", "answer_choice_1": "28 seconds", "answer_choice_2": "15 seconds", "answer_choice_3": "10 seconds", "answer_choice_4": "20 seconds", "answer_id": 4, "answer": "20 seconds", "question_type": [ "Numerical Reasoning", "Action Recognition", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "kettle-e7314fb33e3a4067", "teaspoon-40e19d808e24a612", "mug-aaca81131543fa1a", "cloth-9e81ebc1c7f28c34" ], "reasoning": "1. I watched the video to determine how long it takes for the user to pour water into the mug (mug-aaca81131543fa1a), and also how long the skylight is visible. First, I identified the mug (mug-aaca81131543fa1a), which is visible right at the start of the video, at 00:00.\n2. The user then grabbed a kettle (kettle-e7314fb33e3a4067) at 00:05, and then proceeded to pour water from the kettle (kettle-e7314fb33e3a4067) into the mug (mug-aaca81131543fa1a).\n3. The user poured visible water into the mug (mug-aaca81131543fa1a) from 00:07-00:12, for a total of 5 seconds.\n4. Then, I looked for the skylight . I found it came into view on the ceiling at 00:31.\n5. The user turned off the video at 00:35, and the skylight was visible for this entire time, for a total of 4 seconds.\n6. I multipled 5 seconds by 4 seconds to get a total of 20 seconds.", "referenced_object_masks": [ "kettle-e7314fb33e3a4067", "mug-aaca81131543fa1a" ] }, { "key": "P09-20240623-154547:6675d3cebb5be2b1e3bc5f1a4296fca46516ef42", "video_id": "P09-20240623-154547", "question": "What was directly to my left as I picked up the 4th unique object?", "answer_choice_0": "The fridge", "answer_choice_1": "The kettle", "answer_choice_2": "The table", "answer_choice_3": "The sink", "answer_choice_4": "The oven", "answer_id": 3, "answer": "The sink", "question_type": [ "Action Recognition", "Counting", "Egocentric & Spatial Perception", "Situational Awareness" ], "objects_of_interest": [ "kettle-e7314fb33e3a4067", "teaspoon-40e19d808e24a612", "mug-aaca81131543fa1a", "cloth-9e81ebc1c7f28c34" ], "reasoning": "1. I watched the video in order to figure out what was directly to the left of the user as they picked up their 4th unique object. The 1st unique object the user picked up was the teaspoon (teaspoon-40e19d808e24a612), which they picked up at 00:04 with their right hand.\n2. The 2nd unique object the user picked up was the kettle (kettle-e7314fb33e3a4067), which they also picked up at 00:04.\n3. The 3rd unique object the user picked up was the mug (mug-aaca81131543fa1a), which they picked up at 00:19 after handling it for several seconds on the countertop.\n4. The 4th unique object the user picked up was the cloth (cloth-9e81ebc1c7f28c34), which they picked up at 00:23.\n5. Since this was the 4th unique object, I tried to determine what was to the user's left. Right at 00:23, the user looked briefly to the left for a moment, and part of the sink came into view. Since none of the other objects were to the left of the user at this time, I determined it was the sink .", "referenced_object_masks": [ "teaspoon-40e19d808e24a612", "kettle-e7314fb33e3a4067", "mug-aaca81131543fa1a", "cloth-9e81ebc1c7f28c34" ] }, { "key": "P09-20240623-154547:8ad89ad032d9b54b7bc61a466e3d913fa63802da", "video_id": "P09-20240623-154547", "question": "If I turned around 180 degrees while standing at the cutting board, what is one item that would be directly in front of me?", "answer_choice_0": "A basket", "answer_choice_1": "A mop", "answer_choice_2": "A stool", "answer_choice_3": "A kettle", "answer_choice_4": "A toaster", "answer_id": 2, "answer": "A stool", "question_type": [ "Counterfactual", "Action Recognition", "Object Recognition", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "kettle-e7314fb33e3a4067", "teaspoon-40e19d808e24a612", "mug-aaca81131543fa1a", "cloth-9e81ebc1c7f28c34" ], "reasoning": "1. I watched the video and tried to first locate the cutting board . It is visible right at 00:00, far into the kitchen on a stack of dishes near the sink. It comes more fully into view at 00:20, near the lower right corner of the sink.\n2. Then, I tried to determine what would be 180 degrees in the opposite direction of the cutting board . At 00:00, I determined that the place the user would stand to be in front of the cutting board aligns with the wall on that side of the kitchen.\n3. From 00:30-00:33, the user makes a full turn away from looking at the kitchen to see what is on the opposite side. There are many items present, but I noticed the corner of the same wall I saw at 00:00 on the right side.\n4. Using the wall as a reference point, I aligned it with the tiles on the floor to find a stool on the ground with clothing draped on top of it. Since this approximately aligns, being just one tile over, and none of the other items appear to be nearby, I determined that a stool is the correct answer.", "referenced_object_masks": [] }, { "key": "P09-20240623-154547:8fbe1b43a426d519a114ce162fcff534cb22337b", "video_id": "P09-20240623-154547", "question": "From the spot where I wiped off a surface using a cloth, what steps would I need to take to be standing right in front of the dishwasher facing it?", "answer_choice_0": "Turn right, walk forward, turn left, walk forward", "answer_choice_1": "Turn left, walk forward, turn left, walk forward", "answer_choice_2": "Turn left, walk forward, turn right, walk forward", "answer_choice_3": "Turn right, walk forward, turn left, walk forward", "answer_choice_4": "Turn right, walk forward, turn right, walk forward", "answer_id": 4, "answer": "Turn right, walk forward, turn right, walk forward", "question_type": [ "Egocentric & Spatial Perception", "Action Recognition", "Object Recognition", "Goal Reasoning", "Situational Awareness" ], "objects_of_interest": [ "kettle-e7314fb33e3a4067", "teaspoon-40e19d808e24a612", "mug-aaca81131543fa1a", "cloth-9e81ebc1c7f28c34" ], "reasoning": "1. I watched the video in order to locate where the user is when they used the cloth (cloth-9e81ebc1c7f28c34) to wipe off a surface. I found that the user picked up the cloth (cloth-9e81ebc1c7f28c34) at 00:23, while they are standing near the sink.\n2. Then, the user walked to the right to arrive at the other side of the counter, and they used the cloth (cloth-9e81ebc1c7f28c34) to wipe off the counter surface from 00:25-00:26.\n3. Then, I looked for the dishwasher. Rewinding back, I found the dishwasher right at the start of the video, at 00:00. It is located to the left of the sink.\n4. At 00:00, the rest of the kitchen is visible, including the spots where the user picked up the cloth (cloth-9e81ebc1c7f28c34) and also used the cloth (cloth-9e81ebc1c7f28c34) to wipe off the counter surface. Using these bearings, I can see that the user would need to turn right, walk forward, turn right again, then walk forward once more to be right in front of the dishwasher and facing it.", "referenced_object_masks": [ "cloth-9e81ebc1c7f28c34" ] }, { "key": "P09-20240623-154547:ebf6017dd0f81ebe6c31e18ab9e07eb30f0e783c", "video_id": "P09-20240623-154547", "question": "How many times would I have stirred the liquid in a counter-clockwise motion if I had done so half as many times?", "answer_choice_0": "6", "answer_choice_1": "4", "answer_choice_2": "8", "answer_choice_3": "7", "answer_choice_4": "10", "answer_id": 3, "answer": "7", "question_type": [ "Action Recognition", "Counting", "Counterfactual", "Numerical Reasoning" ], "objects_of_interest": [ "kettle-e7314fb33e3a4067", "teaspoon-40e19d808e24a612", "mug-aaca81131543fa1a", "cloth-9e81ebc1c7f28c34" ], "reasoning": "1. I watched the video and looked for how many times the user stirred the liquid in a counter-clockwise motion. I saw them pour water from the kettle (kettle-e7314fb33e3a4067) into the mug (mug-aaca81131543fa1a) from 00:07-00:12.\n2. Then, the user picked up the teaspoon (teaspoon-40e19d808e24a612) at 00:12 and began to stir the liquid.\n3. I watched carefully to count how many times the user stirred the liquid in a counter-clockwise motion. They did so 14 total times from 00:12-00:19, and then they began to stir in the other direction.\n4. I halved that total by dividing 14 by 2, arriving at a total of 7.", "referenced_object_masks": [ "teaspoon-40e19d808e24a612", "kettle-e7314fb33e3a4067", "mug-aaca81131543fa1a" ] }, { "key": "P09-20240624-160737:0a00f8053bd0921750feb9c3efb3116c40eaf930", "video_id": "P09-20240624-160737", "question": "If I take the number of french fries I eat at 28:45, the number of items in the drying rack at 01:32 and the number of chopping boards in the drawer at 02:08, add them together and then divide by 0.5, what number do I get? If needed, round to the nearest whole number.", "answer_choice_0": "8", "answer_choice_1": "12", "answer_choice_2": "4", "answer_choice_3": "16", "answer_choice_4": "2", "answer_id": 3, "answer": "16", "question_type": [ "Action Recognition", "Counting", "Event Occurence", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "napkin1-ff3dfc6bb3a6dbb3", "napkin2-d768b4e4c7ab80ce", "towel1-1826f09bf065f800", "towel2-0f7b78898e80b33e", "yellow cloth-3e3aec729eecf1a7", "packet of skewers-7da002e549e3efcd", "basket of vegetables-32eb8570f99de8d7", "bowl of onions-b44f1a1f51f58b6b", "bottle of garlic-a540402b60ac255d", "pyrex bowl of meat-3734941861dc436e", "tomato-d0436d2afcc2a187", "chopping board-bd1354ef2860fe76", "knife-8e82ae95af60de12", "tomato2-202aa5f778a18bfc", "tomato3-8babe98f1ad58c22", "cap of bottle-89b397c6223669e3", "bottle of oil-9ea52ecb2a41802b", "bottle of salt-148f7396690107d6", "skewer1-1ad69c6ab1e02769", "onion-56122b1b5acc2eb4", "skewer3-f28fa2dd50f318ef", "skewer2-5bf4594e1353dff0", "skewer4-833ca96f3f4d0492", "tomato4-c670150372b4324a", "baking tray1-eb0d57521614950e", "baking tray2-99f5c911c99fa6c8", "baking tray3-6110605e2d0908c3", "baking parchment's carton-20c32e1df5944c28", "roll of baking parchment-d244461079c7b6c5", "torn baking parchment-8be4476669058797", "excess parchment-717db5d2509ccbe4", "oven tray-f3f0ebf9fd65e098", "bottle of washing up liquid-44aeb1fefaa03368", "sponge-01e3762820a17dfb", "saucepan-11a865104bd8bdad", "saucepan2-72fbcfb9ca70e9bb", "saucepan3-da7e2a48fafd6cfd", "cardboard-e34b1f296597796a", "scale-17b7f3a61816d772", "bag of potato-d2cf2472fe360ea8", "potato-f71a336296a66c7d", "potato2-6efe06781fb8cd5a", "basket2-118a29274c2fca90", "potato3-ecffa1957a89f9d9", "spoon2-5a8c3cd220ea5c8d", "fork-38a431cd7f5aa31c", "steaming rack-b6893992093591c5", "metal piece-b804c225ecc10995", "sieve-5e4e73504f1aff54", "plate-f447e2256500b987", "potato peeler-79adcf2424e50c7b", "spice basket-29ad5a518a1ea133", "spice jar-744fcd8850a35cf4", "spice jar2-d8fc27de9da84572", "spice jar3-56649558910aa1d8", "spice jar4-c5aae489e1c6a207", "spice jar5-6d3421f562fa3a01", "spice jar6-a74bfbfa839be1cd", "spice jar7-2fd58604a256d4fe", "spice jar8-5c9b92f38b7774be", "spice jar9-0e7da638db51b19b", "spice jar10-510b5587f260783b", "spice jar11-0bde0f593b85cd75", "carton spice-9eb3b9027a93c87b", "spice bag-7815399009270245", "spice jar12-eef5cd1baf1b2083", "spice bag2-2f295ed12183a449", "spice jar13-91c136399f59aea3", "spoon3-ed07a45cdd3f18b9", "lid-35b1473664ad2387", "lid2-48e778d8c6502cb3", "bowl-da95a1ef71dace7c", "plate2-a81bd877900165d4", "towel3-2d3d71d35e1ccde8", "food waste bag-5634d3a9f000b117", "saucepan4-2e4068543b09a822", "bottle of soap-42ec72a55d1e381f", "utensil3-e17a604d9b48146a", "wooden spoon-f89d3dae90fd1a2e", "keepcup-e90eb9b3f70e2da5", "ceramic mug-af3c79e0d7cc505c", "white mug-95392472c631b1d0", "mug3-642e39679469a03c", "napkin3-d2784a131878eaab" ], "reasoning": "1. I watched the whole video and saw the user eat 2 homemade french fries at 28:45.\n2. At 01:32, the number of items in the drying rack is 2.\n3. At 02:08, the number of chopping boards in the drawer is 4.\n5. Add together the number of french fries eaten, 2, the number of items in the drying rack , 2 and the number of chopping boards in the drawer, 4 and get 8.\n6. 8 divided by 0.5 is 16.", "referenced_object_masks": [] }, { "key": "P09-20240624-160737:27cf03c36a6720dbb53718ef8dbf58680cfbe384", "video_id": "P09-20240624-160737", "question": "If I take the number of tomato slices put on the second skewer, add it to the weight of the 3 potatoes and raise it to the 1/2 power, what number do I get? Round to the nearest tenth if needed.", "answer_choice_0": "24.7", "answer_choice_1": "24.95", "answer_choice_2": "23.9", "answer_choice_3": "24.3", "answer_choice_4": "25.1", "answer_id": 0, "answer": "24.7", "question_type": [ "Action Recognition", "Counting", "Event Occurence", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "napkin1-ff3dfc6bb3a6dbb3", "napkin2-d768b4e4c7ab80ce", "towel1-1826f09bf065f800", "towel2-0f7b78898e80b33e", "yellow cloth-3e3aec729eecf1a7", "packet of skewers-7da002e549e3efcd", "basket of vegetables-32eb8570f99de8d7", "bowl of onions-b44f1a1f51f58b6b", "bottle of garlic-a540402b60ac255d", "pyrex bowl of meat-3734941861dc436e", "tomato-d0436d2afcc2a187", "chopping board-bd1354ef2860fe76", "knife-8e82ae95af60de12", "tomato2-202aa5f778a18bfc", "tomato3-8babe98f1ad58c22", "cap of bottle-89b397c6223669e3", "bottle of oil-9ea52ecb2a41802b", "bottle of salt-148f7396690107d6", "skewer1-1ad69c6ab1e02769", "onion-56122b1b5acc2eb4", "skewer3-f28fa2dd50f318ef", "skewer2-5bf4594e1353dff0", "skewer4-833ca96f3f4d0492", "tomato4-c670150372b4324a", "baking tray1-eb0d57521614950e", "baking tray2-99f5c911c99fa6c8", "baking tray3-6110605e2d0908c3", "baking parchment's carton-20c32e1df5944c28", "roll of baking parchment-d244461079c7b6c5", "torn baking parchment-8be4476669058797", "excess parchment-717db5d2509ccbe4", "oven tray-f3f0ebf9fd65e098", "bottle of washing up liquid-44aeb1fefaa03368", "sponge-01e3762820a17dfb", "saucepan-11a865104bd8bdad", "saucepan2-72fbcfb9ca70e9bb", "saucepan3-da7e2a48fafd6cfd", "cardboard-e34b1f296597796a", "scale-17b7f3a61816d772", "bag of potato-d2cf2472fe360ea8", "potato-f71a336296a66c7d", "potato2-6efe06781fb8cd5a", "basket2-118a29274c2fca90", "potato3-ecffa1957a89f9d9", "spoon2-5a8c3cd220ea5c8d", "fork-38a431cd7f5aa31c", "steaming rack-b6893992093591c5", "metal piece-b804c225ecc10995", "sieve-5e4e73504f1aff54", "plate-f447e2256500b987", "potato peeler-79adcf2424e50c7b", "spice basket-29ad5a518a1ea133", "spice jar-744fcd8850a35cf4", "spice jar2-d8fc27de9da84572", "spice jar3-56649558910aa1d8", "spice jar4-c5aae489e1c6a207", "spice jar5-6d3421f562fa3a01", "spice jar6-a74bfbfa839be1cd", "spice jar7-2fd58604a256d4fe", "spice jar8-5c9b92f38b7774be", "spice jar9-0e7da638db51b19b", "spice jar10-510b5587f260783b", "spice jar11-0bde0f593b85cd75", "carton spice-9eb3b9027a93c87b", "spice bag-7815399009270245", "spice jar12-eef5cd1baf1b2083", "spice bag2-2f295ed12183a449", "spice jar13-91c136399f59aea3", "spoon3-ed07a45cdd3f18b9", "lid-35b1473664ad2387", "lid2-48e778d8c6502cb3", "bowl-da95a1ef71dace7c", "plate2-a81bd877900165d4", "towel3-2d3d71d35e1ccde8", "food waste bag-5634d3a9f000b117", "saucepan4-2e4068543b09a822", "bottle of soap-42ec72a55d1e381f", "utensil3-e17a604d9b48146a", "wooden spoon-f89d3dae90fd1a2e", "keepcup-e90eb9b3f70e2da5", "ceramic mug-af3c79e0d7cc505c", "white mug-95392472c631b1d0", "mug3-642e39679469a03c", "napkin3-d2784a131878eaab" ], "reasoning": "1. I watched the whole video and saw the user putting food items on the second skewer (skewer2-5bf4594e1353dff0) at 05:48.\n2. By 06:40, the user put 6 tomato slices (tomato-d0436d2afcc2a187, tomato2-202aa5f778a18bfc) on the skewer (skewer2-5bf4594e1353dff0).\n3. At 13:54, the user weighed 3 potatoes (potato-f71a336296a66c7d, potato2-6efe06781fb8cd5a, potato3-ecffa1957a89f9d9).\n4. At 13:59, the kitchen scale (scale-17b7f3a61816d772) says the weight is 604 g.\n5. Add the number of tomato slices (tomato-d0436d2afcc2a187, tomato2-202aa5f778a18bfc), 6 and the weight of the 3 potatoes (potato-f71a336296a66c7d, potato2-6efe06781fb8cd5a, potato3-ecffa1957a89f9d9) and get 610.\n7. Take the square root (same as raising to the 1/2 power) to get 24.69818.\n8. Round to the nearest tenth is 24.7.", "referenced_object_masks": [ "potato-f71a336296a66c7d", "potato2-6efe06781fb8cd5a", "potato3-ecffa1957a89f9d9", "tomato-d0436d2afcc2a187", "tomato2-202aa5f778a18bfc", "scale-17b7f3a61816d772", "skewer2-5bf4594e1353dff0" ] }, { "key": "P09-20240624-160737:280d389d2161af1b4acb6fe519f03792d8c15165", "video_id": "P09-20240624-160737", "question": "If I had placed the second batch of fries on the plate with the first batch, instead of on the large plate I placed them on, where would the second batch of fries sit in relation to the large plate, if I still moved the large plate to the place where I added the majority of fries to it?", "answer_choice_0": "To the right of the large plate", "answer_choice_1": "To the left of the large plate", "answer_choice_2": "Above the large plate", "answer_choice_3": "In front of the large plate", "answer_choice_4": "Behind the large plate", "answer_id": 0, "answer": "To the right of the large plate", "question_type": [ "Egocentric & Spatial Perception", "Counterfactual", "Event Occurence", "Counting" ], "objects_of_interest": [ "napkin1-ff3dfc6bb3a6dbb3", "napkin2-d768b4e4c7ab80ce", "towel1-1826f09bf065f800", "towel2-0f7b78898e80b33e", "yellow cloth-3e3aec729eecf1a7", "packet of skewers-7da002e549e3efcd", "basket of vegetables-32eb8570f99de8d7", "bowl of onions-b44f1a1f51f58b6b", "bottle of garlic-a540402b60ac255d", "pyrex bowl of meat-3734941861dc436e", "tomato-d0436d2afcc2a187", "chopping board-bd1354ef2860fe76", "knife-8e82ae95af60de12", "tomato2-202aa5f778a18bfc", "tomato3-8babe98f1ad58c22", "cap of bottle-89b397c6223669e3", "bottle of oil-9ea52ecb2a41802b", "bottle of salt-148f7396690107d6", "skewer1-1ad69c6ab1e02769", "onion-56122b1b5acc2eb4", "skewer3-f28fa2dd50f318ef", "skewer2-5bf4594e1353dff0", "skewer4-833ca96f3f4d0492", "tomato4-c670150372b4324a", "baking tray1-eb0d57521614950e", "baking tray2-99f5c911c99fa6c8", "baking tray3-6110605e2d0908c3", "baking parchment's carton-20c32e1df5944c28", "roll of baking parchment-d244461079c7b6c5", "torn baking parchment-8be4476669058797", "excess parchment-717db5d2509ccbe4", "oven tray-f3f0ebf9fd65e098", "bottle of washing up liquid-44aeb1fefaa03368", "sponge-01e3762820a17dfb", "saucepan-11a865104bd8bdad", "saucepan2-72fbcfb9ca70e9bb", "saucepan3-da7e2a48fafd6cfd", "cardboard-e34b1f296597796a", "scale-17b7f3a61816d772", "bag of potato-d2cf2472fe360ea8", "potato-f71a336296a66c7d", "potato2-6efe06781fb8cd5a", "basket2-118a29274c2fca90", "potato3-ecffa1957a89f9d9", "spoon2-5a8c3cd220ea5c8d", "fork-38a431cd7f5aa31c", "steaming rack-b6893992093591c5", "metal piece-b804c225ecc10995", "sieve-5e4e73504f1aff54", "plate-f447e2256500b987", "potato peeler-79adcf2424e50c7b", "spice basket-29ad5a518a1ea133", "spice jar-744fcd8850a35cf4", "spice jar2-d8fc27de9da84572", "spice jar3-56649558910aa1d8", "spice jar4-c5aae489e1c6a207", "spice jar5-6d3421f562fa3a01", "spice jar6-a74bfbfa839be1cd", "spice jar7-2fd58604a256d4fe", "spice jar8-5c9b92f38b7774be", "spice jar9-0e7da638db51b19b", "spice jar10-510b5587f260783b", "spice jar11-0bde0f593b85cd75", "carton spice-9eb3b9027a93c87b", "spice bag-7815399009270245", "spice jar12-eef5cd1baf1b2083", "spice bag2-2f295ed12183a449", "spice jar13-91c136399f59aea3", "spoon3-ed07a45cdd3f18b9", "lid-35b1473664ad2387", "lid2-48e778d8c6502cb3", "bowl-da95a1ef71dace7c", "plate2-a81bd877900165d4", "towel3-2d3d71d35e1ccde8", "food waste bag-5634d3a9f000b117", "saucepan4-2e4068543b09a822", "bottle of soap-42ec72a55d1e381f", "utensil3-e17a604d9b48146a", "wooden spoon-f89d3dae90fd1a2e", "keepcup-e90eb9b3f70e2da5", "ceramic mug-af3c79e0d7cc505c", "white mug-95392472c631b1d0", "mug3-642e39679469a03c", "napkin3-d2784a131878eaab" ], "reasoning": "1. To find this answer, I first determine when and where the first batch of fries are plated - this occurs from 24:10-24:36, when the fries are scooped out from the saucepan (saucepan-11a865104bd8bdad) and onto a plate (plate-f447e2256500b987) which sits to the right of the stove .\n2. Next, I determine when and where the second batch of fries are plated - this occurs from 29:31-30:21, where the user scoops the batch of fries out onto a large plate (plate2-a81bd877900165d4). At 29:31, the user scoops a single first fry from the batch and places it on the large plate (plate2-a81bd877900165d4) while that plate sits to the right of the smaller first plate (plate-f447e2256500b987) that the first batch of fries sits on. However, at 29:36, the user moves the large plate (plate2-a81bd877900165d4) from the right to the left side of the stovetop before scooping out any more fries , and scoops the rest out onto the large plate (plate2-a81bd877900165d4) while that plate sits to the left of the stovetop.\n3. Next, I counterfactually imagine that I had scooped the second batch of fries onto the plate (plate-f447e2256500b987) with the first batch , meaning that both batches would sit to the right of the stovetop. I also imagine that the large plate (plate2-a81bd877900165d4) was in the spot where I had scooped the majority of the fries -- to the left of the stovetop. Therefore, in this scenario, the second batch of fries would be located to the right of the large plate (plate2-a81bd877900165d4) after it was moved, making A the correct answer.", "referenced_object_masks": [ "plate2-a81bd877900165d4", "plate-f447e2256500b987", "saucepan-11a865104bd8bdad" ] }, { "key": "P09-20240624-160737:5a7fdea57ed8d526a6eb1b1b6ed10109ef82fb9e", "video_id": "P09-20240624-160737", "question": "What is the original location of the bag of food from which I obtain the first item that I weigh on the food scale, in relation to the kitchen stovetop?", "answer_choice_0": "To the left of the stovetop", "answer_choice_1": "Beneath and to the left of the stovetop", "answer_choice_2": "Above and to the left of the stovetop", "answer_choice_3": "Directly above the stovetop", "answer_choice_4": "To the right of the stovetop", "answer_id": 4, "answer": "To the right of the stovetop", "question_type": [ "Egocentric & Spatial Perception", "Object Recognition", "Event Occurence", "Temporal Reasoning" ], "objects_of_interest": [ "napkin1-ff3dfc6bb3a6dbb3", "napkin2-d768b4e4c7ab80ce", "towel1-1826f09bf065f800", "towel2-0f7b78898e80b33e", "yellow cloth-3e3aec729eecf1a7", "packet of skewers-7da002e549e3efcd", "basket of vegetables-32eb8570f99de8d7", "bowl of onions-b44f1a1f51f58b6b", "bottle of garlic-a540402b60ac255d", "pyrex bowl of meat-3734941861dc436e", "tomato-d0436d2afcc2a187", "chopping board-bd1354ef2860fe76", "knife-8e82ae95af60de12", "tomato2-202aa5f778a18bfc", "tomato3-8babe98f1ad58c22", "cap of bottle-89b397c6223669e3", "bottle of oil-9ea52ecb2a41802b", "bottle of salt-148f7396690107d6", "skewer1-1ad69c6ab1e02769", "onion-56122b1b5acc2eb4", "skewer3-f28fa2dd50f318ef", "skewer2-5bf4594e1353dff0", "skewer4-833ca96f3f4d0492", "tomato4-c670150372b4324a", "baking tray1-eb0d57521614950e", "baking tray2-99f5c911c99fa6c8", "baking tray3-6110605e2d0908c3", "baking parchment's carton-20c32e1df5944c28", "roll of baking parchment-d244461079c7b6c5", "torn baking parchment-8be4476669058797", "excess parchment-717db5d2509ccbe4", "oven tray-f3f0ebf9fd65e098", "bottle of washing up liquid-44aeb1fefaa03368", "sponge-01e3762820a17dfb", "saucepan-11a865104bd8bdad", "saucepan2-72fbcfb9ca70e9bb", "saucepan3-da7e2a48fafd6cfd", "cardboard-e34b1f296597796a", "scale-17b7f3a61816d772", "bag of potato-d2cf2472fe360ea8", "potato-f71a336296a66c7d", "potato2-6efe06781fb8cd5a", "basket2-118a29274c2fca90", "potato3-ecffa1957a89f9d9", "spoon2-5a8c3cd220ea5c8d", "fork-38a431cd7f5aa31c", "steaming rack-b6893992093591c5", "metal piece-b804c225ecc10995", "sieve-5e4e73504f1aff54", "plate-f447e2256500b987", "potato peeler-79adcf2424e50c7b", "spice basket-29ad5a518a1ea133", "spice jar-744fcd8850a35cf4", "spice jar2-d8fc27de9da84572", "spice jar3-56649558910aa1d8", "spice jar4-c5aae489e1c6a207", "spice jar5-6d3421f562fa3a01", "spice jar6-a74bfbfa839be1cd", "spice jar7-2fd58604a256d4fe", "spice jar8-5c9b92f38b7774be", "spice jar9-0e7da638db51b19b", "spice jar10-510b5587f260783b", "spice jar11-0bde0f593b85cd75", "carton spice-9eb3b9027a93c87b", "spice bag-7815399009270245", "spice jar12-eef5cd1baf1b2083", "spice bag2-2f295ed12183a449", "spice jar13-91c136399f59aea3", "spoon3-ed07a45cdd3f18b9", "lid-35b1473664ad2387", "lid2-48e778d8c6502cb3", "bowl-da95a1ef71dace7c", "plate2-a81bd877900165d4", "towel3-2d3d71d35e1ccde8", "food waste bag-5634d3a9f000b117", "saucepan4-2e4068543b09a822", "bottle of soap-42ec72a55d1e381f", "utensil3-e17a604d9b48146a", "wooden spoon-f89d3dae90fd1a2e", "keepcup-e90eb9b3f70e2da5", "ceramic mug-af3c79e0d7cc505c", "white mug-95392472c631b1d0", "mug3-642e39679469a03c", "napkin3-d2784a131878eaab" ], "reasoning": "1. To find this answer, I must first find when the user first weighs an item on the food scale (scale-17b7f3a61816d772). I find that the user brings out the food scale (scale-17b7f3a61816d772) from behind a collection of food on the right side of the countertop and turns it on at 13:34.\n2. Next, I observe the user take out a large bag of potatoes (bag of potato-d2cf2472fe360ea8) from this same collection of food on the right side of the cabinet from 13:35-13:38. They remove a potato (potato-f71a336296a66c7d) at 13:40, and another (potato2-6efe06781fb8cd5a) at 13:52, placing both of these on the food scale (scale-17b7f3a61816d772) from 13:53-13:54. Therefore, since these are the first items placed on the food scale (scale-17b7f3a61816d772), I recognize that the bag of food I need to target is the potato bag (bag of potato-d2cf2472fe360ea8).\n3. I remember that I observed that the user took the potato bag (bag of potato-d2cf2472fe360ea8) out from behind the same collection of food on the right side of the counter from 13:35-13:38, and think of this in relation to the stovetop - the stovetop is in the center of the cabinet, and the potato bag was to the right, making A the correct answer.", "referenced_object_masks": [ "bag of potato-d2cf2472fe360ea8", "scale-17b7f3a61816d772", "potato-f71a336296a66c7d", "potato2-6efe06781fb8cd5a" ] }, { "key": "P09-20240624-160737:6b2a49f09b017a368c9c871cd4e6c5057af58649", "video_id": "P09-20240624-160737", "question": "What is the relative direction of the microwave if the stove is 12 o'clock and the refrigerator is 3 o'clock?", "answer_choice_0": "2 o'clock", "answer_choice_1": "4 o'clock", "answer_choice_2": "6 o'clock", "answer_choice_3": "5 o'clock", "answer_choice_4": "3 o'clock", "answer_id": 0, "answer": "2 o'clock", "question_type": [ "Egocentric & Spatial Perception", "Numerical Reasoning", "Temporal Reasoning" ], "objects_of_interest": [ "napkin1-ff3dfc6bb3a6dbb3", "napkin2-d768b4e4c7ab80ce", "towel1-1826f09bf065f800", "towel2-0f7b78898e80b33e", "yellow cloth-3e3aec729eecf1a7", "packet of skewers-7da002e549e3efcd", "basket of vegetables-32eb8570f99de8d7", "bowl of onions-b44f1a1f51f58b6b", "bottle of garlic-a540402b60ac255d", "pyrex bowl of meat-3734941861dc436e", "tomato-d0436d2afcc2a187", "chopping board-bd1354ef2860fe76", "knife-8e82ae95af60de12", "tomato2-202aa5f778a18bfc", "tomato3-8babe98f1ad58c22", "cap of bottle-89b397c6223669e3", "bottle of oil-9ea52ecb2a41802b", "bottle of salt-148f7396690107d6", "skewer1-1ad69c6ab1e02769", "onion-56122b1b5acc2eb4", "skewer3-f28fa2dd50f318ef", "skewer2-5bf4594e1353dff0", "skewer4-833ca96f3f4d0492", "tomato4-c670150372b4324a", "baking tray1-eb0d57521614950e", "baking tray2-99f5c911c99fa6c8", "baking tray3-6110605e2d0908c3", "baking parchment's carton-20c32e1df5944c28", "roll of baking parchment-d244461079c7b6c5", "torn baking parchment-8be4476669058797", "excess parchment-717db5d2509ccbe4", "oven tray-f3f0ebf9fd65e098", "bottle of washing up liquid-44aeb1fefaa03368", "sponge-01e3762820a17dfb", "saucepan-11a865104bd8bdad", "saucepan2-72fbcfb9ca70e9bb", "saucepan3-da7e2a48fafd6cfd", "cardboard-e34b1f296597796a", "scale-17b7f3a61816d772", "bag of potato-d2cf2472fe360ea8", "potato-f71a336296a66c7d", "potato2-6efe06781fb8cd5a", "basket2-118a29274c2fca90", "potato3-ecffa1957a89f9d9", "spoon2-5a8c3cd220ea5c8d", "fork-38a431cd7f5aa31c", "steaming rack-b6893992093591c5", "metal piece-b804c225ecc10995", "sieve-5e4e73504f1aff54", "plate-f447e2256500b987", "potato peeler-79adcf2424e50c7b", "spice basket-29ad5a518a1ea133", "spice jar-744fcd8850a35cf4", "spice jar2-d8fc27de9da84572", "spice jar3-56649558910aa1d8", "spice jar4-c5aae489e1c6a207", "spice jar5-6d3421f562fa3a01", "spice jar6-a74bfbfa839be1cd", "spice jar7-2fd58604a256d4fe", "spice jar8-5c9b92f38b7774be", "spice jar9-0e7da638db51b19b", "spice jar10-510b5587f260783b", "spice jar11-0bde0f593b85cd75", "carton spice-9eb3b9027a93c87b", "spice bag-7815399009270245", "spice jar12-eef5cd1baf1b2083", "spice bag2-2f295ed12183a449", "spice jar13-91c136399f59aea3", "spoon3-ed07a45cdd3f18b9", "lid-35b1473664ad2387", "lid2-48e778d8c6502cb3", "bowl-da95a1ef71dace7c", "plate2-a81bd877900165d4", "towel3-2d3d71d35e1ccde8", "food waste bag-5634d3a9f000b117", "saucepan4-2e4068543b09a822", "bottle of soap-42ec72a55d1e381f", "utensil3-e17a604d9b48146a", "wooden spoon-f89d3dae90fd1a2e", "keepcup-e90eb9b3f70e2da5", "ceramic mug-af3c79e0d7cc505c", "white mug-95392472c631b1d0", "mug3-642e39679469a03c", "napkin3-d2784a131878eaab" ], "reasoning": "1. At 00:00, the stove is straight ahead in the kitchen.\n2. Its position is 12 o'clock.\n3. At 00:50, the user turns right and walks to the refrigerator .\n4. The microwave is directly to the left of the refrigerator on the counter .\n5. So, the microwave is at 2 o'clock if the refrigerator is at 3 o'clock and the stove is 12 o'clock .", "referenced_object_masks": [] }, { "key": "P09-20240624-160737:791155c79ef42d0c4522d1389d48ae235270dcd9", "video_id": "P09-20240624-160737", "question": "Of all the items I scrub with the sponge in the final two minutes of the video, what percentage of them, rounded to the nearest whole number, are made primarily or completely of glass?", "answer_choice_0": "27%", "answer_choice_1": "14%", "answer_choice_2": "29%", "answer_choice_3": "72%", "answer_choice_4": "54%", "answer_id": 2, "answer": "29%", "question_type": [ "Action Recognition", "Numerical Reasoning", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "napkin1-ff3dfc6bb3a6dbb3", "napkin2-d768b4e4c7ab80ce", "towel1-1826f09bf065f800", "towel2-0f7b78898e80b33e", "yellow cloth-3e3aec729eecf1a7", "packet of skewers-7da002e549e3efcd", "basket of vegetables-32eb8570f99de8d7", "bowl of onions-b44f1a1f51f58b6b", "bottle of garlic-a540402b60ac255d", "pyrex bowl of meat-3734941861dc436e", "tomato-d0436d2afcc2a187", "chopping board-bd1354ef2860fe76", "knife-8e82ae95af60de12", "tomato2-202aa5f778a18bfc", "tomato3-8babe98f1ad58c22", "cap of bottle-89b397c6223669e3", "bottle of oil-9ea52ecb2a41802b", "bottle of salt-148f7396690107d6", "skewer1-1ad69c6ab1e02769", "onion-56122b1b5acc2eb4", "skewer3-f28fa2dd50f318ef", "skewer2-5bf4594e1353dff0", "skewer4-833ca96f3f4d0492", "tomato4-c670150372b4324a", "baking tray1-eb0d57521614950e", "baking tray2-99f5c911c99fa6c8", "baking tray3-6110605e2d0908c3", "baking parchment's carton-20c32e1df5944c28", "roll of baking parchment-d244461079c7b6c5", "torn baking parchment-8be4476669058797", "excess parchment-717db5d2509ccbe4", "oven tray-f3f0ebf9fd65e098", "bottle of washing up liquid-44aeb1fefaa03368", "sponge-01e3762820a17dfb", "saucepan-11a865104bd8bdad", "saucepan2-72fbcfb9ca70e9bb", "saucepan3-da7e2a48fafd6cfd", "cardboard-e34b1f296597796a", "scale-17b7f3a61816d772", "bag of potato-d2cf2472fe360ea8", "potato-f71a336296a66c7d", "potato2-6efe06781fb8cd5a", "basket2-118a29274c2fca90", "potato3-ecffa1957a89f9d9", "spoon2-5a8c3cd220ea5c8d", "fork-38a431cd7f5aa31c", "steaming rack-b6893992093591c5", "metal piece-b804c225ecc10995", "sieve-5e4e73504f1aff54", "plate-f447e2256500b987", "potato peeler-79adcf2424e50c7b", "spice basket-29ad5a518a1ea133", "spice jar-744fcd8850a35cf4", "spice jar2-d8fc27de9da84572", "spice jar3-56649558910aa1d8", "spice jar4-c5aae489e1c6a207", "spice jar5-6d3421f562fa3a01", "spice jar6-a74bfbfa839be1cd", "spice jar7-2fd58604a256d4fe", "spice jar8-5c9b92f38b7774be", "spice jar9-0e7da638db51b19b", "spice jar10-510b5587f260783b", "spice jar11-0bde0f593b85cd75", "carton spice-9eb3b9027a93c87b", "spice bag-7815399009270245", "spice jar12-eef5cd1baf1b2083", "spice bag2-2f295ed12183a449", "spice jar13-91c136399f59aea3", "spoon3-ed07a45cdd3f18b9", "lid-35b1473664ad2387", "lid2-48e778d8c6502cb3", "bowl-da95a1ef71dace7c", "plate2-a81bd877900165d4", "towel3-2d3d71d35e1ccde8", "food waste bag-5634d3a9f000b117", "saucepan4-2e4068543b09a822", "bottle of soap-42ec72a55d1e381f", "utensil3-e17a604d9b48146a", "wooden spoon-f89d3dae90fd1a2e", "keepcup-e90eb9b3f70e2da5", "ceramic mug-af3c79e0d7cc505c", "white mug-95392472c631b1d0", "mug3-642e39679469a03c", "napkin3-d2784a131878eaab" ], "reasoning": "1. I first determine the final two minutes of the video to watch as my target span - since the video is 35:56 long, I watch from 33:56-35:56.\n2. I watch the video from 33:56 on, observing the objects that the user scrubs with the sponge (sponge-01e3762820a17dfb) - I observe the first, second, and third of these immediately - a fork and two spoons (OBJ, utensil3-e17a604d9b48146a) are washed by the user from 33:56-34:04, and then put away. I note that all three are made of metal, and count them as the first 3 in the total of items scrubbed by the sponge (sponge-01e3762820a17dfb).\n3. I observe the next item, a black plastic cup being scrubbed by the user with the sponge (sponge-01e3762820a17dfb) from 34:27-34:44. I note that it is made of plastic, and count it as #4 of items scrubbed by the sponge (sponge-01e3762820a17dfb).\n4. Next, at 34:45, I observe the user scrub a glass cup with the sponge (sponge-01e3762820a17dfb) until 34:49. I note that the object makes 1 glass item, and the 5th total item scrubbed by the sponge (sponge-01e3762820a17dfb).\n5. From 34:49-35:03, I observe the user scrub a ceramic mug with the sponge (sponge-01e3762820a17dfb), wash it out, and place it in the dish-drying rack. I note that this makes the 6th total item scrubbed by the sponge.\n6. From 35:40-35:49, I watch the user scrub a glass bowl with the sponge (sponge-01e3762820a17dfb). I note that this is item #2 made of glass, and item #7 of total items scrubbed.\n7. I watch the remainder of the video and observe no more items scrubbed, and therefore end with a value of 7 total items scrubbed, 2 of which were made of glass. To determine the percentage made of glass, I do the equation 2/7 and obtain, to the nearest whole number, a percentage of 29%, making B the correct answer.", "referenced_object_masks": [ "utensil3-e17a604d9b48146a", "sponge-01e3762820a17dfb" ] }, { "key": "P09-20240624-160737:8e43d8b0e1557dcd5c4770689e9562eb5ff22cb0", "video_id": "P09-20240624-160737", "question": "Which of the following is not an action I perform between finishing creation of the skewers, and placing them in the oven for the first time?", "answer_choice_0": "Turn a dial on the oven", "answer_choice_1": "Wash hands", "answer_choice_2": "Touch a muffin tin", "answer_choice_3": "Place knife in sink", "answer_choice_4": "Open a drawer", "answer_id": 0, "answer": "Turn a dial on the oven", "question_type": [ "Action Recognition", "Object Recognition", "Event Occurence", "Temporal Reasoning" ], "objects_of_interest": [ "napkin1-ff3dfc6bb3a6dbb3", "napkin2-d768b4e4c7ab80ce", "towel1-1826f09bf065f800", "towel2-0f7b78898e80b33e", "yellow cloth-3e3aec729eecf1a7", "packet of skewers-7da002e549e3efcd", "basket of vegetables-32eb8570f99de8d7", "bowl of onions-b44f1a1f51f58b6b", "bottle of garlic-a540402b60ac255d", "pyrex bowl of meat-3734941861dc436e", "tomato-d0436d2afcc2a187", "chopping board-bd1354ef2860fe76", "knife-8e82ae95af60de12", "tomato2-202aa5f778a18bfc", "tomato3-8babe98f1ad58c22", "cap of bottle-89b397c6223669e3", "bottle of oil-9ea52ecb2a41802b", "bottle of salt-148f7396690107d6", "skewer1-1ad69c6ab1e02769", "onion-56122b1b5acc2eb4", "skewer3-f28fa2dd50f318ef", "skewer2-5bf4594e1353dff0", "skewer4-833ca96f3f4d0492", "tomato4-c670150372b4324a", "baking tray1-eb0d57521614950e", "baking tray2-99f5c911c99fa6c8", "baking tray3-6110605e2d0908c3", "baking parchment's carton-20c32e1df5944c28", "roll of baking parchment-d244461079c7b6c5", "torn baking parchment-8be4476669058797", "excess parchment-717db5d2509ccbe4", "oven tray-f3f0ebf9fd65e098", "bottle of washing up liquid-44aeb1fefaa03368", "sponge-01e3762820a17dfb", "saucepan-11a865104bd8bdad", "saucepan2-72fbcfb9ca70e9bb", "saucepan3-da7e2a48fafd6cfd", "cardboard-e34b1f296597796a", "scale-17b7f3a61816d772", "bag of potato-d2cf2472fe360ea8", "potato-f71a336296a66c7d", "potato2-6efe06781fb8cd5a", "basket2-118a29274c2fca90", "potato3-ecffa1957a89f9d9", "spoon2-5a8c3cd220ea5c8d", "fork-38a431cd7f5aa31c", "steaming rack-b6893992093591c5", "metal piece-b804c225ecc10995", "sieve-5e4e73504f1aff54", "plate-f447e2256500b987", "potato peeler-79adcf2424e50c7b", "spice basket-29ad5a518a1ea133", "spice jar-744fcd8850a35cf4", "spice jar2-d8fc27de9da84572", "spice jar3-56649558910aa1d8", "spice jar4-c5aae489e1c6a207", "spice jar5-6d3421f562fa3a01", "spice jar6-a74bfbfa839be1cd", "spice jar7-2fd58604a256d4fe", "spice jar8-5c9b92f38b7774be", "spice jar9-0e7da638db51b19b", "spice jar10-510b5587f260783b", "spice jar11-0bde0f593b85cd75", "carton spice-9eb3b9027a93c87b", "spice bag-7815399009270245", "spice jar12-eef5cd1baf1b2083", "spice bag2-2f295ed12183a449", "spice jar13-91c136399f59aea3", "spoon3-ed07a45cdd3f18b9", "lid-35b1473664ad2387", "lid2-48e778d8c6502cb3", "bowl-da95a1ef71dace7c", "plate2-a81bd877900165d4", "towel3-2d3d71d35e1ccde8", "food waste bag-5634d3a9f000b117", "saucepan4-2e4068543b09a822", "bottle of soap-42ec72a55d1e381f", "utensil3-e17a604d9b48146a", "wooden spoon-f89d3dae90fd1a2e", "keepcup-e90eb9b3f70e2da5", "ceramic mug-af3c79e0d7cc505c", "white mug-95392472c631b1d0", "mug3-642e39679469a03c", "napkin3-d2784a131878eaab" ], "reasoning": "1. To find this answer, I must determine when the user finishes constructing the skewers - I watch the user construct the skewers, starting with the first onion chunk on the first skewer (skewer3-f28fa2dd50f318ef) at 05:05, and ending with a final afterthought of a small piece of tomato on the second skewer (skewer2-5bf4594e1353dff0) from 08:52-08:55. I observe no more skewering after this, and so I use 08:55 as my start time for the target timespan.\n2. I then watch the video until I see the user put the pan (baking tray2-99f5c911c99fa6c8) of skewers (skewer3-f28fa2dd50f318ef, skewer2-5bf4594e1353dff0, skewer4-833ca96f3f4d0492) into the oven for the first time from 10:14-10:18. Therefore, I find that my end time for the target timespan should be 10:18, and that the window of time I should pay attention to to search for actions is from 08:55-10:18. \n3. I watch the video from 08:55-10:18, watching for actions as described by the question choices - I observe the first of these at 08:56, when the user places a knife (knife-8e82ae95af60de12) in the sink, eliminating B as an option.\n4. I continue watching, and observe the next action from the answer choices from 08:58-9:02, when the user washes their hands. This eliminates A as an option.\n5. I continue watching, and observe the next action from the answer choices at 09:19, when the user lifts up a muffin tin (baking tray1-eb0d57521614950e) in the bottom cupboard, touching it with their thumb - this eliminates option C as an answer.\n6. I continue watching until I observe the user open a drawer at 09:40, eliminating E as an option.\n7. With only answer choice D left to find, I watch the rest of the span to ensure that I don't see the user turn a dial on the oven - the user does do this at 10:32, but only after removing the skewer pan (baking tray2-99f5c911c99fa6c8) from the oven after putting it in for the first time. Since the action falls outside the target window, D is the correct answer.", "referenced_object_masks": [ "baking tray2-99f5c911c99fa6c8", "skewer3-f28fa2dd50f318ef", "skewer2-5bf4594e1353dff0", "skewer4-833ca96f3f4d0492", "baking tray1-eb0d57521614950e", "knife-8e82ae95af60de12" ] }, { "key": "P09-20240624-160737:afd8c108334a059f76b957352d79845b14df8a62", "video_id": "P09-20240624-160737", "question": "What is the 5th and 6th item I put on the 3rd skewer?", "answer_choice_0": "pepper and meat", "answer_choice_1": "meat and pepper", "answer_choice_2": "meat and tomato", "answer_choice_3": "tomato and meat", "answer_choice_4": "tomato and pepper", "answer_id": 2, "answer": "meat and tomato", "question_type": [ "Event Occurence", "Object Recognition", "Counting", "Action Recognition" ], "objects_of_interest": [ "napkin1-ff3dfc6bb3a6dbb3", "napkin2-d768b4e4c7ab80ce", "towel1-1826f09bf065f800", "towel2-0f7b78898e80b33e", "yellow cloth-3e3aec729eecf1a7", "packet of skewers-7da002e549e3efcd", "basket of vegetables-32eb8570f99de8d7", "bowl of onions-b44f1a1f51f58b6b", "bottle of garlic-a540402b60ac255d", "pyrex bowl of meat-3734941861dc436e", "tomato-d0436d2afcc2a187", "chopping board-bd1354ef2860fe76", "knife-8e82ae95af60de12", "tomato2-202aa5f778a18bfc", "tomato3-8babe98f1ad58c22", "cap of bottle-89b397c6223669e3", "bottle of oil-9ea52ecb2a41802b", "bottle of salt-148f7396690107d6", "skewer1-1ad69c6ab1e02769", "onion-56122b1b5acc2eb4", "skewer3-f28fa2dd50f318ef", "skewer2-5bf4594e1353dff0", "skewer4-833ca96f3f4d0492", "tomato4-c670150372b4324a", "baking tray1-eb0d57521614950e", "baking tray2-99f5c911c99fa6c8", "baking tray3-6110605e2d0908c3", "baking parchment's carton-20c32e1df5944c28", "roll of baking parchment-d244461079c7b6c5", "torn baking parchment-8be4476669058797", "excess parchment-717db5d2509ccbe4", "oven tray-f3f0ebf9fd65e098", "bottle of washing up liquid-44aeb1fefaa03368", "sponge-01e3762820a17dfb", "saucepan-11a865104bd8bdad", "saucepan2-72fbcfb9ca70e9bb", "saucepan3-da7e2a48fafd6cfd", "cardboard-e34b1f296597796a", "scale-17b7f3a61816d772", "bag of potato-d2cf2472fe360ea8", "potato-f71a336296a66c7d", "potato2-6efe06781fb8cd5a", "basket2-118a29274c2fca90", "potato3-ecffa1957a89f9d9", "spoon2-5a8c3cd220ea5c8d", "fork-38a431cd7f5aa31c", "steaming rack-b6893992093591c5", "metal piece-b804c225ecc10995", "sieve-5e4e73504f1aff54", "plate-f447e2256500b987", "potato peeler-79adcf2424e50c7b", "spice basket-29ad5a518a1ea133", "spice jar-744fcd8850a35cf4", "spice jar2-d8fc27de9da84572", "spice jar3-56649558910aa1d8", "spice jar4-c5aae489e1c6a207", "spice jar5-6d3421f562fa3a01", "spice jar6-a74bfbfa839be1cd", "spice jar7-2fd58604a256d4fe", "spice jar8-5c9b92f38b7774be", "spice jar9-0e7da638db51b19b", "spice jar10-510b5587f260783b", "spice jar11-0bde0f593b85cd75", "carton spice-9eb3b9027a93c87b", "spice bag-7815399009270245", "spice jar12-eef5cd1baf1b2083", "spice bag2-2f295ed12183a449", "spice jar13-91c136399f59aea3", "spoon3-ed07a45cdd3f18b9", "lid-35b1473664ad2387", "lid2-48e778d8c6502cb3", "bowl-da95a1ef71dace7c", "plate2-a81bd877900165d4", "towel3-2d3d71d35e1ccde8", "food waste bag-5634d3a9f000b117", "saucepan4-2e4068543b09a822", "bottle of soap-42ec72a55d1e381f", "utensil3-e17a604d9b48146a", "wooden spoon-f89d3dae90fd1a2e", "keepcup-e90eb9b3f70e2da5", "ceramic mug-af3c79e0d7cc505c", "white mug-95392472c631b1d0", "mug3-642e39679469a03c", "napkin3-d2784a131878eaab" ], "reasoning": "1. I noticed the user begin to work on the third skewer (skewer3-f28fa2dd50f318ef) at 7:11.\n2.I noticed the user first item put on the skewer was a piece of onion at 7:15.\n3. I noticed the user second item put on the skewer was a piece of meat at 7:23.\n4. I noticed the user third item put on the skewer was a piece of meat at 7:26.\n5. I noticed the user fourth item put on the skewer was a piece of tomato at 7:28.\n6. I noticed the user fifth item put on the skewer was a piece of red pepper at 7:47.\n7. I noticed the user fifth item put on the skewer was a piece of meat at 7:56.", "referenced_object_masks": [ "skewer3-f28fa2dd50f318ef" ] }, { "key": "P09-20240624-160737:ea47ce2f6b46dd6c2644108299d5bb537f43b96d", "video_id": "P09-20240624-160737", "question": "What is the first thing I touch after putting the smaller pan used to make homemade french fries on the back burner of the stove?", "answer_choice_0": "Cabinet handle", "answer_choice_1": "Oil", "answer_choice_2": "Fork", "answer_choice_3": "Stove knob", "answer_choice_4": "Plate", "answer_id": 0, "answer": "Cabinet handle", "question_type": [ "Action Recognition", "Event Occurence", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "napkin1-ff3dfc6bb3a6dbb3", "napkin2-d768b4e4c7ab80ce", "towel1-1826f09bf065f800", "towel2-0f7b78898e80b33e", "yellow cloth-3e3aec729eecf1a7", "packet of skewers-7da002e549e3efcd", "basket of vegetables-32eb8570f99de8d7", "bowl of onions-b44f1a1f51f58b6b", "bottle of garlic-a540402b60ac255d", "pyrex bowl of meat-3734941861dc436e", "tomato-d0436d2afcc2a187", "chopping board-bd1354ef2860fe76", "knife-8e82ae95af60de12", "tomato2-202aa5f778a18bfc", "tomato3-8babe98f1ad58c22", "cap of bottle-89b397c6223669e3", "bottle of oil-9ea52ecb2a41802b", "bottle of salt-148f7396690107d6", "skewer1-1ad69c6ab1e02769", "onion-56122b1b5acc2eb4", "skewer3-f28fa2dd50f318ef", "skewer2-5bf4594e1353dff0", "skewer4-833ca96f3f4d0492", "tomato4-c670150372b4324a", "baking tray1-eb0d57521614950e", "baking tray2-99f5c911c99fa6c8", "baking tray3-6110605e2d0908c3", "baking parchment's carton-20c32e1df5944c28", "roll of baking parchment-d244461079c7b6c5", "torn baking parchment-8be4476669058797", "excess parchment-717db5d2509ccbe4", "oven tray-f3f0ebf9fd65e098", "bottle of washing up liquid-44aeb1fefaa03368", "sponge-01e3762820a17dfb", "saucepan-11a865104bd8bdad", "saucepan2-72fbcfb9ca70e9bb", "saucepan3-da7e2a48fafd6cfd", "cardboard-e34b1f296597796a", "scale-17b7f3a61816d772", "bag of potato-d2cf2472fe360ea8", "potato-f71a336296a66c7d", "potato2-6efe06781fb8cd5a", "basket2-118a29274c2fca90", "potato3-ecffa1957a89f9d9", "spoon2-5a8c3cd220ea5c8d", "fork-38a431cd7f5aa31c", "steaming rack-b6893992093591c5", "metal piece-b804c225ecc10995", "sieve-5e4e73504f1aff54", "plate-f447e2256500b987", "potato peeler-79adcf2424e50c7b", "spice basket-29ad5a518a1ea133", "spice jar-744fcd8850a35cf4", "spice jar2-d8fc27de9da84572", "spice jar3-56649558910aa1d8", "spice jar4-c5aae489e1c6a207", "spice jar5-6d3421f562fa3a01", "spice jar6-a74bfbfa839be1cd", "spice jar7-2fd58604a256d4fe", "spice jar8-5c9b92f38b7774be", "spice jar9-0e7da638db51b19b", "spice jar10-510b5587f260783b", "spice jar11-0bde0f593b85cd75", "carton spice-9eb3b9027a93c87b", "spice bag-7815399009270245", "spice jar12-eef5cd1baf1b2083", "spice bag2-2f295ed12183a449", "spice jar13-91c136399f59aea3", "spoon3-ed07a45cdd3f18b9", "lid-35b1473664ad2387", "lid2-48e778d8c6502cb3", "bowl-da95a1ef71dace7c", "plate2-a81bd877900165d4", "towel3-2d3d71d35e1ccde8", "food waste bag-5634d3a9f000b117", "saucepan4-2e4068543b09a822", "bottle of soap-42ec72a55d1e381f", "utensil3-e17a604d9b48146a", "wooden spoon-f89d3dae90fd1a2e", "keepcup-e90eb9b3f70e2da5", "ceramic mug-af3c79e0d7cc505c", "white mug-95392472c631b1d0", "mug3-642e39679469a03c", "napkin3-d2784a131878eaab" ], "reasoning": "1. I watched the whole video and saw the user make 2 batches of french fries starting at 19:32.\n2. At 30:23, the user puts the small pan (saucepan-11a865104bd8bdad) of oil on the back burner .\n3. At 30:25, the user moves right and clutches the handle of the cabinet under the counter to get a larger, red pan (saucepan2-72fbcfb9ca70e9bb).", "referenced_object_masks": [ "saucepan-11a865104bd8bdad", "saucepan2-72fbcfb9ca70e9bb" ] }, { "key": "P09-20240624-160737:f6e087d5a19e496a7e9d09d11ce94fedffca55ee", "video_id": "P09-20240624-160737", "question": "If I multiply the number of magnets on the refrigerator door by the number of times I pour water out of the bowl with the chicken in it and then square this number, what number do I get?", "answer_choice_0": "81", "answer_choice_1": "5184", "answer_choice_2": "4900", "answer_choice_3": "144", "answer_choice_4": "72", "answer_id": 1, "answer": "5184", "question_type": [ "Action Recognition", "Counting", "Event Occurence", "Numerical Reasoning", "Temporal Reasoning" ], "objects_of_interest": [ "napkin1-ff3dfc6bb3a6dbb3", "napkin2-d768b4e4c7ab80ce", "towel1-1826f09bf065f800", "towel2-0f7b78898e80b33e", "yellow cloth-3e3aec729eecf1a7", "packet of skewers-7da002e549e3efcd", "basket of vegetables-32eb8570f99de8d7", "bowl of onions-b44f1a1f51f58b6b", "bottle of garlic-a540402b60ac255d", "pyrex bowl of meat-3734941861dc436e", "tomato-d0436d2afcc2a187", "chopping board-bd1354ef2860fe76", "knife-8e82ae95af60de12", "tomato2-202aa5f778a18bfc", "tomato3-8babe98f1ad58c22", "cap of bottle-89b397c6223669e3", "bottle of oil-9ea52ecb2a41802b", "bottle of salt-148f7396690107d6", "skewer1-1ad69c6ab1e02769", "onion-56122b1b5acc2eb4", "skewer3-f28fa2dd50f318ef", "skewer2-5bf4594e1353dff0", "skewer4-833ca96f3f4d0492", "tomato4-c670150372b4324a", "baking tray1-eb0d57521614950e", "baking tray2-99f5c911c99fa6c8", "baking tray3-6110605e2d0908c3", "baking parchment's carton-20c32e1df5944c28", "roll of baking parchment-d244461079c7b6c5", "torn baking parchment-8be4476669058797", "excess parchment-717db5d2509ccbe4", "oven tray-f3f0ebf9fd65e098", "bottle of washing up liquid-44aeb1fefaa03368", "sponge-01e3762820a17dfb", "saucepan-11a865104bd8bdad", "saucepan2-72fbcfb9ca70e9bb", "saucepan3-da7e2a48fafd6cfd", "cardboard-e34b1f296597796a", "scale-17b7f3a61816d772", "bag of potato-d2cf2472fe360ea8", "potato-f71a336296a66c7d", "potato2-6efe06781fb8cd5a", "basket2-118a29274c2fca90", "potato3-ecffa1957a89f9d9", "spoon2-5a8c3cd220ea5c8d", "fork-38a431cd7f5aa31c", "steaming rack-b6893992093591c5", "metal piece-b804c225ecc10995", "sieve-5e4e73504f1aff54", "plate-f447e2256500b987", "potato peeler-79adcf2424e50c7b", "spice basket-29ad5a518a1ea133", "spice jar-744fcd8850a35cf4", "spice jar2-d8fc27de9da84572", "spice jar3-56649558910aa1d8", "spice jar4-c5aae489e1c6a207", "spice jar5-6d3421f562fa3a01", "spice jar6-a74bfbfa839be1cd", "spice jar7-2fd58604a256d4fe", "spice jar8-5c9b92f38b7774be", "spice jar9-0e7da638db51b19b", "spice jar10-510b5587f260783b", "spice jar11-0bde0f593b85cd75", "carton spice-9eb3b9027a93c87b", "spice bag-7815399009270245", "spice jar12-eef5cd1baf1b2083", "spice bag2-2f295ed12183a449", "spice jar13-91c136399f59aea3", "spoon3-ed07a45cdd3f18b9", "lid-35b1473664ad2387", "lid2-48e778d8c6502cb3", "bowl-da95a1ef71dace7c", "plate2-a81bd877900165d4", "towel3-2d3d71d35e1ccde8", "food waste bag-5634d3a9f000b117", "saucepan4-2e4068543b09a822", "bottle of soap-42ec72a55d1e381f", "utensil3-e17a604d9b48146a", "wooden spoon-f89d3dae90fd1a2e", "keepcup-e90eb9b3f70e2da5", "ceramic mug-af3c79e0d7cc505c", "white mug-95392472c631b1d0", "mug3-642e39679469a03c", "napkin3-d2784a131878eaab" ], "reasoning": "1. I watched the whole video and saw the refrigerator door at 01:06.\n2. At 01:06, the refrigerator door has 9 magnets on it.\n3. The user pours water out of the bowl with the chicken (pyrex bowl of meat-3734941861dc436e) at 01:18, 01:26, 01:31, 01:41, 01:45, 01:52, 03:07 and 03:12.\n4. This is 8 times the user pours water out of the bowl with chicken (pyrex bowl of meat-3734941861dc436e).\n5. I multiply 9 by 8 to get 72.\n6. 72 squared is 5184.", "referenced_object_masks": [ "pyrex bowl of meat-3734941861dc436e" ] }, { "key": "P09-20240624-165332:03ff503aed190fc3a7401a73b02c1972f9cd7c16", "video_id": "P09-20240624-165332", "question": "What is the second item I touch with my left hand after dumping the oil out of the pan into the bowl?", "answer_choice_0": "Plate", "answer_choice_1": "Towel", "answer_choice_2": "Kitchen faucet", "answer_choice_3": "Sponge", "answer_choice_4": "Dish soap", "answer_id": 4, "answer": "Dish soap", "question_type": [ "Action Recognition", "Counting", "Object Recognition" ], "objects_of_interest": [ "timer-cd2fbe9a034f7d09", "tea towel1-ac78df68cbdab8a2", "saucepan-66773cb00653b78c", "fork1-fe15724041f1bfdf", "plate1-610ad85dcf06cbb3", "sponge-6589531b8f2207a8", "bottle of washing up liquid-06d3122337b69260", "glass bowl-eadbfd4406724438", "saucepan2-8f34836c640b69ff", "mug1-7d29d462b78518bd", "tea towel2-54a395259a186c98", "knife1-8f2b482b5d6135e0", "spoon1-c97e9629d3e959a1", "knife2-eabb2277c8d6fb20", "chopping board-d5b53bb340f96c04", "bowl1-b51721974530e201", "tea towel3-f99b8cfee312815a", "tea towel4-77e57c29641363f2", "skewer1-8843cf42e6ff0788", "skewer2-99c7480cab0195c4", "skewer3-f673fabf71dddbdc", "paper-ad891827e812c1ef", "kettle-1ae788643f0d5c9b", "oven glove-9e6f84e378effde0", "baking tray-86f1c87864a56413", "skewers-1c30ae739807c18d", "parchment paper-31fbdcaf19b9322d", "piece of parchment-581e52185e698d4a", "plate2-2536e34f196dd06d", "mug2-01ce9e39070fa4fa", "cup1-f623543a1b5e5f7a", "bottle of milk-4d04265827e36e6a", "jar of sugar-02b40604b89e4992", "bag of tea-1bd7a6d1a6e75643", "tea bag-b7114ac33c412f4f", "spoon2-b821961cc3a66bf2", "metal grid-8c0be21488a43c2e", "coffee pot-1a69ab621024ef98", "lid of jar of sugar-10bd76e9ae392e60" ], "reasoning": "1. I watched the video to see when the user dumps the oil in the pan (saucepan-66773cb00653b78c) to a bowl (bowl1-b51721974530e201), noting that this occurs from 18:57-19:02.\n2. From 19:06-19:07, I saw the user wash the pan (saucepan-66773cb00653b78c) while grabbing the kitchen faucet with their left hand and the kitchen sponge (sponge-6589531b8f2207a8) with their right. This makes the kitchen faucet the first item the user touched with their left hand.\n3. While still holding the sponge (sponge-6589531b8f2207a8) in their right hand, the user crosses their left hand over to grab the dish soap (bottle of washing up liquid-06d3122337b69260) from 19:08-19:10.\n4. Thus, the dish soap (bottle of washing up liquid-06d3122337b69260) is the second item the user grabbed with their left hand after dumping the oil .", "referenced_object_masks": [ "sponge-6589531b8f2207a8", "bowl1-b51721974530e201", "saucepan-66773cb00653b78c", "bottle of washing up liquid-06d3122337b69260" ] }, { "key": "P09-20240624-165332:327aa9c919405d8af7d23539a24f1e0cea87a3f6", "video_id": "P09-20240624-165332", "question": "When I put my meal down at the counter, in which direction would I turn to face the last location of the milk?", "answer_choice_0": "Back right", "answer_choice_1": "Above", "answer_choice_2": "Directly ahead", "answer_choice_3": "By my feet", "answer_choice_4": "Back left", "answer_id": 4, "answer": "Back left", "question_type": [ "Object Recognition", "Action Recognition", "Temporal Reasoning", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "timer-cd2fbe9a034f7d09", "tea towel1-ac78df68cbdab8a2", "saucepan-66773cb00653b78c", "fork1-fe15724041f1bfdf", "plate1-610ad85dcf06cbb3", "sponge-6589531b8f2207a8", "bottle of washing up liquid-06d3122337b69260", "glass bowl-eadbfd4406724438", "saucepan2-8f34836c640b69ff", "mug1-7d29d462b78518bd", "tea towel2-54a395259a186c98", "knife1-8f2b482b5d6135e0", "spoon1-c97e9629d3e959a1", "knife2-eabb2277c8d6fb20", "chopping board-d5b53bb340f96c04", "bowl1-b51721974530e201", "tea towel3-f99b8cfee312815a", "tea towel4-77e57c29641363f2", "skewer1-8843cf42e6ff0788", "skewer2-99c7480cab0195c4", "skewer3-f673fabf71dddbdc", "paper-ad891827e812c1ef", "kettle-1ae788643f0d5c9b", "oven glove-9e6f84e378effde0", "baking tray-86f1c87864a56413", "skewers-1c30ae739807c18d", "parchment paper-31fbdcaf19b9322d", "piece of parchment-581e52185e698d4a", "plate2-2536e34f196dd06d", "mug2-01ce9e39070fa4fa", "cup1-f623543a1b5e5f7a", "bottle of milk-4d04265827e36e6a", "jar of sugar-02b40604b89e4992", "bag of tea-1bd7a6d1a6e75643", "tea bag-b7114ac33c412f4f", "spoon2-b821961cc3a66bf2", "metal grid-8c0be21488a43c2e", "coffee pot-1a69ab621024ef98", "lid of jar of sugar-10bd76e9ae392e60" ], "reasoning": "1. I watch the video and observe the user stand to the right of the stovetop and turn right to put the milk (bottle of milk-4d04265827e36e6a) on the fridge door's bottom shelf between 22:56 and 23:00.\n2. I watch the video from 23:00 to the end and observe the user turn left at the counter to the right of the stove and walk forward until reaching the counter at 23:36.\n3. I observe the user place their mug and plate on the counter at 23:37.\n4. I determine that at 23:37, the user would need to turn to their back left to see the milk's (bottle of milk-4d04265827e36e6a) location.", "referenced_object_masks": [ "bottle of milk-4d04265827e36e6a" ] }, { "key": "P09-20240624-165332:47e0753fc03044b9263d55232e5fde584b72fe83", "video_id": "P09-20240624-165332", "question": "Before hanging the yellow towel on the handle next to the stove, how many times do I fold it?", "answer_choice_0": "4", "answer_choice_1": "5", "answer_choice_2": "1", "answer_choice_3": "2", "answer_choice_4": "3", "answer_id": 4, "answer": "3", "question_type": [ "Object Recognition", "Temporal Reasoning", "Counting", "Action Recognition" ], "objects_of_interest": [ "timer-cd2fbe9a034f7d09", "tea towel1-ac78df68cbdab8a2", "saucepan-66773cb00653b78c", "fork1-fe15724041f1bfdf", "plate1-610ad85dcf06cbb3", "sponge-6589531b8f2207a8", "bottle of washing up liquid-06d3122337b69260", "glass bowl-eadbfd4406724438", "saucepan2-8f34836c640b69ff", "mug1-7d29d462b78518bd", "tea towel2-54a395259a186c98", "knife1-8f2b482b5d6135e0", "spoon1-c97e9629d3e959a1", "knife2-eabb2277c8d6fb20", "chopping board-d5b53bb340f96c04", "bowl1-b51721974530e201", "tea towel3-f99b8cfee312815a", "tea towel4-77e57c29641363f2", "skewer1-8843cf42e6ff0788", "skewer2-99c7480cab0195c4", "skewer3-f673fabf71dddbdc", "paper-ad891827e812c1ef", "kettle-1ae788643f0d5c9b", "oven glove-9e6f84e378effde0", "baking tray-86f1c87864a56413", "skewers-1c30ae739807c18d", "parchment paper-31fbdcaf19b9322d", "piece of parchment-581e52185e698d4a", "plate2-2536e34f196dd06d", "mug2-01ce9e39070fa4fa", "cup1-f623543a1b5e5f7a", "bottle of milk-4d04265827e36e6a", "jar of sugar-02b40604b89e4992", "bag of tea-1bd7a6d1a6e75643", "tea bag-b7114ac33c412f4f", "spoon2-b821961cc3a66bf2", "metal grid-8c0be21488a43c2e", "coffee pot-1a69ab621024ef98", "lid of jar of sugar-10bd76e9ae392e60" ], "reasoning": "1. I identify the towel (tea towel1-ac78df68cbdab8a2) in the user's right hand at 00:15 at the opened oven .\n2. At 14:59, I observe the user reach for the towel (tea towel1-ac78df68cbdab8a2) on the counter\n3. At 15:15, I observe the user hang the towel (tea towel1-ac78df68cbdab8a2) on the drawer handle to the left of the oven .\n4. Between 14:59 and 15:15, I count the user fold the towel (tea towel1-ac78df68cbdab8a2) 3 times.", "referenced_object_masks": [ "tea towel1-ac78df68cbdab8a2" ] }, { "key": "P09-20240624-165332:544bc45609ba0968e0dd13dca4377eec048b2b0e", "video_id": "P09-20240624-165332", "question": "After clearing out the sink's food catcher, just after my second visit to the sink, how many swipes of the sponge do I make while cleaning the sink?", "answer_choice_0": "90", "answer_choice_1": "50", "answer_choice_2": "10", "answer_choice_3": "20", "answer_choice_4": "70", "answer_id": 4, "answer": "70", "question_type": [ "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counting" ], "objects_of_interest": [ "timer-cd2fbe9a034f7d09", "tea towel1-ac78df68cbdab8a2", "saucepan-66773cb00653b78c", "fork1-fe15724041f1bfdf", "plate1-610ad85dcf06cbb3", "sponge-6589531b8f2207a8", "bottle of washing up liquid-06d3122337b69260", "glass bowl-eadbfd4406724438", "saucepan2-8f34836c640b69ff", "mug1-7d29d462b78518bd", "tea towel2-54a395259a186c98", "knife1-8f2b482b5d6135e0", "spoon1-c97e9629d3e959a1", "knife2-eabb2277c8d6fb20", "chopping board-d5b53bb340f96c04", "bowl1-b51721974530e201", "tea towel3-f99b8cfee312815a", "tea towel4-77e57c29641363f2", "skewer1-8843cf42e6ff0788", "skewer2-99c7480cab0195c4", "skewer3-f673fabf71dddbdc", "paper-ad891827e812c1ef", "kettle-1ae788643f0d5c9b", "oven glove-9e6f84e378effde0", "baking tray-86f1c87864a56413", "skewers-1c30ae739807c18d", "parchment paper-31fbdcaf19b9322d", "piece of parchment-581e52185e698d4a", "plate2-2536e34f196dd06d", "mug2-01ce9e39070fa4fa", "cup1-f623543a1b5e5f7a", "bottle of milk-4d04265827e36e6a", "jar of sugar-02b40604b89e4992", "bag of tea-1bd7a6d1a6e75643", "tea bag-b7114ac33c412f4f", "spoon2-b821961cc3a66bf2", "metal grid-8c0be21488a43c2e", "coffee pot-1a69ab621024ef98", "lid of jar of sugar-10bd76e9ae392e60" ], "reasoning": "1. At 15:44, I observe the user clear out the sink's food catcher with the towel (tea towel3-f99b8cfee312815a).\n2. At 16:14, I observe the user return to the sink for the first time.\n3. At 16:32, I observe the user return to the sink for the second time.\n4. At 16:44, I observe the user pick up the sponge (sponge-6589531b8f2207a8) to the right of the sink and begin cleaning.\n5. At 17:38, I observe the user leave the sink .\n6. From 16:33 to 17:38, I count the user swipe with the sponge (sponge-6589531b8f2207a8) 70 times.", "referenced_object_masks": [ "tea towel3-f99b8cfee312815a", "sponge-6589531b8f2207a8" ] }, { "key": "P09-20240624-165332:71e0f68a3dce8071690d48e4be8d8cf8d9d4cc74", "video_id": "P09-20240624-165332", "question": "After transferring the newly-cooked fries to the plate and before I reach for the oven glove, how many times do I touch the fries with a fork prong?", "answer_choice_0": "30", "answer_choice_1": "6", "answer_choice_2": "2", "answer_choice_3": "15", "answer_choice_4": "20", "answer_id": 3, "answer": "15", "question_type": [ "Object Recognition", "Action Recognition", "Temporal Reasoning", "Counting" ], "objects_of_interest": [ "timer-cd2fbe9a034f7d09", "tea towel1-ac78df68cbdab8a2", "saucepan-66773cb00653b78c", "fork1-fe15724041f1bfdf", "plate1-610ad85dcf06cbb3", "sponge-6589531b8f2207a8", "bottle of washing up liquid-06d3122337b69260", "glass bowl-eadbfd4406724438", "saucepan2-8f34836c640b69ff", "mug1-7d29d462b78518bd", "tea towel2-54a395259a186c98", "knife1-8f2b482b5d6135e0", "spoon1-c97e9629d3e959a1", "knife2-eabb2277c8d6fb20", "chopping board-d5b53bb340f96c04", "bowl1-b51721974530e201", "tea towel3-f99b8cfee312815a", "tea towel4-77e57c29641363f2", "skewer1-8843cf42e6ff0788", "skewer2-99c7480cab0195c4", "skewer3-f673fabf71dddbdc", "paper-ad891827e812c1ef", "kettle-1ae788643f0d5c9b", "oven glove-9e6f84e378effde0", "baking tray-86f1c87864a56413", "skewers-1c30ae739807c18d", "parchment paper-31fbdcaf19b9322d", "piece of parchment-581e52185e698d4a", "plate2-2536e34f196dd06d", "mug2-01ce9e39070fa4fa", "cup1-f623543a1b5e5f7a", "bottle of milk-4d04265827e36e6a", "jar of sugar-02b40604b89e4992", "bag of tea-1bd7a6d1a6e75643", "tea bag-b7114ac33c412f4f", "spoon2-b821961cc3a66bf2", "metal grid-8c0be21488a43c2e", "coffee pot-1a69ab621024ef98", "lid of jar of sugar-10bd76e9ae392e60" ], "reasoning": "1. From 09:40 to 09:49, I observe the user add the newly-cooked fries from the pan (saucepan-66773cb00653b78c) to the plate .\n2. At 10:37, I observe the user pick up the oven glove (oven glove-9e6f84e378effde0).\n3. Between 09:49 and 10:37, I count the prongs of the fork (fork1-fe15724041f1bfdf) come into contact with a fry 15 times.", "referenced_object_masks": [ "oven glove-9e6f84e378effde0", "fork1-fe15724041f1bfdf", "saucepan-66773cb00653b78c" ] }, { "key": "P09-20240624-165332:8f4b124f5b95cfb6bae289c62d2ada82e308d94f", "video_id": "P09-20240624-165332", "question": "If I reversed the pan's first 3 stove burner locations (swap near with far, left with right), and I included where the pan was originally located, what would said locations be?", "answer_choice_0": "near right, far right, far left", "answer_choice_1": "far left, near left, near right", "answer_choice_2": "near right, far right, near right", "answer_choice_3": "near left, near right, far right", "answer_choice_4": "near left, far left, near left", "answer_id": 2, "answer": "near right, far right, near right", "question_type": [ "Object Recognition", "Action Recognition", "Temporal Reasoning", "Egocentric & Spatial Perception", "Situational Awareness", "Event Occurence", "Counterfactual" ], "objects_of_interest": [ "timer-cd2fbe9a034f7d09", "tea towel1-ac78df68cbdab8a2", "saucepan-66773cb00653b78c", "fork1-fe15724041f1bfdf", "plate1-610ad85dcf06cbb3", "sponge-6589531b8f2207a8", "bottle of washing up liquid-06d3122337b69260", "glass bowl-eadbfd4406724438", "saucepan2-8f34836c640b69ff", "mug1-7d29d462b78518bd", "tea towel2-54a395259a186c98", "knife1-8f2b482b5d6135e0", "spoon1-c97e9629d3e959a1", "knife2-eabb2277c8d6fb20", "chopping board-d5b53bb340f96c04", "bowl1-b51721974530e201", "tea towel3-f99b8cfee312815a", "tea towel4-77e57c29641363f2", "skewer1-8843cf42e6ff0788", "skewer2-99c7480cab0195c4", "skewer3-f673fabf71dddbdc", "paper-ad891827e812c1ef", "kettle-1ae788643f0d5c9b", "oven glove-9e6f84e378effde0", "baking tray-86f1c87864a56413", "skewers-1c30ae739807c18d", "parchment paper-31fbdcaf19b9322d", "piece of parchment-581e52185e698d4a", "plate2-2536e34f196dd06d", "mug2-01ce9e39070fa4fa", "cup1-f623543a1b5e5f7a", "bottle of milk-4d04265827e36e6a", "jar of sugar-02b40604b89e4992", "bag of tea-1bd7a6d1a6e75643", "tea bag-b7114ac33c412f4f", "spoon2-b821961cc3a66bf2", "metal grid-8c0be21488a43c2e", "coffee pot-1a69ab621024ef98", "lid of jar of sugar-10bd76e9ae392e60" ], "reasoning": "1. At 00:22, I identify the pan's (saucepan-66773cb00653b78c) original location on the stove's far left burner .\n2. A 00:24, I observe the user place the pan (saucepan-66773cb00653b78c) on the stove's near left burner \n3. At 09:53, I locate the pan (saucepan-66773cb00653b78c) back in its original location.\n4. I determine the pan's (saucepan-66773cb00653b78c) first 3 positions on the stove are far left, near left, far left.\n5. I determine the pan's (saucepan-66773cb00653b78c) first 3 positions reversed would be near right, far right, near right.", "referenced_object_masks": [ "saucepan-66773cb00653b78c" ] }, { "key": "P09-20240624-165332:9942478f87689cb46ae7fdf383e10bc072ed45d4", "video_id": "P09-20240624-165332", "question": "After returning the electric kettle to its base for the 2nd time, What is the 2nd object I grab with my left hand?", "answer_choice_0": "A timer", "answer_choice_1": "A cup", "answer_choice_2": "A towel", "answer_choice_3": "A plate", "answer_choice_4": "A pan", "answer_id": 2, "answer": "A towel", "question_type": [ "Action Recognition", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "timer-cd2fbe9a034f7d09", "tea towel1-ac78df68cbdab8a2", "saucepan-66773cb00653b78c", "fork1-fe15724041f1bfdf", "plate1-610ad85dcf06cbb3", "sponge-6589531b8f2207a8", "bottle of washing up liquid-06d3122337b69260", "glass bowl-eadbfd4406724438", "saucepan2-8f34836c640b69ff", "mug1-7d29d462b78518bd", "tea towel2-54a395259a186c98", "knife1-8f2b482b5d6135e0", "spoon1-c97e9629d3e959a1", "knife2-eabb2277c8d6fb20", "chopping board-d5b53bb340f96c04", "bowl1-b51721974530e201", "tea towel3-f99b8cfee312815a", "tea towel4-77e57c29641363f2", "skewer1-8843cf42e6ff0788", "skewer2-99c7480cab0195c4", "skewer3-f673fabf71dddbdc", "paper-ad891827e812c1ef", "kettle-1ae788643f0d5c9b", "oven glove-9e6f84e378effde0", "baking tray-86f1c87864a56413", "skewers-1c30ae739807c18d", "parchment paper-31fbdcaf19b9322d", "piece of parchment-581e52185e698d4a", "plate2-2536e34f196dd06d", "mug2-01ce9e39070fa4fa", "cup1-f623543a1b5e5f7a", "bottle of milk-4d04265827e36e6a", "jar of sugar-02b40604b89e4992", "bag of tea-1bd7a6d1a6e75643", "tea bag-b7114ac33c412f4f", "spoon2-b821961cc3a66bf2", "metal grid-8c0be21488a43c2e", "coffee pot-1a69ab621024ef98", "lid of jar of sugar-10bd76e9ae392e60" ], "reasoning": "1. I watched to find when the user moves the electric kettle (kettle-1ae788643f0d5c9b), noting that this occurs between 09:53 and 10:01 when the user picks up the kettle (kettle-1ae788643f0d5c9b) and fills it with water at the sink, then returns it (kettle-1ae788643f0d5c9b) to its base at 10:01.\n2. I saw the user pick up the electric kettle (kettle-1ae788643f0d5c9b) for a second time at 23:07 and place it back on its base at 23:14. \n3. Beginning at 23:14, I looked for the user to grab objects and saw them grab the cup (cup1-f623543a1b5e5f7a) with their left hand at 23:15. I counted this as the first object. \n4. At 23:18, I saw the user grab a kitchen towel (tea towel4-77e57c29641363f2) with their left hand. I counted this as the second object.", "referenced_object_masks": [ "tea towel4-77e57c29641363f2", "kettle-1ae788643f0d5c9b", "cup1-f623543a1b5e5f7a" ] }, { "key": "P09-20240624-165332:a4b7e050a880c4cdf11457993c4e3343375db1ef", "video_id": "P09-20240624-165332", "question": "If I took the total amount of time I spent cleaning the stove, using the moment I took off the grates as a starting point and the moment I finished cleaning the stovetop with the sponge as an end point, and subtracted it from the total length of the video, how long would the video be?", "answer_choice_0": "22:22", "answer_choice_1": "21:13", "answer_choice_2": "22:56", "answer_choice_3": "23:42", "answer_choice_4": "23:08", "answer_id": 4, "answer": "23:08", "question_type": [ "Counterfactual", "Action Recognition", "Object Recognition", "Numerical Reasoning", "Temporal Reasoning" ], "objects_of_interest": [ "timer-cd2fbe9a034f7d09", "tea towel1-ac78df68cbdab8a2", "saucepan-66773cb00653b78c", "fork1-fe15724041f1bfdf", "plate1-610ad85dcf06cbb3", "sponge-6589531b8f2207a8", "bottle of washing up liquid-06d3122337b69260", "glass bowl-eadbfd4406724438", "saucepan2-8f34836c640b69ff", "mug1-7d29d462b78518bd", "tea towel2-54a395259a186c98", "knife1-8f2b482b5d6135e0", "spoon1-c97e9629d3e959a1", "knife2-eabb2277c8d6fb20", "chopping board-d5b53bb340f96c04", "bowl1-b51721974530e201", "tea towel3-f99b8cfee312815a", "tea towel4-77e57c29641363f2", "skewer1-8843cf42e6ff0788", "skewer2-99c7480cab0195c4", "skewer3-f673fabf71dddbdc", "paper-ad891827e812c1ef", "kettle-1ae788643f0d5c9b", "oven glove-9e6f84e378effde0", "baking tray-86f1c87864a56413", "skewers-1c30ae739807c18d", "parchment paper-31fbdcaf19b9322d", "piece of parchment-581e52185e698d4a", "plate2-2536e34f196dd06d", "mug2-01ce9e39070fa4fa", "cup1-f623543a1b5e5f7a", "bottle of milk-4d04265827e36e6a", "jar of sugar-02b40604b89e4992", "bag of tea-1bd7a6d1a6e75643", "tea bag-b7114ac33c412f4f", "spoon2-b821961cc3a66bf2", "metal grid-8c0be21488a43c2e", "coffee pot-1a69ab621024ef98", "lid of jar of sugar-10bd76e9ae392e60" ], "reasoning": "1. I watched the video to see when the user cleaned the stovetop , observing the user begin to take the grates off the stovetop at 19:16.\n2. I watched the user clean the stovetop with a sponge (sponge-6589531b8f2207a8) until 19:48, meaning they cleaned the stovetop from 19:16-19:48, for a total of 32 seconds.\n3. I then noted that the video is 23:40 in length, so I finished by doing the final math of 23:40 - 00:32 = 23:08.", "referenced_object_masks": [ "sponge-6589531b8f2207a8" ] }, { "key": "P09-20240624-165332:e98a19abcfc5108b24128ac101e8831b71a028bf", "video_id": "P09-20240624-165332", "question": "What is the primary color of the second towel with which I reach into the oven?", "answer_choice_0": "White", "answer_choice_1": "Black", "answer_choice_2": "Grey", "answer_choice_3": "Blue", "answer_choice_4": "Red", "answer_id": 2, "answer": "Grey", "question_type": [ "Action Recognition", "Object Recognition", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "timer-cd2fbe9a034f7d09", "tea towel1-ac78df68cbdab8a2", "saucepan-66773cb00653b78c", "fork1-fe15724041f1bfdf", "plate1-610ad85dcf06cbb3", "sponge-6589531b8f2207a8", "bottle of washing up liquid-06d3122337b69260", "glass bowl-eadbfd4406724438", "saucepan2-8f34836c640b69ff", "mug1-7d29d462b78518bd", "tea towel2-54a395259a186c98", "knife1-8f2b482b5d6135e0", "spoon1-c97e9629d3e959a1", "knife2-eabb2277c8d6fb20", "chopping board-d5b53bb340f96c04", "bowl1-b51721974530e201", "tea towel3-f99b8cfee312815a", "tea towel4-77e57c29641363f2", "skewer1-8843cf42e6ff0788", "skewer2-99c7480cab0195c4", "skewer3-f673fabf71dddbdc", "paper-ad891827e812c1ef", "kettle-1ae788643f0d5c9b", "oven glove-9e6f84e378effde0", "baking tray-86f1c87864a56413", "skewers-1c30ae739807c18d", "parchment paper-31fbdcaf19b9322d", "piece of parchment-581e52185e698d4a", "plate2-2536e34f196dd06d", "mug2-01ce9e39070fa4fa", "cup1-f623543a1b5e5f7a", "bottle of milk-4d04265827e36e6a", "jar of sugar-02b40604b89e4992", "bag of tea-1bd7a6d1a6e75643", "tea bag-b7114ac33c412f4f", "spoon2-b821961cc3a66bf2", "metal grid-8c0be21488a43c2e", "coffee pot-1a69ab621024ef98", "lid of jar of sugar-10bd76e9ae392e60" ], "reasoning": "1. As I watched the video, I observed the user open the oven from 04:40-04:41.\n2. After opening the oven , I saw the user reach into the oven with a primarily white towel (tea towel1-ac78df68cbdab8a2) in their right hand, though they did not fully reach in to grab the tray in the oven .\n3. I then watched the user, from 04:47-04:52, pull the tray out with the right hand that was holding a grey towel (tea towel2-54a395259a186c98).\n4. This shows that the user reached for the second time with a towel that was primarily grey.", "referenced_object_masks": [ "tea towel1-ac78df68cbdab8a2", "tea towel2-54a395259a186c98" ] }, { "key": "P09-20240624-165332:ffcacfd02c2ac3e17c03b2d6a13829f4b7584797", "video_id": "P09-20240624-165332", "question": "How many dishes do I wash before I put the potatoes in the pan with oil?", "answer_choice_0": "1", "answer_choice_1": "4", "answer_choice_2": "2", "answer_choice_3": "5", "answer_choice_4": "6", "answer_id": 2, "answer": "2", "question_type": [ "Action Recognition", "Counting", "Object Recognition" ], "objects_of_interest": [ "timer-cd2fbe9a034f7d09", "tea towel1-ac78df68cbdab8a2", "saucepan-66773cb00653b78c", "fork1-fe15724041f1bfdf", "plate1-610ad85dcf06cbb3", "sponge-6589531b8f2207a8", "bottle of washing up liquid-06d3122337b69260", "glass bowl-eadbfd4406724438", "saucepan2-8f34836c640b69ff", "mug1-7d29d462b78518bd", "tea towel2-54a395259a186c98", "knife1-8f2b482b5d6135e0", "spoon1-c97e9629d3e959a1", "knife2-eabb2277c8d6fb20", "chopping board-d5b53bb340f96c04", "bowl1-b51721974530e201", "tea towel3-f99b8cfee312815a", "tea towel4-77e57c29641363f2", "skewer1-8843cf42e6ff0788", "skewer2-99c7480cab0195c4", "skewer3-f673fabf71dddbdc", "paper-ad891827e812c1ef", "kettle-1ae788643f0d5c9b", "oven glove-9e6f84e378effde0", "baking tray-86f1c87864a56413", "skewers-1c30ae739807c18d", "parchment paper-31fbdcaf19b9322d", "piece of parchment-581e52185e698d4a", "plate2-2536e34f196dd06d", "mug2-01ce9e39070fa4fa", "cup1-f623543a1b5e5f7a", "bottle of milk-4d04265827e36e6a", "jar of sugar-02b40604b89e4992", "bag of tea-1bd7a6d1a6e75643", "tea bag-b7114ac33c412f4f", "spoon2-b821961cc3a66bf2", "metal grid-8c0be21488a43c2e", "coffee pot-1a69ab621024ef98", "lid of jar of sugar-10bd76e9ae392e60" ], "reasoning": "1. I watched the video to identify when the user starts putting the potatoes into the pan (saucepan-66773cb00653b78c) with oil , which begins from 01:35-01:42.\n2. I then restarted the video to identify any dishes that are washed. I saw that from 00:56-01:02, the user scrubbed the glass bowl (glass bowl-eadbfd4406724438).\n3. Next, the user grabs and cleans the saucepan (saucepan2-8f34836c640b69ff) from 01:02-01:08.\n4. Between 01:24 and 01:35, no other dishes are washed, so the total number of dishes the user washed before adding the potatoes to the pan (saucepan-66773cb00653b78c) with oil is 2.", "referenced_object_masks": [ "glass bowl-eadbfd4406724438", "saucepan2-8f34836c640b69ff", "saucepan-66773cb00653b78c" ] }, { "key": "P09-20240625-103827:253a75429284777c17916a95b5548d2d60ded3ca", "video_id": "P09-20240625-103827", "question": "What object do I last touch before I stack 5 items on top of each other?", "answer_choice_0": "A frying pan", "answer_choice_1": "A cutlery holder", "answer_choice_2": "A mug", "answer_choice_3": "A utensil", "answer_choice_4": "A glass", "answer_id": 0, "answer": "A frying pan", "question_type": [ "Action Recognition", "Object Recognition", "Temporal Reasoning", "Counting" ], "objects_of_interest": [ "plate1-86960984141460d4", "plate2-f74bc4f839a5335c", "plate3-13d33ec3f78bd4be", "plate4-104c4cf4c2c9b810", "frying pan-4c308e358f651b2d", "plate5-59a3580609ee9d96", "bowl1-615e764b9ba0aacb", "bowl2-bc7cbe5bf483b8dc", "bowl3-1e4a02790b07dfd9", "bowl4-55f044c3c89c6576", "cutlery holder-5697f03ac5e651f6", "utensil1-cd85a2fe044b3c51", "utensil2-23fa02de9c84f375", "utensil3-43f8b3e34deb3f9b", "utensil4-0d809ec11e519e18", "utensil5-6c698d895fd073d9", "utensil6-d34eb404e4d13972", "utensil7-9e788a5bac5a9c15", "utensil8-2de4e37195b82173", "utensil9-2a602f382422867d", "utensil10-f09566e3e9ee6d04", "utensil11-5f4acbcb6ba50623", "utensil12-a5b9a479f33f37c8", "utensil13-03ecc2ec13538744", "utensil14-0a758bd98322504b", "utensil15-e85c323d5ced15fd", "utensil16-6e76de0b487b2901", "utensil17-f3d3efde23956143", "chopping board-7bec465d6a9d7d9d", "glass1-d7f866b9a3ca2aad", "glass2-e426c3a31db9e5d9", "glass3-5c585021b7048fbc", "glass4-8d04efb15f205198", "wine glass-86e97ce270aa11ea", "slotted spatula-4616f31f8d5a68eb", "wooden spoon-2dcd07f0ad3aa5b3", "knife2-cbb44997e5d37ae6", "tongs-9f8514f51a331569", "knife3-6473c4173847bd49", "glass5-a69fe8aefd01a69b", "glass6-7e95783b015424c4", "mug1-3253fbe2a58a89de", "mug2-0642aee73d40b31d", "glass7-c321f8d0c3256e0a", "glass8-31e786e6e20da0c5", "glass9-bfae74c0258b7079", "baking tray-f402487f1b81620a" ], "reasoning": "1. I watched the video and looked for when the user stacked 5 objects on top of each other. \n2. I noticed that the user stacked 5 objects on top of each other from 00:27-00:33, as the user removed them from the dishwasher . This occurs first with a plate (plate5-59a3580609ee9d96), then a bowl (bowl1-615e764b9ba0aacb), a smaller bowl (bowl2-bc7cbe5bf483b8dc), another bowl (bowl3-1e4a02790b07dfd9), and then finally one smaller bowl (bowl4-55f044c3c89c6576).\n3. In order to see what the user touched last before this time, I went back in time in the video. At 00:20, I noticed the user grabbed a frying pan (frying pan-4c308e358f651b2d) as they grabbed the original plate (plate5-59a3580609ee9d96).\n4. At 00:23, the user set down the frying pan (frying pan-4c308e358f651b2d) near the sink, and then resuming grabbing the aforementioned objects to stack them on top of the plate (plate5-59a3580609ee9d96). That makes the frying pan (frying pan-4c308e358f651b2d) the final answer.", "referenced_object_masks": [ "bowl4-55f044c3c89c6576", "bowl1-615e764b9ba0aacb", "bowl2-bc7cbe5bf483b8dc", "frying pan-4c308e358f651b2d", "bowl3-1e4a02790b07dfd9", "plate5-59a3580609ee9d96" ] }, { "key": "P09-20240625-103827:5b0ee17340975c947d4e48b423c5dd04f6da94aa", "video_id": "P09-20240625-103827", "question": "How many objects from the dishwasher do I place upside down inside the cupboard?", "answer_choice_0": "10", "answer_choice_1": "8", "answer_choice_2": "5", "answer_choice_3": "6", "answer_choice_4": "4", "answer_id": 1, "answer": "8", "question_type": [ "Action Recognition", "Counting", "Object Recognition", "Situational Awareness" ], "objects_of_interest": [ "plate1-86960984141460d4", "plate2-f74bc4f839a5335c", "plate3-13d33ec3f78bd4be", "plate4-104c4cf4c2c9b810", "frying pan-4c308e358f651b2d", "plate5-59a3580609ee9d96", "bowl1-615e764b9ba0aacb", "bowl2-bc7cbe5bf483b8dc", "bowl3-1e4a02790b07dfd9", "bowl4-55f044c3c89c6576", "cutlery holder-5697f03ac5e651f6", "utensil1-cd85a2fe044b3c51", "utensil2-23fa02de9c84f375", "utensil3-43f8b3e34deb3f9b", "utensil4-0d809ec11e519e18", "utensil5-6c698d895fd073d9", "utensil6-d34eb404e4d13972", "utensil7-9e788a5bac5a9c15", "utensil8-2de4e37195b82173", "utensil9-2a602f382422867d", "utensil10-f09566e3e9ee6d04", "utensil11-5f4acbcb6ba50623", "utensil12-a5b9a479f33f37c8", "utensil13-03ecc2ec13538744", "utensil14-0a758bd98322504b", "utensil15-e85c323d5ced15fd", "utensil16-6e76de0b487b2901", "utensil17-f3d3efde23956143", "chopping board-7bec465d6a9d7d9d", "glass1-d7f866b9a3ca2aad", "glass2-e426c3a31db9e5d9", "glass3-5c585021b7048fbc", "glass4-8d04efb15f205198", "wine glass-86e97ce270aa11ea", "slotted spatula-4616f31f8d5a68eb", "wooden spoon-2dcd07f0ad3aa5b3", "knife2-cbb44997e5d37ae6", "tongs-9f8514f51a331569", "knife3-6473c4173847bd49", "glass5-a69fe8aefd01a69b", "glass6-7e95783b015424c4", "mug1-3253fbe2a58a89de", "mug2-0642aee73d40b31d", "glass7-c321f8d0c3256e0a", "glass8-31e786e6e20da0c5", "glass9-bfae74c0258b7079", "baking tray-f402487f1b81620a" ], "reasoning": "1. I watched the video and looked for when the user grabbed objects from the dishwasher and then placed them upside-down inside the cupboard. Until 01:35, no objects were placed in the cupboard upside-down.\n2. From 01:35-01:41, I noticed the user placing 4 glasses (glass1-d7f866b9a3ca2aad, glass2-e426c3a31db9e5d9, glass3-5c585021b7048fbc, glass4-8d04efb15f205198) inside of the cupboard. As they place them down, they put them upside-down inside the cupboard. \n3. Looking back, I confirmed these 4 glasses (glass1-d7f866b9a3ca2aad, glass2-e426c3a31db9e5d9, glass3-5c585021b7048fbc, glass4-8d04efb15f205198) were from inside the dishwasher by finding them at 01:22-01:27. This is when the user removed them from the dishwasher and then immediately turned to the right toward the cupboard at 01:28.\n4. I continued watching, looking for other instances. From 01:56-02:03, I noticed the user grabbing 2 additional glasses (glass5-a69fe8aefd01a69b, glass6-7e95783b015424c4) as well as 2 mugs (mug1-3253fbe2a58a89de, mug2-0642aee73d40b31d). From 02:06-02:16, I watched as the user placed all 4 of them upside-down inside the cupboard.\n5. Watching the rest of the video, I did not notice the user adding any other objects upside-down inside the cupboard.\n6. I added 4+4 to get a total of 8 objects.", "referenced_object_masks": [ "glass1-d7f866b9a3ca2aad", "glass2-e426c3a31db9e5d9", "glass3-5c585021b7048fbc", "glass4-8d04efb15f205198", "mug1-3253fbe2a58a89de", "mug2-0642aee73d40b31d", "glass5-a69fe8aefd01a69b", "glass6-7e95783b015424c4" ] }, { "key": "P09-20240625-103827:c9d30abedbaf247253e56b3be0aebb031e4c5208", "video_id": "P09-20240625-103827", "question": "From 02:20 - 02:24, what is the approximate journey of the mug's highest and lowest altitudes in relation to my perspective?", "answer_choice_0": "Near the feet to above the user's head", "answer_choice_1": "Above the user's head to waist level", "answer_choice_2": "Waist level to above the user's head", "answer_choice_3": "Knee high to eye level", "answer_choice_4": "Eye level to waist level", "answer_id": 3, "answer": "Knee high to eye level", "question_type": [ "Temporal Reasoning", "Situational Awareness", "Egocentric & Spatial Perception" ], "objects_of_interest": [ "plate1-86960984141460d4", "plate2-f74bc4f839a5335c", "plate3-13d33ec3f78bd4be", "plate4-104c4cf4c2c9b810", "frying pan-4c308e358f651b2d", "plate5-59a3580609ee9d96", "bowl1-615e764b9ba0aacb", "bowl2-bc7cbe5bf483b8dc", "bowl3-1e4a02790b07dfd9", "bowl4-55f044c3c89c6576", "cutlery holder-5697f03ac5e651f6", "utensil1-cd85a2fe044b3c51", "utensil2-23fa02de9c84f375", "utensil3-43f8b3e34deb3f9b", "utensil4-0d809ec11e519e18", "utensil5-6c698d895fd073d9", "utensil6-d34eb404e4d13972", "utensil7-9e788a5bac5a9c15", "utensil8-2de4e37195b82173", "utensil9-2a602f382422867d", "utensil10-f09566e3e9ee6d04", "utensil11-5f4acbcb6ba50623", "utensil12-a5b9a479f33f37c8", "utensil13-03ecc2ec13538744", "utensil14-0a758bd98322504b", "utensil15-e85c323d5ced15fd", "utensil16-6e76de0b487b2901", "utensil17-f3d3efde23956143", "chopping board-7bec465d6a9d7d9d", "glass1-d7f866b9a3ca2aad", "glass2-e426c3a31db9e5d9", "glass3-5c585021b7048fbc", "glass4-8d04efb15f205198", "wine glass-86e97ce270aa11ea", "slotted spatula-4616f31f8d5a68eb", "wooden spoon-2dcd07f0ad3aa5b3", "knife2-cbb44997e5d37ae6", "tongs-9f8514f51a331569", "knife3-6473c4173847bd49", "glass5-a69fe8aefd01a69b", "glass6-7e95783b015424c4", "mug1-3253fbe2a58a89de", "mug2-0642aee73d40b31d", "glass7-c321f8d0c3256e0a", "glass8-31e786e6e20da0c5", "glass9-bfae74c0258b7079", "baking tray-f402487f1b81620a" ], "reasoning": "1. I found the initial timestamp at 02:20, and located the initial resting point of the mug (mug2-0642aee73d40b31d).\n2. The mug (mug2-0642aee73d40b31d) traveled from the top shelf of the dishwasher, which is either at or slightly above the user's knee, because I saw the drawer next to the dishwasher at 01:50 as being at, or slightly above the user's knee.\n3. The user then takes the mug (mug2-0642aee73d40b31d) out of the dishwasher, turns to their right, and as they move to place the mug in the cabinet, slightly lift their right arm to pass over a few obstacles, before slightly lowering it to place the mug (mug2-0642aee73d40b31d).\n4. With this trajectory, we see that the path went from lowest to highest, with the heights being approximately knee level and eye level.", "referenced_object_masks": [ "mug2-0642aee73d40b31d" ] }, { "key": "P09-20240625-103827:d6716d06b4ffa8644613ffbab8c0ddd763cf20e1", "video_id": "P09-20240625-103827", "question": "If you subtracted the number of steps it took me to reach the dishwasher upon my first visit, from the number of steps it took to reach upon my second visit, what would the number be?", "answer_choice_0": "1", "answer_choice_1": "4", "answer_choice_2": ", 3", "answer_choice_3": "2", "answer_choice_4": "0", "answer_id": 4, "answer": "0", "question_type": [ "Action Recognition", "Object Recognition", "Counting", "Numerical Reasoning" ], "objects_of_interest": [ "plate1-86960984141460d4", "plate2-f74bc4f839a5335c", "plate3-13d33ec3f78bd4be", "plate4-104c4cf4c2c9b810", "frying pan-4c308e358f651b2d", "plate5-59a3580609ee9d96", "bowl1-615e764b9ba0aacb", "bowl2-bc7cbe5bf483b8dc", "bowl3-1e4a02790b07dfd9", "bowl4-55f044c3c89c6576", "cutlery holder-5697f03ac5e651f6", "utensil1-cd85a2fe044b3c51", "utensil2-23fa02de9c84f375", "utensil3-43f8b3e34deb3f9b", "utensil4-0d809ec11e519e18", "utensil5-6c698d895fd073d9", "utensil6-d34eb404e4d13972", "utensil7-9e788a5bac5a9c15", "utensil8-2de4e37195b82173", "utensil9-2a602f382422867d", "utensil10-f09566e3e9ee6d04", "utensil11-5f4acbcb6ba50623", "utensil12-a5b9a479f33f37c8", "utensil13-03ecc2ec13538744", "utensil14-0a758bd98322504b", "utensil15-e85c323d5ced15fd", "utensil16-6e76de0b487b2901", "utensil17-f3d3efde23956143", "chopping board-7bec465d6a9d7d9d", "glass1-d7f866b9a3ca2aad", "glass2-e426c3a31db9e5d9", "glass3-5c585021b7048fbc", "glass4-8d04efb15f205198", "wine glass-86e97ce270aa11ea", "slotted spatula-4616f31f8d5a68eb", "wooden spoon-2dcd07f0ad3aa5b3", "knife2-cbb44997e5d37ae6", "tongs-9f8514f51a331569", "knife3-6473c4173847bd49", "glass5-a69fe8aefd01a69b", "glass6-7e95783b015424c4", "mug1-3253fbe2a58a89de", "mug2-0642aee73d40b31d", "glass7-c321f8d0c3256e0a", "glass8-31e786e6e20da0c5", "glass9-bfae74c0258b7079", "baking tray-f402487f1b81620a" ], "reasoning": "1. I found the initial point in time where the user began their approach to the dishwasher at 00:01. They took 2 steps before reaching the dishwasher.\n2. The next time the user moved toward the dishwasher was at 00:16, where they turned around, and then took 2 steps forward to reach the dishwasher again.\n3. If we subtract the 2 - 2, the answer is 0.", "referenced_object_masks": [] }, { "key": "P09-20240625-103827:e9368f036e4e717addf269f63fa10bc9db784b8a", "video_id": "P09-20240625-103827", "question": "Out of all of the objects I hold from 01:45 - 01:59, I hold the fewest objects made of what kind of material?", "answer_choice_0": "Wood", "answer_choice_1": "Paper", "answer_choice_2": "Ceramic", "answer_choice_3": "Glass", "answer_choice_4": "Plastic", "answer_id": 0, "answer": "Wood", "question_type": [ "Counting", "Object Recognition", "Temporal Reasoning" ], "objects_of_interest": [ "plate1-86960984141460d4", "plate2-f74bc4f839a5335c", "plate3-13d33ec3f78bd4be", "plate4-104c4cf4c2c9b810", "frying pan-4c308e358f651b2d", "plate5-59a3580609ee9d96", "bowl1-615e764b9ba0aacb", "bowl2-bc7cbe5bf483b8dc", "bowl3-1e4a02790b07dfd9", "bowl4-55f044c3c89c6576", "cutlery holder-5697f03ac5e651f6", "utensil1-cd85a2fe044b3c51", "utensil2-23fa02de9c84f375", "utensil3-43f8b3e34deb3f9b", "utensil4-0d809ec11e519e18", "utensil5-6c698d895fd073d9", "utensil6-d34eb404e4d13972", "utensil7-9e788a5bac5a9c15", "utensil8-2de4e37195b82173", "utensil9-2a602f382422867d", "utensil10-f09566e3e9ee6d04", "utensil11-5f4acbcb6ba50623", "utensil12-a5b9a479f33f37c8", "utensil13-03ecc2ec13538744", "utensil14-0a758bd98322504b", "utensil15-e85c323d5ced15fd", "utensil16-6e76de0b487b2901", "utensil17-f3d3efde23956143", "chopping board-7bec465d6a9d7d9d", "glass1-d7f866b9a3ca2aad", "glass2-e426c3a31db9e5d9", "glass3-5c585021b7048fbc", "glass4-8d04efb15f205198", "wine glass-86e97ce270aa11ea", "slotted spatula-4616f31f8d5a68eb", "wooden spoon-2dcd07f0ad3aa5b3", "knife2-cbb44997e5d37ae6", "tongs-9f8514f51a331569", "knife3-6473c4173847bd49", "glass5-a69fe8aefd01a69b", "glass6-7e95783b015424c4", "mug1-3253fbe2a58a89de", "mug2-0642aee73d40b31d", "glass7-c321f8d0c3256e0a", "glass8-31e786e6e20da0c5", "glass9-bfae74c0258b7079", "baking tray-f402487f1b81620a" ], "reasoning": "1. I found the initial timestamp at 01:45, and looked to see what kinds of objects the user handled.\n2. Out of the 5 objects the user handled in 15 seconds, the user touched 2 knives (knife2-cbb44997e5d37ae6) (knife3-6473c4173847bd49), a wooden spoon (wooden spoon-2dcd07f0ad3aa5b3), two glasses (glass5-a69fe8aefd01a69b) (glass6-7e95783b015424c4), a spatula (slotted spatula-4616f31f8d5a68eb) and a pair of tongs (tongs-9f8514f51a331569).\n3. The only kinds of materials handled were plastic, metal, wooden, and glass, with the wooden spoon (wooden spoon-2dcd07f0ad3aa5b3) being the item that contained the material with the least representation.", "referenced_object_masks": [ "glass5-a69fe8aefd01a69b", "slotted spatula-4616f31f8d5a68eb", "tongs-9f8514f51a331569", "glass6-7e95783b015424c4", "knife3-6473c4173847bd49", "wooden spoon-2dcd07f0ad3aa5b3", "knife2-cbb44997e5d37ae6" ] } ]