diff --git a/extensions/src/scratch3_jibo/index.ts b/extensions/src/scratch3_jibo/index.ts index 934a7023d..3169e5672 100644 --- a/extensions/src/scratch3_jibo/index.ts +++ b/extensions/src/scratch3_jibo/index.ts @@ -1,95 +1,685 @@ -import { ArgumentType, BlockType, BlockUtilityWithID, Environment, ExtensionMenuDisplayDetails, Language, Menu, SaveDataHandler, block, buttonBlock, extension, tryCastToArgumentType, untilTimePassed, scratch } from "$common"; -// import jibo from "./jibo.png"; -// import five from "./five.png"; - -const details: ExtensionMenuDisplayDetails = { - name: "Simple Typescript Extension", - description: "Skeleton for a typescript extension", - implementationLanguage: Language.English, - [Language.Español]: { - name: "Extensión simple Typescript", - description: "Ejemplo de una extensión simple usando Typescript" - }, - blockColor: "#822fbd", - menuColor: "#4ed422", - menuSelectColor: "#9e0d2c", - tags: ["PRG Internal"], -} +// firebase +import database from './firebase'; + +import { ArgumentType, BlockType } from "$common"; +import { BlockDefinitions, MenuItem } from "$common"; +import { Extension } from "$common"; +import { RuntimeEvent } from "$common"; + +import VirtualJibo from "./virtualJibo/virtualJibo"; +import { Color, ColorType, colorDef } from "./jiboUtils/ColorDef"; +import { Direction, DirType, directionDef } from "./jiboUtils/LookAtDef"; +import { + Dance, DanceType, danceFiles, + Emotion, EmotionType, emotionFiles, + Icon, IconType, iconFiles, + Audio, AudioType, audioFiles +} from "./jiboUtils/AnimDef"; + +/** Import our svelte components */ +import ColorArgUI from "./ColorArgument.svelte"; +import EmojiArgUI from "./EmojiArgument.svelte"; +import IconArgUI from "./IconArgument.svelte"; + +import ROSLIB from "roslib"; +import BlockUtility from '$root/packages/scratch-vm/src/engine/block-utility'; + +const EXTENSION_ID = "jibo"; + +// jibo's name +var jiboName: string = ""; +// var databaseRef = database.ref("Jibo-Name/" + jiboName); + +type Details = { + name: "Jibo", + description: "Program your favorite social robot, Jibo. This extension works with a physical or virtual Jibo.", + iconURL: "jibo_icon.png", + insetIconURL: "jibo_inset_icon.png", + tags: ["Made by PRG"], +}; + +type Blocks = { + JiboButton: () => void; + JiboTTS: (text: string) => void; + JiboAsk: (text: string) => void; + JiboListen: () => any; + JiboEmote: (emotion: string) => void; + JiboIcon: (icon: string) => void; + JiboDance: (dance: string) => void; + JiboAudio: (audio: string) => void; // new audio block + //JiboVolume: (text: string) => void; // new volume block + JiboLED: (color: string) => void; + JiboLook: (dir: string) => void; // (x_angle: string, y_angle: string, z_angle: string) => void; +}; -export default class SimpleTypescript extends extension(details, "ui", "customSaveData", "indicators") { - count: number = 0; - value: number = 4; +var jibo_event = { + // readyForNext: true, + msg_type: "", + // anim_transition: 0, + // attention_mode: 1, + // audio_filename: "", + // do_anim_transition: false, + // do_attention_mode: false, + // do_led: false, + // do_lookat: false, + // do_motion: false, + // do_sound_playback: false, + // do_tts: false, + // do_volume: false, + // led_color: [0, 100, 0], //red, green, blue + // lookat: [0, 0, 0], //x, y, z + // motion: "", + // tts_duration_stretch: 0, + // tts_pitch: 0, + // tts_text: "", + // volume: 0, +}; - logOptions: Menu = { - items: ['1', 'two', 'three'], - acceptsReporters: true, - handler: (x: any) => tryCastToArgumentType(ArgumentType.String, x, () => { - alert(`Unsopported input: ${x}`); - return ""; - }) +class FirebaseQueue { + async timedFinish(timeoutFn: () => Promise): Promise { + const requests = [ + timeoutFn(), + this.animFinished(), + ]; + return Promise.race(requests); } - override saveDataHandler = new SaveDataHandler({ - Extension: SimpleTypescript, - onSave: ({ count }) => ({ count }), - onLoad: (self, { count }) => self.count = count + async ASR_received(): Promise { + return new Promise((resolve, reject) => { + console.log("Waiting to hear from JiboAsrEvent"); + const pathRef = database.ref("Jibo-Name/" + jiboName); + var eventKey: any; + var eventData: any; + pathRef.on("value", (snapshot) => { + // Loop through the child snapshots of JiboAsrResult + snapshot.forEach((childSnapshot) => { + eventKey = childSnapshot.key; + eventData = childSnapshot.val(); + }); + if (eventData.msg_type === "JiboAsrResult") { + pathRef.off(); + // console.log("eventData is: " + JSON.stringify(eventData)); + var transcription = eventData.transcription; + console.log("Jibo heard: " + transcription); + resolve(transcription); + } + }); + }); + } + async animFinished(): Promise { + return new Promise((resolve, reject) => { + console.log("Waiting for default message from database"); + const pathRef = database.ref("Jibo-Name/" + jiboName); + var eventKey: any; + var eventData: any; + pathRef.on("value", (snapshot) => { + // Loop through the child snapshots of JiboAsrResult + snapshot.forEach((childSnapshot) => { + eventKey = childSnapshot.key; + eventData = childSnapshot.val(); + }); + console.log("last event is"); + console.log(eventData); + if (eventData.msg_type === "default") { + pathRef.off(); + resolve(); + } + }); + }); + } + + async pushToFirebase(data: any, awaitFn: () => Promise) { + if (jiboName != "") { + database.ref("Jibo-Name/" + jiboName).push({ ...data }); + await new Promise(r => setTimeout(r, 2000)); // wait a bit before proceeding + await awaitFn(); + } + else { + console.log("No Jibo Name added."); + } + } +} +const queue = new FirebaseQueue(); + +export async function setJiboName(name: string): Promise { + var jiboNameRef = database.ref("Jibo-Name"); + return new Promise((resolve) => { + jiboNameRef + .once("value", (snapshot) => { + localStorage.setItem("prevJiboName", name); + if (snapshot.hasChild(name)) { + console.log("'" + name + "' exists."); + jiboName = name; + resolve(); + } else { + database.ref("Jibo-Name/" + name).push(jibo_event); + jiboName = name; + console.log( + "'" + name + "' did not exist, and has now been created." + ); + resolve(); + } + }); }); +} + +export default class Scratch3Jibo extends Extension { + ros: any; // TODO + connected: boolean; + rosbridgeIP: string; + jbVolume: string; + asr_out: any; + dances: MenuItem[]; + dirs: MenuItem[]; + audios: MenuItem[]; // new + virtualJibo: VirtualJibo; - increment() { - this.count++; + init() { + this.dances = Object.entries(Dance).map(([dance, def]) => ({ + text: Dance[dance], + value: Dance[dance], + })); + this.dirs = Object.entries(Direction).map(([direction]) => ({ + text: Direction[direction], + value: Direction[direction], + })); + this.audios = Object.entries(Audio).map(([audio, def]) => ({ // new + value: Audio[audio], + text: Audio[audio], + })); + this.runtime.registerPeripheralExtension(EXTENSION_ID, this); + this.runtime.connectPeripheral(EXTENSION_ID, 0); + this.runtime.on(RuntimeEvent.PeripheralConnected, this.connect.bind(this)); + + this.ros = null; + this.connected = false; + this.rosbridgeIP = "ws://localhost:9090"; // rosbridgeIP option includes port + this.jbVolume = "60"; + this.asr_out = ""; + + this.RosConnect({ rosIP: "localhost" }); + + this.virtualJibo = new VirtualJibo(); + this.virtualJibo.init(this.runtime); } - incrementBy(amount: number) { - this.count += amount; + checkBusy(self: Scratch3Jibo) { + // checking state + var state_listener = new ROSLIB.Topic({ + ros: this.ros, + name: "/jibo_state", + messageType: "jibo_msgs/JiboState", + }); + + state_listener.subscribe(function (message: any) { + state_listener.unsubscribe(); + }); } - async init(env: Environment) { - console.log("help"); + defineTranslations() { + return undefined; } - @(scratch.command((self, tag) => tag`Indicate and log ${{ type: "string", options: self.logOptions }} to the console`)) - log(value: string) { - console.log(value); + + + defineBlocks(): BlockDefinitions { + return { + JiboButton: (self: Scratch3Jibo) => ({ + type: BlockType.Button, + arg: { + type: ArgumentType.String, + defaultValue: "Jibo's name here", + }, + text: () => `Connect/Disconnect Jibo`, + operation: async () => { + if (jiboName === "") + this.openUI("jiboNameModal", "Connect Jibo"); + else + jiboName = ""; + }, + }), + JiboTTS: () => ({ + type: BlockType.Command, + arg: { + type: ArgumentType.String, + defaultValue: "Hello, I am Jibo", + }, + text: (text: string) => `say ${text}`, + operation: async (text: string, { target }: BlockUtility) => { + let virtualJ = this.virtualJibo.say(text, target); + let physicalJ = this.jiboTTSFn(text); + await Promise.all([virtualJ, physicalJ]); + } + }), + JiboAsk: () => ({ + type: BlockType.Command, + arg: { + type: ArgumentType.String, + defaultValue: "How are you?", + }, + text: (text: string) => `ask ${text} and wait`, + operation: async (text: string, { target }: BlockUtility) => { + let virtualJ = this.virtualJibo.say(text, target);; + let awaitResponse; + // TODO test + if (jiboName === "") awaitResponse = this.virtualJibo.ask(text); + else awaitResponse = this.jiboAskFn(text); + + await Promise.all([virtualJ, awaitResponse]); + } + }), + JiboListen: () => ({ + type: BlockType.Reporter, + text: `answer`, + operation: () => + this.jiboListenFn(), + }), + // JiboState: () => ({ // helpful for debugging + // type:BlockType.Command, + // text: `read state`, + // operation: () => self.JiboState() + // }), + JiboDance: () => ({ + type: BlockType.Command, + arg: { + type: ArgumentType.String, + options: this.dances, + }, + text: (dname) => `play ${dname} dance`, + operation: async (dance: DanceType) => { + const akey = danceFiles[dance].file; + await this.jiboDanceFn(akey, 5000); + }, + }), + JiboAudio: () => ({ + type: BlockType.Command, + arg: { + type: ArgumentType.String, + options: this.audios, + }, + text: (audioname) => `play ${audioname} audio`, + operation: async (audio: AudioType) => { + const audiokey = audioFiles[audio].file; + await this.jiboAudioFn(audiokey); + }, + }), + /* Jibo block still does not work + // new volume block start + JiboVolume: () => ({ + type: BlockType.Command, + arg: { + type: ArgumentType.String, + defaultValue: "60", + }, + text: (volume: string) => `set volume to ${volume}`, + operation: (volume: string) => + this.jiboVolumeFn(volume), + }), + // new volume block end + */ + JiboEmote: () => ({ + type: BlockType.Command, + arg: this.makeCustomArgument({ + component: EmojiArgUI, + initial: { + value: Emotion.Happy, + text: "Happy", + }, + }), + text: (aname) => `play ${aname} emotion`, + operation: async (anim: EmotionType, { target }: BlockUtility) => { + let virtualJ = this.virtualJibo.anim(anim, "emotion", target); + const akey = emotionFiles[anim].file; + let physicalJ = this.jiboAnimFn(akey, 1000); + await Promise.all([virtualJ, physicalJ]); + }, + }), + JiboIcon: () => ({ + type: BlockType.Command, + arg: this.makeCustomArgument({ + component: IconArgUI, + initial: { + value: Icon.Taco, + text: "taco", + }, + }), + text: (aname) => `show ${aname} icon`, + operation: async (icon: IconType, { target }: BlockUtility) => { + let virtualJ = this.virtualJibo.anim(icon, "icon", target); + const akey = iconFiles[icon].file; + let physicalJ = this.jiboAnimFn(akey, 1000); + await Promise.all([virtualJ, physicalJ]); + } + }), + JiboLED: () => ({ + type: BlockType.Command, + arg: this.makeCustomArgument({ + component: ColorArgUI, + initial: { + value: Color.Blue, + text: "blue", + }, + }), + text: (cname) => `set LED ring to ${cname}`, + operation: async (color: ColorType, { target }: BlockUtility) => { + let virtualJ = this.virtualJibo.setLED(color, target); + let physicalJ = this.jiboLEDFn(color); + await Promise.all([virtualJ, physicalJ]); + } + }), + JiboLook: () => ({ + type: BlockType.Command, + arg: { + type: ArgumentType.String, + options: this.dirs, + }, + text: (dname) => `look ${dname}`, + operation: async (dir: DirType, { target }: BlockUtility) => { + let virtualJ = this.virtualJibo.lookAt(dir, target); + let physicalJ = this.jiboLookFn(dir); + await Promise.all([virtualJ, physicalJ]); + }, + }), + }; } - @(scratch.command` - Indicate ${{ type: "string", defaultValue: "Howdy!" }} - as ${{ type: "string", options: ["error", "success", "warning"] }} - for ${{ type: "number", options: [1, 3, 5] }} - seconds - `) - async indicateMessage(value: string, type: typeof this.IndicatorType, time: number) { - const position = "category"; - const msg = `This is a ${type} indicator for ${value}!`; - const [{ close }] = await Promise.all([ - this.indicate({ position, type, msg }), untilTimePassed(time * 1000) - ]); - close(); + /* The following 4 functions have to exist for the peripherial indicator */ + connect() { + console.log(`Jibo this.connect ${jiboName}`); + this.jiboTTSFn("Hey there. I am ready to program now"); + } + disconnect() { + } + scan() { } + isConnected() { + console.log("isConnected status: " + jiboName); + return !(jiboName === ""); } - @(scratch.button`Dummy UI`) - dummyUI() { - this.openUI("Dummy", "Howdy"); + RosConnect(args: { rosIP: any }) { + const rosIP = args.rosIP.toString(); + this.rosbridgeIP = "ws://" + rosIP + ":9090"; + // log.log("ROS: Attempting to connect to rosbridge at " + this.rosbridgeIP); + + if (!this.connected) { + this.ros = new ROSLIB.Ros({ + url: this.rosbridgeIP, + }); + + // If connection is successful + let connect_cb_factory = function (self: Scratch3Jibo) { + return function () { + self.connected = true; + // send jibo welcome message + let welcomeText = `Hello there. I am ready for you to program me.`; + self.jiboTTSFn(welcomeText); + }; + }; + let connect_cb = connect_cb_factory(this); + this.ros.on("connection", function () { + connect_cb(); + // log.info('ROS: Connected to websocket server.'); + }); + + // If connection fails + let error_cb_factory = function (self: Scratch3Jibo) { + return function () { + self.connected = false; + }; + }; + let error_cb = error_cb_factory(this); + this.ros.on("error", function (error: any) { + error_cb(); + // log.error('ROS: Error connecting to websocket server: ', error); + }); + + // If connection ends + let disconnect_cb_factory = function (self: Scratch3Jibo) { + return function () { + self.connected = false; + }; + }; + let disconnect_cb = disconnect_cb_factory(this); + this.ros.on("close", function () { + disconnect_cb(); + // log.info('ROS: Connection to websocket server closed.'); + }); + } + this.JiboState(); + this.JiboPublish({ + do_attention_mode: true, + attention_mode: 1, + do_anim_transition: true, + anim_transition: 0, + do_led: true, + led_color: { x: 0, y: 0, z: 0 }, + }); + this.JiboASR_receive(); + return this.connected; } - @(scratch.button`Open Counter`) - counterUI() { - this.openUI("Counter", "Pretty cool, right?"); + async jiboTTSFn(text: string) { + var jibo_msg = { + // readyForNext: false, + msg_type: "JiboAction", + do_tts: true, + tts_text: text, + do_lookat: false, + do_motion: false, + do_sound_playback: false, + volume: parseFloat(this.jbVolume), + }; + + // write to firebase + await queue.pushToFirebase(jibo_msg, queue.animFinished); + + await this.JiboPublish(jibo_msg); } - @(scratch.button`Show colors`) - colorUI() { - this.openUI("Palette"); + // TODO figure out why Jibo seems to ignore this value + async jiboVolumeFn(volume: string) { + // update Jibo's volume + this.jbVolume = volume; } - // @(scratch.command`This is what jibo looks like ${{ type: "image", uri: jibo, alt: "Picture of Jibo", flipRTL: true }}`) - // imageBlock(jibo: "inline image") { - // } + async jiboAskFn(text: string) { + // say question + await this.jiboTTSFn(text); + // making the ASR request + await this.JiboASR_request(); - // @(scratch.reporter`${{ type: "number", defaultValue: 1 }} + ${{ type: "image", uri: five, alt: "golden five" }} - ${"number"}`) - // addFive(lhs: number, five: "inline image", rhs: number, { blockID }: BlockUtilityWithID) { - // console.log(blockID); - // return lhs + 5 - rhs; - // } -} + // wait for sensor to return + this.asr_out = await queue.ASR_received(); + } + async jiboListenFn() { + if (jiboName === "") return this.virtualJibo.answer; + return this.asr_out; + } + + async jiboLEDFn(color: string) { + let ledValue = colorDef[color].value; + if (color === "random") { + const randomColorIdx = Math.floor( + // exclude random and off + Math.random() * (Object.keys(colorDef).length - 2) + ); + const randomColor = Object.keys(colorDef)[randomColorIdx]; + ledValue = colorDef[randomColor].value; + } + + // must be "var" does not work with "let" + var jibo_msg = { + // readyForNext: false, + msg_type: "JiboAction", + do_led: true, + led_color: ledValue, + }; + + // write to firebase + var timer = () => new Promise((resolve, reject) => { + setTimeout(resolve, 500); + }); + await queue.pushToFirebase(jibo_msg, + () => queue.timedFinish(timer) + ); // set 500ms time limit on led command + + await this.JiboPublish(jibo_msg); + } + + // there is no message when the look finishes. Just using a set time to finish block + async jiboLookFn(dir: string) { + let coords = directionDef[dir].value; + let jibo_msg = { + do_lookat: true, + lookat: { + x: coords.x, + y: coords.y, + z: coords.z, + }, + }; + + // write to firebase + var timer = () => new Promise((resolve, reject) => { + setTimeout(resolve, 1000); // wait a second for movement to complete + }); + await queue.pushToFirebase(jibo_msg, timer) + + await this.JiboPublish(jibo_msg); + } + + async jiboAnimFn(animation_key: string, delay: number) { + console.log("the animation file is: " + animation_key); // debug statement + var jibo_msg = { + // readyForNext: false, + msg_type: "JiboAction", + do_motion: true, + do_sound_playback: false, + do_tts: false, + do_lookat: false, + motion: animation_key, + }; + + // write to firebase + var timer = (delay) => new Promise((resolve, reject) => { + setTimeout(resolve, delay); // using timer because animFinished does not seem to be reliable + }); + await queue.pushToFirebase(jibo_msg, timer.bind(delay)); // delay before next command + + await this.JiboPublish(jibo_msg); + } + + async jiboDanceFn(animation_key: string, delay: number) { + await this.jiboAnimFn(animation_key, delay); + // transition back to neutral + var timer = () => new Promise((resolve, reject) => { + setTimeout(resolve, 500); + }); + await queue.pushToFirebase({ + do_anim_transition: true, + anim_transition: 0 + }, timer); + + await this.JiboPublish({ do_anim_transition: true, anim_transition: 0 }); + } + async jiboAudioFn(audio_file: string) { + console.log("the audio file is: " + audio_file); // debug statement + var jibo_msg = { + // readyForNext: false, + msg_type: "JiboAction", + do_motion: false, + do_sound_playback: true, + do_tts: false, + do_lookat: false, + audio_filename: audio_file, + }; + + // write to firebase + await queue.pushToFirebase(jibo_msg, queue.animFinished); + + await this.JiboPublish(jibo_msg); + } + + async JiboPublish(msg: any) { + if (!this.connected) { + console.log("ROS is not connected"); + return false; + } + var cmdVel = new ROSLIB.Topic({ + ros: this.ros, + name: "/jibo", + messageType: "jibo_msgs/JiboAction", + }); + // console.log(msg); + var jibo_msg = new ROSLIB.Message(msg); + cmdVel.publish(jibo_msg); + await new Promise((r) => setTimeout(r, 500)); + } + + JiboState() { + // Subscribing to a Topic + // ---------------------- + + console.log("listening..."); + + var state_listener = new ROSLIB.Topic({ + ros: this.ros, + name: "/jibo_state", + messageType: "jibo_msgs/JiboState", + }); + + state_listener.subscribe(function (message: any) { + console.log("Received message on " + state_listener.name + ": "); + console.log(message); + state_listener.unsubscribe(); + }); + } + async JiboASR_request() { + // if (!this.connected) { + // console.log("ROS is not connetced"); + // return false; + // } + // var cmdVel = new ROSLIB.Topic({ + // ros: this.ros, + // name: "/jibo_asr_command", + // messageType: "jibo_msgs/JiboAsrCommand", + // }); + // var jibo_msg = new ROSLIB.Message({ heyjibo: false, command: 1 }); + var jibo_msg = { + msg_type: "JiboAsrCommand", + command: 1, + heyjibo: false, + detectend: false, + continuous: false, + incremental: false, + alternatives: false, + rule: "", + }; + + var timer = () => new Promise((resolve, reject) => { + setTimeout(resolve, 500); + }); + await queue.pushToFirebase(jibo_msg, timer); // delay a bit before next command + // cmdVel.publish(jibo_msg); + } + + async JiboASR_receive(): Promise { + return new Promise((resolve) => { + var asr_listener = new ROSLIB.Topic({ + ros: this.ros, + name: "/jibo_asr_result", + messageType: "jibo_msgs/JiboAsrResult", + }); + + asr_listener.subscribe(function (message: { transcription: unknown }) { + console.log("Received message on " + asr_listener.name + ": "); + console.log(message); + asr_listener.unsubscribe(); + //this.asr_out = message.transcription; + resolve(message.transcription); + // return readAsrAnswer(message.transcription); + }); + }); + + } +} diff --git a/extensions/src/scratch3_jibo/jiboNameModal.svelte b/extensions/src/scratch3_jibo/jiboNameModal.svelte index f462ddd9d..d41def6df 100644 --- a/extensions/src/scratch3_jibo/jiboNameModal.svelte +++ b/extensions/src/scratch3_jibo/jiboNameModal.svelte @@ -2,7 +2,7 @@ import type Extension from "."; import { ReactiveInvoke, reactiveInvoke, activeClass, color } from "$common"; // my imports - // import { setJiboName } from "./index"; + import { setJiboName } from "./index"; /** * @summary This is a reference to the instance of your extension. @@ -45,7 +45,7 @@ if (validJiboName(inputText)) { inputText = inputText.toLowerCase(); inputText = inputText.trim(); - //await setJiboName(inputText); + await setJiboName(inputText); // run extensions "connect" function once name is set invoke("connect"); errorVisible = true; diff --git a/extensions/src/scratch3_jibo/package.json b/extensions/src/scratch3_jibo/package.json index 0ecfb106c..de26891ee 100644 --- a/extensions/src/scratch3_jibo/package.json +++ b/extensions/src/scratch3_jibo/package.json @@ -1,16 +1,20 @@ { - "name": "jibo-extension", + "name": "scratch3_jibo-extension", "version": "1.0.0", "description": "An extension created using the PRG AI Blocks framework", "main": "index.ts", "scripts": { "directory": "echo scratch3_jibo", - "dev": "pnpm --filter prg-extension-root dev -i scratch3_jibo", - "test": "pnpm --filter prg-extension-root test scratch3_jibo/index.test.ts" + "dev": "npm run dev --prefix ../../../ -- only=scratch3_jibo", + "test": "npm run test --prefix ../../ -- scratch3_jibo/index.test.ts" }, "author": "", "license": "ISC", "dependencies": { - "firebase": "^9.22.2" + "firebase": "^9.22.2", + "roslib": "^1.3.0" + }, + "devDependencies": { + "@types/roslib": "^1.3.0" } } diff --git a/extensions/src/scratch3_jibo/pnpm-lock.yaml b/extensions/src/scratch3_jibo/pnpm-lock.yaml index 0d444bfdf..6db568c10 100644 --- a/extensions/src/scratch3_jibo/pnpm-lock.yaml +++ b/extensions/src/scratch3_jibo/pnpm-lock.yaml @@ -11,6 +11,13 @@ importers: firebase: specifier: ^9.22.2 version: 9.23.0 + roslib: + specifier: ^1.3.0 + version: 1.4.1 + devDependencies: + '@types/roslib': + specifier: ^1.3.0 + version: 1.3.5 packages: @@ -236,11 +243,28 @@ packages: '@protobufjs/utf8@1.1.0': resolution: {integrity: sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==} + '@socket.io/component-emitter@3.1.2': + resolution: {integrity: sha512-9BCxFwvbGg/RsZK9tjXd8s4UcwR0MWeFQ1XEKIQVVvAGJyINdrqKMcTRyLoK8Rse1GjzLV9cwjWV1olXRWEXVA==} + + '@types/cors@2.8.17': + resolution: {integrity: sha512-8CGDvrBj1zgo2qE+oS3pOCyYNqCPryMWY2bGfwA0dcfopWGgxs+78df0Rs3rc9THP4JkOhLsAa+15VdpAqkcUA==} + '@types/long@4.0.2': resolution: {integrity: sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==} - '@types/node@22.13.1': - resolution: {integrity: sha512-jK8uzQlrvXqEU91UxiK5J7pKHyzgnI1Qnl0QDHIgVGuolJhRb9EEl28Cj9b3rGR8B2lhFCtvIm5os8lFnO/1Ew==} + '@types/node@22.13.2': + resolution: {integrity: sha512-Z+r8y3XL9ZpI2EY52YYygAFmo2/oWfNSj4BCpAXE2McAexDk8VcnBMGC9Djn9gTKt4d2T/hhXqmPzo4hfIXtTg==} + + '@types/roslib@1.3.5': + resolution: {integrity: sha512-rye0xL6oZQFUaC79PXpM6zhYflpHuMTiEdEYkra5psBbTQ+m049UKMXzBFci8UgptULG+CB86wJBjD9q3WB5rw==} + + '@xmldom/xmldom@0.8.10': + resolution: {integrity: sha512-2WALfTl4xo2SkGCYRt6rDTFfk9R1czmBvUQy12gK2KuRKIpWEhcbbzy8EZXtz/jkRqHX8bFEc6FC1HjX4TUWYw==} + engines: {node: '>=10.0.0'} + + accepts@1.3.8: + resolution: {integrity: sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==} + engines: {node: '>= 0.6'} ansi-regex@5.0.1: resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} @@ -250,6 +274,13 @@ packages: resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} engines: {node: '>=8'} + base64id@2.0.0: + resolution: {integrity: sha512-lGe34o6EHj9y3Kts9R4ZYs/Gr+6N7MCaMlIFA3F1R2O5/m7K06AxfSeO5530PEERE6/WyEg3lsuyw4GHlPZHog==} + engines: {node: ^4.5.0 || >= 5.9} + + cbor-js@0.1.0: + resolution: {integrity: sha512-7sQ/TvDZPl7csT1Sif9G0+MA0I0JOVah8+wWlJVQdVEgIbCzlN/ab3x+uvMNsc34TUvO6osQTAmB2ls80JX6tw==} + cliui@7.0.4: resolution: {integrity: sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==} @@ -264,13 +295,41 @@ packages: color-name@1.1.4: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + cookie@0.7.2: + resolution: {integrity: sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==} + engines: {node: '>= 0.6'} + + cors@2.8.5: + resolution: {integrity: sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==} + engines: {node: '>= 0.10'} + + debug@4.3.7: + resolution: {integrity: sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + emoji-regex@8.0.0: resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} + engine.io-parser@5.2.3: + resolution: {integrity: sha512-HqD3yTBfnBxIrbnM1DoD6Pcq8NECnh8d4As1Qgh0z5Gg3jRRIqijury0CL3ghu/edArpUYiYqQiDUQBIs4np3Q==} + engines: {node: '>=10.0.0'} + + engine.io@6.6.4: + resolution: {integrity: sha512-ZCkIjSYNDyGn0R6ewHDtXgns/Zre/NT6Agvq1/WobF7JXgFff4SeDroKiCO3fNJreU9YG429Sc81o4w5ok/W5g==} + engines: {node: '>=10.2.0'} + escalade@3.2.0: resolution: {integrity: sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==} engines: {node: '>=6'} + eventemitter2@6.4.9: + resolution: {integrity: sha512-JEPTiaOt9f04oa6NOkc4aH+nVp5I3wEjpHbIPqfgCdD5v5bUzy7xQqwcVO2aDQgOWhI28da57HksMrzK9HlRxg==} + faye-websocket@0.11.4: resolution: {integrity: sha512-CzbClwlXAuiRQAlUyfqPgvPoNKTckTPGfwZV4ZdAhVcP2lh9KUxJg2b5GkE7XbjKQ3YJnQ9z6D9ntLAlB+tP8g==} engines: {node: '>=0.8.0'} @@ -304,6 +363,21 @@ packages: long@5.3.0: resolution: {integrity: sha512-5vvY5yF1zF/kXk+L94FRiTDa1Znom46UjPCH6/XbSvS8zBKMFBHTJk8KDMqJ+2J6QezQFi7k1k8v21ClJYHPaw==} + mime-db@1.52.0: + resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} + engines: {node: '>= 0.6'} + + mime-types@2.1.35: + resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} + engines: {node: '>= 0.6'} + + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + negotiator@0.6.3: + resolution: {integrity: sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==} + engines: {node: '>= 0.6'} + node-fetch@2.6.7: resolution: {integrity: sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==} engines: {node: 4.x || >=6.0.0} @@ -313,6 +387,13 @@ packages: encoding: optional: true + object-assign@4.1.1: + resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} + engines: {node: '>=0.10.0'} + + pngparse@2.0.1: + resolution: {integrity: sha512-RyB1P0BBwt3CNIZ5wT53lR1dT3CUtopnMOuP8xZdHjPhI/uXNNRnkx1yQb/3MMMyyMeo6p19fiIRHcLopWIkxA==} + protobufjs@6.11.4: resolution: {integrity: sha512-5kQWPaJHi1WoCpjTGszzQ32PG2F4+wRY6BmAT4Vfw56Q2FZ4YZzK20xUYQH4YkfehY1e6QSICrJquM6xXZNcrw==} hasBin: true @@ -325,9 +406,24 @@ packages: resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==} engines: {node: '>=0.10.0'} + roslib@1.4.1: + resolution: {integrity: sha512-l3BOHqG99RHb73XROykj8o2rRaUqqYwN0E6C1EkH+R1GIfDjMaUGPaCNEoKKmsXT0Vu0EOyL1BudQtdVlMsgjA==} + engines: {node: '>=0.10'} + safe-buffer@5.2.1: resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} + socket.io-adapter@2.5.5: + resolution: {integrity: sha512-eLDQas5dzPgOWCk9GuuJC2lBqItuhKI4uxGgo9aIV7MYbk2h9Q6uULEh8WBzThoI7l+qU9Ast9fVUmkqPP9wYg==} + + socket.io-parser@4.2.4: + resolution: {integrity: sha512-/GbIKmo8ioc+NIWIhwdecY0ge+qVBSMdgxGygevmdHj24bsfgtCmcUUcQ5ZzcylGFHsN3k4HB4Cgkl96KVnuew==} + engines: {node: '>=10.0.0'} + + socket.io@4.8.1: + resolution: {integrity: sha512-oZ7iUCxph8WYRHHcjBEc9unw3adt5CmSNlppj/5Q4k2RIrhl8Z5yY2Xr4j9zj0+wzVZ0bxmYoGSzKJnRl6A4yg==} + engines: {node: '>=10.2.0'} + string-width@4.2.3: resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} engines: {node: '>=8'} @@ -345,6 +441,10 @@ packages: undici-types@6.20.0: resolution: {integrity: sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==} + vary@1.1.2: + resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==} + engines: {node: '>= 0.8'} + webidl-conversions@3.0.1: resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} @@ -356,6 +456,12 @@ packages: resolution: {integrity: sha512-OqedPIGOfsDlo31UNwYbCFMSaO9m9G/0faIHj5/dZFDMFqPTcx6UwqyOy3COEaEOg/9VsGIpdqn62W5KhoKSpg==} engines: {node: '>=0.8.0'} + webworkify-webpack@2.1.5: + resolution: {integrity: sha512-2akF8FIyUvbiBBdD+RoHpoTbHMQF2HwjcxfDvgztAX5YwbZNyrtfUMgvfgFVsgDhDPVTlkbb5vyasqDHfIDPQw==} + + webworkify@1.5.0: + resolution: {integrity: sha512-AMcUeyXAhbACL8S2hqqdqOLqvJ8ylmIbNwUIqQujRSouf4+eUFaXbG6F1Rbu+srlJMmxQWsiU7mOJi0nMBfM1g==} + whatwg-url@5.0.0: resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==} @@ -363,6 +469,30 @@ packages: resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==} engines: {node: '>=10'} + ws@8.17.1: + resolution: {integrity: sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + + ws@8.18.0: + resolution: {integrity: sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + y18n@5.0.8: resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} engines: {node: '>=10'} @@ -701,7 +831,7 @@ snapshots: '@grpc/grpc-js@1.7.3': dependencies: '@grpc/proto-loader': 0.7.13 - '@types/node': 22.13.1 + '@types/node': 22.13.2 '@grpc/proto-loader@0.6.13': dependencies: @@ -741,18 +871,39 @@ snapshots: '@protobufjs/utf8@1.1.0': {} + '@socket.io/component-emitter@3.1.2': {} + + '@types/cors@2.8.17': + dependencies: + '@types/node': 22.13.2 + '@types/long@4.0.2': {} - '@types/node@22.13.1': + '@types/node@22.13.2': dependencies: undici-types: 6.20.0 + '@types/roslib@1.3.5': + dependencies: + eventemitter2: 6.4.9 + + '@xmldom/xmldom@0.8.10': {} + + accepts@1.3.8: + dependencies: + mime-types: 2.1.35 + negotiator: 0.6.3 + ansi-regex@5.0.1: {} ansi-styles@4.3.0: dependencies: color-convert: 2.0.1 + base64id@2.0.0: {} + + cbor-js@0.1.0: {} + cliui@7.0.4: dependencies: string-width: 4.2.3 @@ -771,10 +922,41 @@ snapshots: color-name@1.1.4: {} + cookie@0.7.2: {} + + cors@2.8.5: + dependencies: + object-assign: 4.1.1 + vary: 1.1.2 + + debug@4.3.7: + dependencies: + ms: 2.1.3 + emoji-regex@8.0.0: {} + engine.io-parser@5.2.3: {} + + engine.io@6.6.4: + dependencies: + '@types/cors': 2.8.17 + '@types/node': 22.13.2 + accepts: 1.3.8 + base64id: 2.0.0 + cookie: 0.7.2 + cors: 2.8.5 + debug: 4.3.7 + engine.io-parser: 5.2.3 + ws: 8.17.1 + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + escalade@3.2.0: {} + eventemitter2@6.4.9: {} + faye-websocket@0.11.4: dependencies: websocket-driver: 0.7.4 @@ -826,10 +1008,24 @@ snapshots: long@5.3.0: {} + mime-db@1.52.0: {} + + mime-types@2.1.35: + dependencies: + mime-db: 1.52.0 + + ms@2.1.3: {} + + negotiator@0.6.3: {} + node-fetch@2.6.7: dependencies: whatwg-url: 5.0.0 + object-assign@4.1.1: {} + + pngparse@2.0.1: {} + protobufjs@6.11.4: dependencies: '@protobufjs/aspromise': 1.1.2 @@ -843,7 +1039,7 @@ snapshots: '@protobufjs/pool': 1.1.0 '@protobufjs/utf8': 1.1.0 '@types/long': 4.0.2 - '@types/node': 22.13.1 + '@types/node': 22.13.2 long: 4.0.0 protobufjs@7.4.0: @@ -858,13 +1054,59 @@ snapshots: '@protobufjs/path': 1.1.2 '@protobufjs/pool': 1.1.0 '@protobufjs/utf8': 1.1.0 - '@types/node': 22.13.1 + '@types/node': 22.13.2 long: 5.3.0 require-directory@2.1.1: {} + roslib@1.4.1: + dependencies: + '@xmldom/xmldom': 0.8.10 + cbor-js: 0.1.0 + eventemitter2: 6.4.9 + object-assign: 4.1.1 + pngparse: 2.0.1 + socket.io: 4.8.1 + webworkify: 1.5.0 + webworkify-webpack: 2.1.5 + ws: 8.18.0 + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + safe-buffer@5.2.1: {} + socket.io-adapter@2.5.5: + dependencies: + debug: 4.3.7 + ws: 8.17.1 + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + + socket.io-parser@4.2.4: + dependencies: + '@socket.io/component-emitter': 3.1.2 + debug: 4.3.7 + transitivePeerDependencies: + - supports-color + + socket.io@4.8.1: + dependencies: + accepts: 1.3.8 + base64id: 2.0.0 + cors: 2.8.5 + debug: 4.3.7 + engine.io: 6.6.4 + socket.io-adapter: 2.5.5 + socket.io-parser: 4.2.4 + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + string-width@4.2.3: dependencies: emoji-regex: 8.0.0 @@ -881,6 +1123,8 @@ snapshots: undici-types@6.20.0: {} + vary@1.1.2: {} + webidl-conversions@3.0.1: {} websocket-driver@0.7.4: @@ -891,6 +1135,10 @@ snapshots: websocket-extensions@0.1.4: {} + webworkify-webpack@2.1.5: {} + + webworkify@1.5.0: {} + whatwg-url@5.0.0: dependencies: tr46: 0.0.3 @@ -902,6 +1150,10 @@ snapshots: string-width: 4.2.3 strip-ansi: 6.0.1 + ws@8.17.1: {} + + ws@8.18.0: {} + y18n@5.0.8: {} yargs-parser@20.2.9: {} diff --git a/extensions/src/scratch3_jibo/virtualJibo/jiboBody/black.png b/extensions/src/scratch3_jibo/virtualJibo/jiboBody/black.png new file mode 100644 index 000000000..98cacd183 Binary files /dev/null and b/extensions/src/scratch3_jibo/virtualJibo/jiboBody/black.png differ diff --git a/extensions/src/scratch3_jibo/virtualJibo/jiboBody/blue.png b/extensions/src/scratch3_jibo/virtualJibo/jiboBody/blue.png new file mode 100644 index 000000000..5de60f65e Binary files /dev/null and b/extensions/src/scratch3_jibo/virtualJibo/jiboBody/blue.png differ diff --git a/extensions/src/scratch3_jibo/virtualJibo/jiboBody/cyan.png b/extensions/src/scratch3_jibo/virtualJibo/jiboBody/cyan.png new file mode 100644 index 000000000..0c8cc279d Binary files /dev/null and b/extensions/src/scratch3_jibo/virtualJibo/jiboBody/cyan.png differ diff --git a/extensions/src/scratch3_jibo/virtualJibo/jiboBody/green.png b/extensions/src/scratch3_jibo/virtualJibo/jiboBody/green.png new file mode 100644 index 000000000..6d8b14215 Binary files /dev/null and b/extensions/src/scratch3_jibo/virtualJibo/jiboBody/green.png differ diff --git a/extensions/src/scratch3_jibo/virtualJibo/jiboBody/magenta.png b/extensions/src/scratch3_jibo/virtualJibo/jiboBody/magenta.png new file mode 100644 index 000000000..82490b787 Binary files /dev/null and b/extensions/src/scratch3_jibo/virtualJibo/jiboBody/magenta.png differ diff --git a/extensions/src/scratch3_jibo/virtualJibo/jiboBody/red.png b/extensions/src/scratch3_jibo/virtualJibo/jiboBody/red.png new file mode 100644 index 000000000..4306a6a71 Binary files /dev/null and b/extensions/src/scratch3_jibo/virtualJibo/jiboBody/red.png differ diff --git a/extensions/src/scratch3_jibo/virtualJibo/jiboBody/white.png b/extensions/src/scratch3_jibo/virtualJibo/jiboBody/white.png new file mode 100644 index 000000000..e73e6d851 Binary files /dev/null and b/extensions/src/scratch3_jibo/virtualJibo/jiboBody/white.png differ diff --git a/extensions/src/scratch3_jibo/virtualJibo/jiboBody/yellow.png b/extensions/src/scratch3_jibo/virtualJibo/jiboBody/yellow.png new file mode 100644 index 000000000..29994ed8d Binary files /dev/null and b/extensions/src/scratch3_jibo/virtualJibo/jiboBody/yellow.png differ diff --git a/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Airplane.png b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Airplane.png new file mode 100644 index 000000000..4fe2cefba Binary files /dev/null and b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Airplane.png differ diff --git a/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Apple.png b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Apple.png new file mode 100644 index 000000000..0a79e4f3d Binary files /dev/null and b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Apple.png differ diff --git a/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Art.png b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Art.png new file mode 100644 index 000000000..f300ce857 Binary files /dev/null and b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Art.png differ diff --git a/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Bowling.png b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Bowling.png new file mode 100644 index 000000000..18ad11829 Binary files /dev/null and b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Bowling.png differ diff --git a/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Checkmark.png b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Checkmark.png new file mode 100644 index 000000000..13bb43d88 Binary files /dev/null and b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Checkmark.png differ diff --git a/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Curious.png b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Curious.png new file mode 100644 index 000000000..726c6801a Binary files /dev/null and b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Curious.png differ diff --git a/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Exclamation.png b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Exclamation.png new file mode 100644 index 000000000..45d87bfcb Binary files /dev/null and b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Exclamation.png differ diff --git a/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Eye1.svg b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Eye1.svg new file mode 100644 index 000000000..12eafb75d --- /dev/null +++ b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Eye1.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Eye2.svg b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Eye2.svg new file mode 100644 index 000000000..4690d1788 --- /dev/null +++ b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Eye2.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Eye3.svg b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Eye3.svg new file mode 100644 index 000000000..47a4209ba --- /dev/null +++ b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Eye3.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Eye4.svg b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Eye4.svg new file mode 100644 index 000000000..1940edf56 --- /dev/null +++ b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Eye4.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Eye5.svg b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Eye5.svg new file mode 100644 index 000000000..acdaec94c --- /dev/null +++ b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Eye5.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Football.png b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Football.png new file mode 100644 index 000000000..b11d472be Binary files /dev/null and b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Football.png differ diff --git a/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Frustrated.png b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Frustrated.png new file mode 100644 index 000000000..ad6cea0dd Binary files /dev/null and b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Frustrated.png differ diff --git a/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Happy.png b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Happy.png new file mode 100644 index 000000000..c1723d912 Binary files /dev/null and b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Happy.png differ diff --git a/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Heart.png b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Heart.png new file mode 100644 index 000000000..5c9877f11 Binary files /dev/null and b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Heart.png differ diff --git a/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Laugh.png b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Laugh.png new file mode 100644 index 000000000..30dec8243 Binary files /dev/null and b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Laugh.png differ diff --git a/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Magic.png b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Magic.png new file mode 100644 index 000000000..02fd05de0 Binary files /dev/null and b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Magic.png differ diff --git a/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Music.png b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Music.png new file mode 100644 index 000000000..03f08d3fb Binary files /dev/null and b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Music.png differ diff --git a/extensions/src/scratch3_jibo/virtualJibo/jiboEye/No.png b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/No.png new file mode 100644 index 000000000..fd9d2fc9f Binary files /dev/null and b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/No.png differ diff --git a/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Ocean.png b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Ocean.png new file mode 100644 index 000000000..2a680f573 Binary files /dev/null and b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Ocean.png differ diff --git a/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Penguin.png b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Penguin.png new file mode 100644 index 000000000..f5d32e1d1 Binary files /dev/null and b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Penguin.png differ diff --git a/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Puzzled.png b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Puzzled.png new file mode 100644 index 000000000..5f02a2501 Binary files /dev/null and b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Puzzled.png differ diff --git a/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Rainbow.png b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Rainbow.png new file mode 100644 index 000000000..7a8605c7e Binary files /dev/null and b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Rainbow.png differ diff --git a/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Robot.png b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Robot.png new file mode 100644 index 000000000..cae6f2cb2 Binary files /dev/null and b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Robot.png differ diff --git a/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Rocket.png b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Rocket.png new file mode 100644 index 000000000..50856395a Binary files /dev/null and b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Rocket.png differ diff --git a/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Sad.png b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Sad.png new file mode 100644 index 000000000..b060f4225 Binary files /dev/null and b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Sad.png differ diff --git a/extensions/src/scratch3_jibo/virtualJibo/jiboEye/SadEyes.png b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/SadEyes.png new file mode 100644 index 000000000..433352195 Binary files /dev/null and b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/SadEyes.png differ diff --git a/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Snowflake.png b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Snowflake.png new file mode 100644 index 000000000..133645583 Binary files /dev/null and b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Snowflake.png differ diff --git a/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Success.png b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Success.png new file mode 100644 index 000000000..7567267b2 Binary files /dev/null and b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Success.png differ diff --git a/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Taco.png b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Taco.png new file mode 100644 index 000000000..f94755efd Binary files /dev/null and b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Taco.png differ diff --git a/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Thinking.png b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Thinking.png new file mode 100644 index 000000000..e045a345c Binary files /dev/null and b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Thinking.png differ diff --git a/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Videogame.png b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Videogame.png new file mode 100644 index 000000000..c35e731d7 Binary files /dev/null and b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Videogame.png differ diff --git a/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Yes.png b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Yes.png new file mode 100644 index 000000000..c1e367b8b Binary files /dev/null and b/extensions/src/scratch3_jibo/virtualJibo/jiboEye/Yes.png differ diff --git a/extensions/src/scratch3_jibo/virtualJibo/virtualJibo.ts b/extensions/src/scratch3_jibo/virtualJibo/virtualJibo.ts new file mode 100644 index 000000000..284efb3bd --- /dev/null +++ b/extensions/src/scratch3_jibo/virtualJibo/virtualJibo.ts @@ -0,0 +1,550 @@ +import Target from "$scratch-vm/engine/target"; +import type RenderedTarget from "$scratch-vm/sprites/rendered-target"; +import MockBitmapAdapter from "$common/extension/mixins/optional/addCostumes/MockBitmapAdapter"; +import { getUrlHelper } from "$common/extension/mixins/optional/addCostumes/utils"; + +import { Color, ColorType } from "../jiboUtils/ColorDef"; +import { Direction, DirType } from "../jiboUtils/LookAtDef"; +import { + Dance, DanceType, + Emotion, EmotionType, + Icon, IconType, + Audio, AudioType +} from "../jiboUtils/AnimDef"; + +import jiboBodyBlack from "./jiboBody/black.png"; +import jiboBodyRed from "./jiboBody/red.png"; +import jiboBodyYellow from "./jiboBody/yellow.png"; +import jiboBodyGreen from "./jiboBody/green.png"; +import jiboBodyCyan from "./jiboBody/cyan.png"; +import jiboBodyBlue from "./jiboBody/blue.png"; +import jiboBodyMagenta from "./jiboBody/magenta.png"; +import jiboBodyWhite from "./jiboBody/white.png"; + +import jiboEyeAirplane from "./jiboEye/Airplane.png"; +import jiboEyeApple from "./jiboEye/Apple.png"; +import jiboEyeArt from "./jiboEye/Art.png"; +import jiboEyeBowling from "./jiboEye/Bowling.png"; +import jiboEyeCheckmark from "./jiboEye/Checkmark.png"; +import jiboEyeExclamation from "./jiboEye/Exclamation.png"; +import jiboEyeFootball from "./jiboEye/Football.png"; +import jiboEyeHeart from "./jiboEye/Heart.png"; +import jiboEyeMagic from "./jiboEye/Magic.png"; +import jiboEyeOcean from "./jiboEye/Ocean.png"; +import jiboEyePenguin from "./jiboEye/Penguin.png"; +import jiboEyeRainbow from "./jiboEye/Rainbow.png"; +import jiboEyeRobot from "./jiboEye/Robot.png"; +import jiboEyeRocket from "./jiboEye/Rocket.png"; +import jiboEyeSnowflake from "./jiboEye/Snowflake.png"; +import jiboEyeTaco from "./jiboEye/Taco.png"; +import jiboEyeVideoGame from "./jiboEye/Videogame.png"; +import jiboEyeCurious from "./jiboEye/Curious.png"; +import jiboEyeFrustrated from "./jiboEye/Frustrated.png"; +import jiboEyeHappy from "./jiboEye/Happy.png"; +import jiboEyeLaugh from "./jiboEye/Laugh.png"; +import jiboEyeNo from "./jiboEye/No.png"; +import jiboEyePuzzled from "./jiboEye/Puzzled.png"; +import jiboEyeSad from "./jiboEye/Sad.png"; +import jiboEyeSadEyes from "./jiboEye/SadEyes.png"; +import jiboEyeSuccess from "./jiboEye/Success.png"; +import jiboEyeThinking from "./jiboEye/Thinking.png"; +import jiboEyeYes from "./jiboEye/Yes.png"; + +import jiboEye1 from "./jiboEye/Eye1.svg"; +import jiboEye2 from "./jiboEye/Eye2.svg"; +import jiboEye3 from "./jiboEye/Eye3.svg"; +import jiboEye4 from "./jiboEye/Eye4.svg"; +import jiboEye5 from "./jiboEye/Eye5.svg"; + +import Runtime from "$root/packages/scratch-vm/src/engine/runtime"; + +let bitmapAdapter: MockBitmapAdapter; +let urlHelper: ReturnType; + +const rendererKey: keyof RenderedTarget = "renderer"; +const isRenderedTarget = (target: Target | RenderedTarget): target is RenderedTarget => rendererKey in target; + +const JIBO_BODY = "jibo-body"; +const JIBO_EYE = "jibo-eye"; + +const DEFAULT_JIBO_EYE = { + dx: 0, // jibo eye = jibo body + dx * jibo body size + dy: .76, // jibo eye = (jibo body) + dy * jibo body size + dsize: .65, // jibo eye = jibo body * dsize + diconSize: .35, +} + +type Coords = { + dx: number; + dy: number; +}; +type DirDefType = { + value: Coords; +}; +const directionDef: Record = { + [Direction.up]: { + value: { dx: 0, dy: 1 }, + }, + [Direction.down]: { + value: { dx: 0, dy: 0.45 }, + }, + [Direction.left]: { + value: { dx: 0.45, dy: 0.76 }, + }, + [Direction.right]: { + value: { dx: -0.45, dy: 0.76 }, + }, + [Direction.forward]: { + value: { dx: 0, dy: 0.76 }, + }, +}; + +type ImageDefType = { + imageData: string; +}; + +const jiboEyeDef: Record = { + "Eye1": { + imageData: jiboEye1, + }, + "Eye2": { + imageData: jiboEye2, + }, + "Eye3": { + imageData: jiboEye3, + }, + "Eye4": { + imageData: jiboEye4, + }, + "Eye5": { + imageData: jiboEye5, + }, +}; +const JIBO_EYE_ANIM = [ + "Eye1", "Eye2", "Eye2", "Eye3", "Eye4", "Eye5", "Eye3", "Eye2", "Eye1" +]; + +const colorDef: Record = { + [Color.Red]: { + imageData: jiboBodyRed, + }, + [Color.Yellow]: { + imageData: jiboBodyYellow, + }, + [Color.Green]: { + imageData: jiboBodyGreen, + }, + [Color.Cyan]: { + imageData: jiboBodyCyan, + }, + [Color.Blue]: { + imageData: jiboBodyBlue, + }, + [Color.Magenta]: { + imageData: jiboBodyMagenta, + }, + [Color.White]: { + imageData: jiboBodyWhite, + }, + [Color.Random]: { + imageData: "" + }, + [Color.Off]: { + imageData: jiboBodyBlack, + }, +}; + +type AnimFileType = { + imageData: string; +}; +const iconFiles: Record = { + [Emotion.Curious]: { + imageData: jiboEyeCurious, + }, + [Emotion.Frustrated]: { + imageData: jiboEyeFrustrated, + }, + [Emotion.Happy]: { + imageData: jiboEyeHappy, + }, + [Emotion.Laugh]: { + imageData: jiboEyeLaugh, + }, + [Emotion.No]: { + imageData: jiboEyeNo, + }, + [Emotion.Puzzled]: { + imageData: jiboEyePuzzled, + }, + [Emotion.Sad]: { + imageData: jiboEyeSad, + }, + [Emotion.SadEyes]: { + imageData: jiboEyeSadEyes, + }, + [Emotion.Success]: { + imageData: jiboEyeSuccess, + }, + [Emotion.Thinking]: { + imageData: jiboEyeThinking, + }, + [Emotion.Yes]: { + imageData: jiboEyeYes, + }, + [Icon.Airplane]: { + imageData: jiboEyeAirplane, + }, + [Icon.Apple]: { + imageData: jiboEyeApple, + }, + [Icon.Art]: { + imageData: jiboEyeArt, + }, + [Icon.Bowling]: { + imageData: jiboEyeBowling, + }, + [Icon.Checkmark]: { + imageData: jiboEyeCheckmark, + }, + [Icon.ExclamationPoint]: { + imageData: jiboEyeExclamation, + }, + [Icon.Football]: { + imageData: jiboEyeFootball, + }, + [Icon.Heart]: { + imageData: jiboEyeHeart, + }, + [Icon.Magic]: { + imageData: jiboEyeMagic, + }, + [Icon.Ocean]: { + imageData: jiboEyeOcean, + }, + [Icon.Penguin]: { + imageData: jiboEyePenguin, + }, + [Icon.Rainbow]: { + imageData: jiboEyeRainbow, + }, + [Icon.Robot]: { + imageData: jiboEyeRobot, + }, + [Icon.Rocket]: { + imageData: jiboEyeRocket, + }, + [Icon.Snowflake]: { + imageData: jiboEyeSnowflake, + }, + [Icon.Taco]: { + imageData: jiboEyeTaco, + }, + [Icon.VideoGame]: { + imageData: jiboEyeVideoGame, + }, +}; + +export default class Scratch3VirtualJibo { + runtime: Runtime; + answer: string; + + init(runtime: Runtime) { + this.runtime = runtime; + this.answer = ""; + } + + resetJiboEyeTarget(target: Target, type: string = "eye") { + let bodyTarget = this.getJiboBodyTarget(target); + let eyeTarget = this.getJiboEyeTarget(target); + + if (!isRenderedTarget(bodyTarget) || !isRenderedTarget(eyeTarget)) { + console.warn("Eye could not be reset as the supplied target didn't lead to rendered eye and body targets"); + return false; + } + + if (eyeTarget) { + let mult = type === "eye" ? + 1 : + DEFAULT_JIBO_EYE.diconSize / DEFAULT_JIBO_EYE.dsize; + let newX = bodyTarget.x + DEFAULT_JIBO_EYE.dx * bodyTarget.size; + let newY = bodyTarget.y + DEFAULT_JIBO_EYE.dy * bodyTarget.size; + let newSize = bodyTarget.size * DEFAULT_JIBO_EYE.dsize * mult; + eyeTarget.setXY(newX, newY, null); + eyeTarget.setSize(newSize); + eyeTarget.goToFront(); + } + } + async setSpriteCostume(target: Target, name: string, imageDataURI: string) { + // try to set the costume of the target by name + let foundCostume = this.setCostumeByName(target, name); + + if (!foundCostume) { + console.log("Did not find the costume we wanted. Adding new one"); + // if not, add and set the costume + await this.addCostumeBitmap(target, imageDataURI, "add and set", name); + } + } + getJiboBodyTarget(currentTarget: Target): Target { + // find the jibo-body sprite + let spriteTarget; + if (currentTarget.getName().startsWith(JIBO_BODY)) { + // first see if the current target is a Jibo body + // if so, assume this is the one we want to edit + spriteTarget = currentTarget; + } else if (currentTarget.getName().startsWith(JIBO_EYE)) { + // next see if this is a Jibo eye, and select the corresponding jibo body (same suffix) + let jiboEyeName = currentTarget.getName(); + let suffix = jiboEyeName.substring(jiboEyeName.indexOf(JIBO_EYE) + JIBO_EYE.length); + let matches = this.runtime.targets.filter((target) => target.getName().startsWith(JIBO_BODY + suffix)); + if (matches.length > 0) spriteTarget = matches[0]; + } else { + // otherwise, pick the first Jibo body you see + let matches = this.runtime.targets.filter((target) => target.getName().startsWith(JIBO_BODY)); + if (matches.length > 0) spriteTarget = matches[0]; + } + return spriteTarget; + } + getJiboEyeTarget(currentTarget: Target): Target { + // find the jibo-eye sprite + let spriteTarget; + if (currentTarget.getName().startsWith(JIBO_EYE)) { + // first see if the current target is a Jibo eye + // if so, assume this is the one we want to edit + spriteTarget = currentTarget; + } else if (currentTarget.getName().startsWith(JIBO_BODY)) { + // next see if this is a Jibo body, and select the corresponding jibo eye (same suffix) + let jiboBodyName = currentTarget.getName(); + let suffix = jiboBodyName.substring(jiboBodyName.indexOf(JIBO_BODY) + JIBO_BODY.length); + let matches = this.runtime.targets.filter((target) => target.getName().startsWith(JIBO_EYE + suffix)); + if (matches.length > 0) spriteTarget = matches[0]; + } else { + // otherwise, pick the first Jibo eye you see + let matches = this.runtime.targets.filter((target) => target.getName().startsWith(JIBO_EYE)); + if (matches.length > 0) spriteTarget = matches[0]; + } + return spriteTarget; + } + + async say(text: string, currentTarget: Target) { + let spriteTarget = this.getJiboBodyTarget(currentTarget); + if (spriteTarget) { + // emit the say function + this.runtime.emit('SAY', spriteTarget, 'say', text); + // wait for a bit of time + let wordCount = text.match(/\S+/g).length; + await new Promise((r) => setTimeout(r, 500 * wordCount)); + this.runtime.emit('SAY', spriteTarget, 'say', ''); + } else { + console.log("No Jibo body found"); + } + } + async ask(text: string) { + // wait for stage to get answer + this.runtime.emit('QUESTION', text); + this.answer = await this.answer_receive(); + } + answer_receive(): Promise { + return new Promise((resolve, reject) => { + this.runtime.once('ANSWER', (answer) => { + // TODO this introduces a bug with the sensing blocks, improve if possible + resolve(answer); + }); + }); + } + + /* update the appearance of virtual Jibo's LED*/ + async setLED(color: string, currentTarget: Target) { + // find the jibo-body sprite to edit + let spriteTarget = this.getJiboBodyTarget(currentTarget); + if (spriteTarget) { + // change the Sprite costume + if (color == "random") { + const randomColorIdx = Math.floor( + // exclude random and off + Math.random() * (Object.keys(colorDef).length - 2) + ); + color = Object.keys(colorDef)[randomColorIdx]; + } + + let imageData = colorDef[color].imageData; + await this.setSpriteCostume(spriteTarget, color, imageData); + } else { + console.log("No Jibo body found"); + } + } + + async blink(jiboEye: Target) { + this.resetJiboEyeTarget(jiboEye); + for (let i = 0; i < JIBO_EYE_ANIM.length; i++) { + let costumeName = JIBO_EYE_ANIM[i]; + let imageData = jiboEyeDef[costumeName].imageData; + await this.setSpriteCostume(jiboEye, costumeName, imageData); + await new Promise((r) => setTimeout(r, 50)); + } + } + async jumpTransition(jiboEye: Target, newAnim: string, imageData: string) { + let type = newAnim.includes("Eye") ? "eye" : "icon"; + if (!isRenderedTarget(jiboEye)) { + console.warn("Eye could not be reset as the supplied target wasn't a rendered target"); + return false; + } + + // move up 5 loops + for (let i = 0; i < 5; i++) { + jiboEye.setXY(jiboEye.x, jiboEye.y + 5, null) + await new Promise((r) => setTimeout(r, 50)); + } + // move eye down 7 loops + for (let i = 0; i < 7; i++) { + jiboEye.setXY(jiboEye.x, jiboEye.y - 5, null) + await new Promise((r) => setTimeout(r, 50)); + } + // switch costume + await this.setSpriteCostume(jiboEye, newAnim, imageData); + this.resetJiboEyeTarget(jiboEye, type); + // move up 4 loops + for (let i = 0; i < 4; i++) { + jiboEye.setXY(jiboEye.x, jiboEye.y + 5, null) + await new Promise((r) => setTimeout(r, 50)); + } + // move down 2 loops + for (let i = 0; i < 2; i++) { + jiboEye.setXY(jiboEye.x, jiboEye.y - 5, null) + await new Promise((r) => setTimeout(r, 50)); + } + } + async anim(animation: string, commandType: string, currentTarget: Target) { + // find the jibo-eye sprite to edit + let spriteTarget = this.getJiboEyeTarget(currentTarget); + if (!isRenderedTarget(spriteTarget)) { + console.warn("No rendered jibo-eye target could be found"); + return false; + } + + // change the Sprite costume + let imageDataURI; + //if (commandType == "dance") imageDataURI = danceFiles[animation].imageData; + if (commandType == "emotion" || commandType == "icon") { + imageDataURI = iconFiles[animation].imageData; + await this.jumpTransition(spriteTarget, animation, imageDataURI); + await new Promise((r) => setTimeout(r, 3000)); + await this.jumpTransition(spriteTarget, "Eye1", jiboEyeDef["Eye1"].imageData); + // finish a blink + await this.blink(spriteTarget); + } + } + async lookAt(direction: string, currentTarget: Target) { + // find the jibo-body and jibo-eye sprites to edit + let eyeTarget = this.getJiboEyeTarget(currentTarget); + let bodyTarget = this.getJiboBodyTarget(currentTarget); + if (!isRenderedTarget(eyeTarget) || !(isRenderedTarget(bodyTarget))) { + console.warn("Eye could not be reset as the supplied target wasn't a rendered target"); + return false; + } + + let coords = directionDef[direction].value; + let newX = bodyTarget.x + coords.dx * bodyTarget.size; + let newY = bodyTarget.y + coords.dy * bodyTarget.size; + let xStepSize = (newX - eyeTarget.x) / 10; + let yStepSize = (newY - eyeTarget.y) / 10; + for (let i = 0; i < 10; i++) { + eyeTarget.setXY( + eyeTarget.x + xStepSize, + eyeTarget.y + yStepSize, + null + ); + await new Promise((r) => setTimeout(r, 50)); + } + } + + + // Copied code from /workspace/prg-extension-boilerplate/extensions/src/common/extension/mixins/optional/addCostumes/index.ts + // until I figure out a better way + + /** + * Add a costume to the current sprite based on some image data + * @param {Target} target (e.g. `util.target`) + * @param {ImageData} image What image to use to create the costume + * @param {"add only" | "add and set"} action What action should be applied + * - **_add only_**: generates the costume and append it it to the sprite's costume library + * - **_add and set_**: Both generate the costume (adding it to the sprite's costume library) and set it as the sprite's current costume + * @param {string?} name optional name to attach to the costume + */ + async addCostume(target: Target, image: ImageData, action: "add only" | "add and set", name?: string) { + if (!isRenderedTarget(target)) return console.warn("Costume could not be added as the supplied target wasn't a rendered target"); + + name ??= `virtualJibo_generated_${Date.now()}`; + bitmapAdapter ??= new MockBitmapAdapter(); + urlHelper ??= getUrlHelper(image); + + // storage is of type: https://github.com/LLK/scratch-storage/blob/develop/src/ScratchStorage.js + const { storage } = this.runtime; + const dataFormat = storage.DataFormat.PNG; + const assetType = storage.AssetType.ImageBitmap; + const dataBuffer = await bitmapAdapter.importBitmap(urlHelper.getDataURL(image)); + + const asset = storage.createAsset(assetType, dataFormat, dataBuffer, null, true); + const { assetId } = asset; + const costume = { name, dataFormat, asset, md5: `${assetId}.${dataFormat}`, assetId }; + + await this.runtime.addCostume(costume); + + const { length } = target.getCostumes(); + + target.addCostume(costume, length); + if (action === "add and set") target.setCostume(length); + } + + /** + * Add a costume to the current sprite based on a bitmpa input + * @param {Target} target (e.g. `util.target`) + * @param {string} bitmapImage What image to use to create the costume + * @param {"add only" | "add and set"} action What action should be applied + * - **_add only_**: generates the costume and append it it to the sprite's costume library + * - **_add and set_**: Both generate the costume (adding it to the sprite's costume library) and set it as the sprite's current costume + * @param {string?} name optional name to attach to the costume + */ + async addCostumeBitmap(target: Target, bitmapImage: string, action: "add only" | "add and set", name?: string) { + if (!isRenderedTarget(target)) return console.warn("Costume could not be added as the supplied target wasn't a rendered target"); + + name ??= `virtualJibo_generated_${Date.now()}`; + bitmapAdapter ??= new MockBitmapAdapter(); + //urlHelper ??= getUrlHelper(image); + + // storage is of type: https://github.com/LLK/scratch-storage/blob/develop/src/ScratchStorage.js + const { storage } = this.runtime; + const dataFormat = storage.DataFormat.PNG; + const assetType = storage.AssetType.ImageBitmap; + const dataBuffer = await bitmapAdapter.importBitmap(bitmapImage); + + const asset = storage.createAsset(assetType, dataFormat, dataBuffer, null, true); + const { assetId } = asset; + const costume = { name, dataFormat, asset, md5: `${assetId}.${dataFormat}`, assetId }; + + await this.runtime.addCostume(costume); + + const { length } = target.getCostumes(); + + target.addCostume(costume, length); + if (action === "add and set") target.setCostume(length); + } + + /** + * Add a costume to the current sprite based on same image data + * @param {Target} target (e.g. `util.target`) + * @param {string?} name costume name to look for + */ + setCostumeByName(target: Target, name: string): boolean { + if (!isRenderedTarget(target)) { + console.warn("Costume could not be set as the supplied target wasn't a rendered target"); + return false; + } + + let costumeIdx = target.getCostumeIndexByName(name); + if (costumeIdx >= 0) { + target.setCostume(costumeIdx); + return true; + } + return false; + } +} \ No newline at end of file