- Jump To … +
  ./demo/canvas-001.js ./demo/canvas-002.js ./demo/canvas-003.js ./demo/canvas-004.js ./demo/canvas-005.js ./demo/canvas-006.js ./demo/canvas-007.js ./demo/canvas-008.js ./demo/canvas-009.js ./demo/canvas-010.js ./demo/canvas-011.js ./demo/canvas-012.js ./demo/canvas-013.js ./demo/canvas-014.js ./demo/canvas-015.js ./demo/canvas-015a.js ./demo/canvas-016.js ./demo/canvas-017.js ./demo/canvas-018.js ./demo/canvas-019.js ./demo/canvas-019a.js ./demo/canvas-020.js ./demo/canvas-021.js ./demo/canvas-022.js ./demo/canvas-023.js ./demo/canvas-024.js ./demo/canvas-025.js ./demo/canvas-026.js ./demo/canvas-027.js ./demo/canvas-028.js ./demo/canvas-029.js ./demo/canvas-030.js ./demo/canvas-031.js ./demo/canvas-032.js ./demo/canvas-033.js ./demo/canvas-034.js ./demo/canvas-035.js ./demo/canvas-036.js ./demo/canvas-037.js ./demo/canvas-038.js ./demo/canvas-039.js ./demo/canvas-040.js ./demo/canvas-041.js ./demo/canvas-042.js ./demo/canvas-043.js ./demo/canvas-044.js ./demo/canvas-045.js ./demo/canvas-046.js ./demo/canvas-047.js ./demo/canvas-048.js ./demo/canvas-049.js ./demo/canvas-050.js ./demo/canvas-051.js ./demo/canvas-052.js ./demo/canvas-053.js ./demo/canvas-054.js ./demo/canvas-055.js ./demo/canvas-056.js ./demo/canvas-058.js ./demo/canvas-059.js ./demo/canvas-060.js ./demo/canvas-061.js ./demo/canvas-062.js ./demo/canvas-063.js ./demo/canvas-064.js ./demo/core-001.js ./demo/delaunator-001.js ./demo/delaunator-002.js ./demo/dom-001.js ./demo/dom-002.js ./demo/dom-003.js ./demo/dom-004.js ./demo/dom-005.js ./demo/dom-006.js ./demo/dom-007.js ./demo/dom-008.js ./demo/dom-009.js ./demo/dom-010.js ./demo/dom-011.js ./demo/dom-012.js ./demo/dom-013.js ./demo/dom-015.js ./demo/dom-016.js ./demo/dom-017.js ./demo/dom-018.js ./demo/filters-001.js ./demo/filters-002.js ./demo/filters-002a.js ./demo/filters-003.js ./demo/filters-004.js ./demo/filters-005.js ./demo/filters-006.js ./demo/filters-007.js ./demo/filters-008.js ./demo/filters-009.js ./demo/filters-010.js ./demo/filters-011.js ./demo/filters-012.js ./demo/filters-013.js ./demo/filters-014.js ./demo/filters-015.js ./demo/filters-016.js ./demo/filters-017.js ./demo/filters-018.js ./demo/filters-019.js ./demo/filters-020.js ./demo/filters-021.js ./demo/filters-022.js ./demo/filters-023.js ./demo/filters-024.js ./demo/filters-025.js ./demo/filters-026.js ./demo/filters-027.js ./demo/filters-028.js ./demo/filters-101.js ./demo/filters-102.js ./demo/filters-103.js ./demo/filters-104.js ./demo/filters-105.js ./demo/filters-501.js ./demo/filters-502.js ./demo/filters-503.js ./demo/filters-504.js ./demo/filters-505.js ./demo/mediapipe-001.js ./demo/mediapipe-002.js ./demo/mediapipe-003.js ./demo/modules-001.js ./demo/modules-002.js ./demo/modules-003.js ./demo/modules-004.js ./demo/modules-005.js ./demo/modules-006.js ./demo/packets-001.js ./demo/packets-002.js ./demo/particles-001.js ./demo/particles-002.js ./demo/particles-003.js ./demo/particles-004.js ./demo/particles-005.js ./demo/particles-006.js ./demo/particles-007.js ./demo/particles-008.js ./demo/particles-009.js ./demo/particles-010.js ./demo/particles-011.js ./demo/particles-012.js ./demo/particles-013.js ./demo/particles-014.js ./demo/particles-015.js ./demo/particles-016.js ./demo/rapier-001.js ./demo/snippets-001.js ./demo/snippets-002.js ./demo/snippets-003.js ./demo/snippets-004.js ./demo/snippets-005.js ./demo/snippets-006.js ./demo/tensorflow-001.js ./demo/tensorflow-002.js ./demo/utilities.js

- §

  # Demo Mediapipe 003

  MediaPipe Face Mesh - working with the mesh coordinates

- §

  Run code

      import * as scrawl from '../source/scrawl.js';

      import { reportSpeed } from './utilities.js';

- §

  Magic numbers

      const width = 1280,
          height = 720;

- §

  #### Scene setup

      const canvas = scrawl.library.artefact.mycanvas;

- §

  Create and manipulate the face worm

      let wormStart = 220,
          wormEnd = 270;

      const wormstartInput = document.querySelector('#wormstart'),
          wormendInput = document.querySelector('#wormend');

- §

  @ts-expect-error

      wormstartInput.value = wormStart;

- §

  @ts-expect-error

      wormendInput.value = wormEnd;

      const worm = scrawl.makePolyline({

          name: 'face-worm',

          strokeStyle: 'red',
          lineJoin: 'round',
          lineCap: 'round',
          lineWidth: 3,

          method: 'draw',

          mapToPins: true,
          tension: 0.3,
      });

      const updateLabelsAndWorm = function (asset) {

          const { entitys, mesh } = asset

          if (mesh.length && Array.isArray(mesh[0])) {

              const face = mesh[0];

- §

  We only need to create the labels once

              if (!entitys.length) {

                  face.forEach((coord, index) => {

                      entitys.push(scrawl.makePhrase({
                          name: `label-${index}`,
                          text: `${index}`,
                          handle: ['center', 'center'],
                          width: 20,
                          justify: 'center',
                          font: '10px Arial',
                      }));
                  });
              }

- §

  Update label coordinates with new mesh data

              entitys.forEach((e, index) => {

                  const coord = face[index];

                  const {x, y} = coord;

                  e.set({
                      startX: `${x * 100}%`,
                      startY: `${y * 100}%`,
                  });
              });

- §

  Check for perilous user input

              if (isNaN(wormStart)) wormStart = 0;
              if (wormStart < 0) wormStart = 0;
              if (wormStart > 468) wormStart = 468;

              if (isNaN(wormEnd)) wormEnd = 0;
              if (wormEnd < 0) wormEnd = 0;
              if (wormEnd > 468) wormEnd = 468;

              if (wormEnd < wormStart) {

                  const temp = wormStart;
                  wormStart = wormEnd;
                  wormEnd = temp;
              }

- §

  @ts-expect-error

              wormstartInput.value = wormStart;

- §

  @ts-expect-error

              wormendInput.value = wormEnd;

- §

  Update the worm’s pins

              worm.set({
                  pins: entitys.slice(wormStart, wormEnd),
              });
          }
      };

- §

  #### MediaPipe functionality

  We’ll handle everything in a raw asset object

      const myAsset = scrawl.makeRawAsset({

          name: 'mediapipe-model-interpreter',

          userAttributes: [{

              key: 'mesh',
              defaultValue: false,
              setter: function (item) {

                  if (item) {

                      const {image:img, multiFaceLandmarks:mesh} = item;

                      if (img) {

                          this.canvasWidth =  img.width;
                          this.canvasHeight = img.height;
                      }

                      if (mesh) this.mesh = mesh;

                      this.dirtyData = true;
                  }
              },
          },{
              key: 'entitys',
              defaultValue: [],
              setter: () => {},
          },{
              key: 'canvasWidth',
              defaultValue: 0,
              setter: () => {},
          },{
              key: 'canvasHeight',
              defaultValue: 0,
              setter: () => {},
          }],

          updateSource: function (assetWrapper) {

              const { mesh } = assetWrapper;

              if (mesh && mesh.length) updateLabelsAndWorm(this);
          },
      });

- §

  The forever loop function, which captures the MediaPipe model’s output and passes it on to our raw asset for processing

      const perform = function (mesh) {

          myAsset.set({ mesh });

- §

  To get the raw asset working, something needs to subscribe to it - even though we’re not using the asset to output any graphics in this demo. We only need to create the Picture entity once.

          if (!output) output = scrawl.makePicture({

              name: 'output',
              asset: 'mediapipe-model-interpreter',
          });
      };

- §

  ##### Import and use media stream

      let video, model, output;

- §

  Capture the media stream

      scrawl.importMediaStream({

          name: 'device-camera',
          audio: false,
      })
      .then(mycamera => {

          video = mycamera;

- §

  @ts-expect-error

          video.source.width = width;

- §

  @ts-expect-error

          video.source.height = height;

          scrawl.makePicture({

              name: 'background',
              asset: mycamera.name,

              dimensions: ['100%', '100%'],
              copyDimensions: ['100%', '100%'],

              globalAlpha: 0.2,
          });

- §

  Start the MediaPipe model

      /* eslint-disable */

- §

  @ts-expect-error

          model = new FaceMesh({

      /* eslint-enable */
              locateFile: (file) => `https://cdn.jsdelivr.net/npm/@mediapipe/face_mesh/${file}`,
          });

          model.setOptions({
              maxNumFaces: 1,
              minDetectionConfidence: 0.5,
              minTrackingConfidence: 0.5
          });

          model.onResults(perform);

- §

  Use MediaPipe’s camera functionality to get updates to the forever loop

      /* eslint-disable */

- §

  @ts-expect-error

          const mediaPipeCamera = new Camera(video.source, {

      /* eslint-enable */
              onFrame: async () => {

                  await model.send({image: video.source});
              },

              width,
              height,
          });

          mediaPipeCamera.start();
      })
      .catch(err => console.log(err.message));

- §

  #### Scene animation

  Function to display frames-per-second data, and other information relevant to the demo

      const report = reportSpeed('#reportmessage');

- §

  Create the Display cycle animation

      scrawl.makeRender({

          name: 'demo-animation',
          target: canvas,
          afterShow: report,
      });

- §

  #### User interaction

      scrawl.addNativeListener(['input', 'change'], (e) => {

          if (e && e.target) {

              e.preventDefault();
              e.stopPropagation();

              const target = e.target;

              if ('wormstart' === target.id) wormStart = parseInt(target.value, 10);
              else if ('wormend' === target.id) wormEnd = parseInt(target.value, 10);
          }

      }, '.controlItem');

- §

  #### Development and testing

      console.log(scrawl.library);