- Jump To … +
  ./demo/canvas-001.js ./demo/canvas-002.js ./demo/canvas-003.js ./demo/canvas-004.js ./demo/canvas-005.js ./demo/canvas-006.js ./demo/canvas-007.js ./demo/canvas-008.js ./demo/canvas-009.js ./demo/canvas-010.js ./demo/canvas-011.js ./demo/canvas-012.js ./demo/canvas-013.js ./demo/canvas-014.js ./demo/canvas-015.js ./demo/canvas-015a.js ./demo/canvas-016.js ./demo/canvas-017.js ./demo/canvas-018.js ./demo/canvas-019.js ./demo/canvas-019a.js ./demo/canvas-020.js ./demo/canvas-021.js ./demo/canvas-022.js ./demo/canvas-023.js ./demo/canvas-024.js ./demo/canvas-025.js ./demo/canvas-026.js ./demo/canvas-027.js ./demo/canvas-028.js ./demo/canvas-029.js ./demo/canvas-030.js ./demo/canvas-031.js ./demo/canvas-032.js ./demo/canvas-033.js ./demo/canvas-034.js ./demo/canvas-035.js ./demo/canvas-036.js ./demo/canvas-037.js ./demo/canvas-038.js ./demo/canvas-039.js ./demo/canvas-040.js ./demo/canvas-041.js ./demo/canvas-042.js ./demo/canvas-043.js ./demo/canvas-044.js ./demo/canvas-045.js ./demo/canvas-046.js ./demo/canvas-047.js ./demo/canvas-048.js ./demo/canvas-049.js ./demo/canvas-050.js ./demo/canvas-051.js ./demo/canvas-052.js ./demo/canvas-053.js ./demo/canvas-054.js ./demo/canvas-055.js ./demo/canvas-056.js ./demo/canvas-058.js ./demo/canvas-059.js ./demo/canvas-060.js ./demo/canvas-061.js ./demo/canvas-062.js ./demo/canvas-063.js ./demo/canvas-064.js ./demo/core-001.js ./demo/delaunator-001.js ./demo/delaunator-002.js ./demo/dom-001.js ./demo/dom-002.js ./demo/dom-003.js ./demo/dom-004.js ./demo/dom-005.js ./demo/dom-006.js ./demo/dom-007.js ./demo/dom-008.js ./demo/dom-009.js ./demo/dom-010.js ./demo/dom-011.js ./demo/dom-012.js ./demo/dom-013.js ./demo/dom-015.js ./demo/dom-016.js ./demo/dom-017.js ./demo/dom-018.js ./demo/filters-001.js ./demo/filters-002.js ./demo/filters-002a.js ./demo/filters-003.js ./demo/filters-004.js ./demo/filters-005.js ./demo/filters-006.js ./demo/filters-007.js ./demo/filters-008.js ./demo/filters-009.js ./demo/filters-010.js ./demo/filters-011.js ./demo/filters-012.js ./demo/filters-013.js ./demo/filters-014.js ./demo/filters-015.js ./demo/filters-016.js ./demo/filters-017.js ./demo/filters-018.js ./demo/filters-019.js ./demo/filters-020.js ./demo/filters-021.js ./demo/filters-022.js ./demo/filters-023.js ./demo/filters-024.js ./demo/filters-025.js ./demo/filters-026.js ./demo/filters-027.js ./demo/filters-028.js ./demo/filters-101.js ./demo/filters-102.js ./demo/filters-103.js ./demo/filters-104.js ./demo/filters-105.js ./demo/filters-501.js ./demo/filters-502.js ./demo/filters-503.js ./demo/filters-504.js ./demo/filters-505.js ./demo/mediapipe-001.js ./demo/mediapipe-002.js ./demo/mediapipe-003.js ./demo/modules-001.js ./demo/modules-002.js ./demo/modules-003.js ./demo/modules-004.js ./demo/modules-005.js ./demo/modules-006.js ./demo/packets-001.js ./demo/packets-002.js ./demo/particles-001.js ./demo/particles-002.js ./demo/particles-003.js ./demo/particles-004.js ./demo/particles-005.js ./demo/particles-006.js ./demo/particles-007.js ./demo/particles-008.js ./demo/particles-009.js ./demo/particles-010.js ./demo/particles-011.js ./demo/particles-012.js ./demo/particles-013.js ./demo/particles-014.js ./demo/particles-015.js ./demo/particles-016.js ./demo/rapier-001.js ./demo/snippets-001.js ./demo/snippets-002.js ./demo/snippets-003.js ./demo/snippets-004.js ./demo/snippets-005.js ./demo/snippets-006.js ./demo/tensorflow-001.js ./demo/tensorflow-002.js ./demo/utilities.js

- §

  # Demo Tensorflow 001

  Tensorflow tfjs-models / body-pix experiment - follow my eyes

- §

  Run code

      import * as scrawl from '../source/scrawl.js';

      import { reportSpeed } from './utilities.js';

- §

  #### Scene setup

      const canvas = scrawl.library.artefact.mycanvas;

- §

  #### TensorFlow functionality

  We’ll handle everything in a raw asset object, which a Picture entity can then use as its source

      const myAsset = scrawl.makeRawAsset({

          name: 'tensorflow-model-interpreter',

          userAttributes: [{

- §

  We’re only interested in the metadata generated by the tensorflow model in this demo

              key: 'parts',
              defaultValue: {},
              setter: function (item) {

                  if (item && item.allPoses && item.allPoses.length) {

- §

  @ts-expect-error

                      const { parts, leftEyeX, leftEyeY, leftEye, rightEyeX, rightEyeY, rightEye, wobbleDamper } = this;

                      const segs = item.allPoses[0];

                      segs.keypoints.forEach(s => parts[s.part] = s.position);

                      if (parts.leftEye != null) {

                          this.dirtyData = true;

                          const eye = parts.leftEye;

                          if (leftEyeX.length > wobbleDamper) leftEyeX.shift();
                          if (leftEyeY.length > wobbleDamper) leftEyeY.shift();

                          leftEyeX.push(eye.x);
                          leftEyeY.push(eye.y);

                          leftEye[0] = Math.round(leftEyeX.reduce((a, v) => a + v, 0) / leftEyeX.length);
                          leftEye[1] = Math.round(leftEyeY.reduce((a, v) => a + v, 0) / leftEyeY.length);
                      }

                      if (parts.rightEye != null) {

                          this.dirtyData = true;

                          const eye = parts.rightEye;

                          if (rightEyeX.length > wobbleDamper) rightEyeX.shift();
                          if (rightEyeY.length > wobbleDamper) rightEyeY.shift();

                          rightEyeX.push(eye.x);
                          rightEyeY.push(eye.y);

                          rightEye[0] = Math.round(rightEyeX.reduce((a, v) => a + v, 0) / rightEyeX.length);
                          rightEye[1] = Math.round(rightEyeY.reduce((a, v) => a + v, 0) / rightEyeY.length);
                      }
                  }

- §

  We can also check for image dimensions as that info is also passed on by the model output

                  if (item && item.width && item.height) {

                      if (this.canvasWidth !== item.width) {

                          this.canvasWidth = item.width;
                          this.dirtyData = true;
                      }

                      if (this.canvasHeight !== item.height) {

                          this.canvasHeight = item.height;
                          this.dirtyData = true;
                      }
                  }
              },
          },{
              key: 'leftEyeX',
              defaultValue: [],
              setter: () => {},
          },{
              key: 'leftEyeY',
              defaultValue: [],
              setter: () => {},
          },{
              key: 'rightEyeX',
              defaultValue: [],
              setter: () => {},
          },{
              key: 'rightEyeY',
              defaultValue: [],
              setter: () => {},
          },{
              key: 'wobbleDamper',
              defaultValue: 2,
          },{
              key: 'leftEye',
              defaultValue: [0, 0],
              setter: () => {},
          },{
              key: 'rightEye',
              defaultValue: [0, 0],
              setter: () => {},
          },{
              key: 'canvasWidth',
              defaultValue: 0,
              setter: () => {},
          },{
              key: 'canvasHeight',
              defaultValue: 0,
              setter: () => {},
          }],

- §

  `assetWrapper` is the same as `this` when function is declared with the function keyword

          updateSource: function (assetWrapper) {

              const { element, engine, leftEye, rightEye, canvasWidth, canvasHeight } = assetWrapper;
              const end = 2 * Math.PI;

- §

  Clear the canvas, resizing it if required

              element.width = canvasWidth;
              element.height = canvasHeight;

- §

  Draw our filled circles onto the canvas

              engine.globalAlpha = 0.5;

              engine.fillStyle = 'red';
              engine.beginPath();
              engine.arc(...leftEye, 50, 0, end);
              engine.fill();

              engine.fillStyle = 'orange';
              engine.beginPath();
              engine.arc(...rightEye, 50, 0, end);
              engine.fill();
          },
      });

- §

  The forever loop function, which captures the tensorflow model’s output and passes it on to our raw asset for processing

      const perform = function (net) {

          net.segmentPerson(video.source)
          .then(parts => {

              myAsset.set({parts});
              perform(net);
          })
          .catch(e => console.log(e));
      };

- §

  ##### Import and use livestream

  convenience handle for the media stream asset

      let video;

- §

  Capture the media stream

      scrawl.importMediaStream({
          name: 'device-camera',
          audio: false,
      })
      .then(mycamera => {

          video = mycamera;

- §

  This fixes the issue in Firefox where the media stream will crash Tensorflow if the stream’s video element’s dimensions have not been set @ts-expect-error

          video.source.width = "1280";

- §

  @ts-expect-error

          video.source.height = "720";

- §

  Take the media stream and display it in our canvas element

          scrawl.makePicture({

              name: 'mediastream-video',
              asset: mycamera.name,

              width: '100%',
              height: '100%',

              copyWidth: '100%',
              copyHeight: '100%',
          });

- §

  Start the TensorFlow model

      /* eslint-disable */

- §

  @ts-expect-error

          bodyPix.load()
      /* eslint-enable */
          .then (net => {

- §

  Display the visual generated by our raw asset

              scrawl.makePicture({

                  name: 'tensorflow-data-output',
                  asset: 'tensorflow-model-interpreter',
                  order: 1,

                  dimensions: ['100%', '100%'],
                  copyDimensions: ['100%', '100%'],
              });

- §

  Invoke the forever loop

              perform(net);
          })
          .catch(e => console.log('ERROR: ', e));
      })
      .catch(err => console.log(err.message));

- §

  #### Scene animation

  Function to display frames-per-second data, and other information relevant to the demo

      const report = reportSpeed('#reportmessage');

- §

  Create the Display cycle animation

      scrawl.makeRender({

          name: 'demo-animation',
          target: canvas,
          afterShow: report,
      });

      console.log(scrawl.library);