- Jump To … +
  ./demo/canvas-001.js ./demo/canvas-002.js ./demo/canvas-003.js ./demo/canvas-004.js ./demo/canvas-005.js ./demo/canvas-006.js ./demo/canvas-007.js ./demo/canvas-008.js ./demo/canvas-009.js ./demo/canvas-010.js ./demo/canvas-011.js ./demo/canvas-012.js ./demo/canvas-013.js ./demo/canvas-014.js ./demo/canvas-015.js ./demo/canvas-015a.js ./demo/canvas-016.js ./demo/canvas-017.js ./demo/canvas-018.js ./demo/canvas-019.js ./demo/canvas-019a.js ./demo/canvas-020.js ./demo/canvas-021.js ./demo/canvas-022.js ./demo/canvas-023.js ./demo/canvas-024.js ./demo/canvas-025.js ./demo/canvas-026.js ./demo/canvas-027.js ./demo/canvas-028.js ./demo/canvas-029.js ./demo/canvas-030.js ./demo/canvas-031.js ./demo/canvas-032.js ./demo/canvas-033.js ./demo/canvas-034.js ./demo/canvas-035.js ./demo/canvas-036.js ./demo/canvas-037.js ./demo/canvas-038.js ./demo/canvas-039.js ./demo/canvas-040.js ./demo/canvas-041.js ./demo/canvas-042.js ./demo/canvas-043.js ./demo/canvas-044.js ./demo/canvas-045.js ./demo/canvas-046.js ./demo/canvas-047.js ./demo/canvas-048.js ./demo/canvas-049.js ./demo/canvas-050.js ./demo/canvas-051.js ./demo/canvas-052.js ./demo/canvas-053.js ./demo/canvas-054.js ./demo/canvas-055.js ./demo/canvas-056.js ./demo/canvas-058.js ./demo/canvas-059.js ./demo/canvas-060.js ./demo/canvas-061.js ./demo/canvas-062.js ./demo/canvas-063.js ./demo/canvas-064.js ./demo/core-001.js ./demo/delaunator-001.js ./demo/delaunator-002.js ./demo/dom-001.js ./demo/dom-002.js ./demo/dom-003.js ./demo/dom-004.js ./demo/dom-005.js ./demo/dom-006.js ./demo/dom-007.js ./demo/dom-008.js ./demo/dom-009.js ./demo/dom-010.js ./demo/dom-011.js ./demo/dom-012.js ./demo/dom-013.js ./demo/dom-015.js ./demo/dom-016.js ./demo/dom-017.js ./demo/dom-018.js ./demo/filters-001.js ./demo/filters-002.js ./demo/filters-002a.js ./demo/filters-003.js ./demo/filters-004.js ./demo/filters-005.js ./demo/filters-006.js ./demo/filters-007.js ./demo/filters-008.js ./demo/filters-009.js ./demo/filters-010.js ./demo/filters-011.js ./demo/filters-012.js ./demo/filters-013.js ./demo/filters-014.js ./demo/filters-015.js ./demo/filters-016.js ./demo/filters-017.js ./demo/filters-018.js ./demo/filters-019.js ./demo/filters-020.js ./demo/filters-021.js ./demo/filters-022.js ./demo/filters-023.js ./demo/filters-024.js ./demo/filters-025.js ./demo/filters-026.js ./demo/filters-027.js ./demo/filters-028.js ./demo/filters-101.js ./demo/filters-102.js ./demo/filters-103.js ./demo/filters-104.js ./demo/filters-105.js ./demo/filters-501.js ./demo/filters-502.js ./demo/filters-503.js ./demo/filters-504.js ./demo/filters-505.js ./demo/mediapipe-001.js ./demo/mediapipe-002.js ./demo/mediapipe-003.js ./demo/modules-001.js ./demo/modules-002.js ./demo/modules-003.js ./demo/modules-004.js ./demo/modules-005.js ./demo/modules-006.js ./demo/packets-001.js ./demo/packets-002.js ./demo/particles-001.js ./demo/particles-002.js ./demo/particles-003.js ./demo/particles-004.js ./demo/particles-005.js ./demo/particles-006.js ./demo/particles-007.js ./demo/particles-008.js ./demo/particles-009.js ./demo/particles-010.js ./demo/particles-011.js ./demo/particles-012.js ./demo/particles-013.js ./demo/particles-014.js ./demo/particles-015.js ./demo/particles-016.js ./demo/rapier-001.js ./demo/snippets-001.js ./demo/snippets-002.js ./demo/snippets-003.js ./demo/snippets-004.js ./demo/snippets-005.js ./demo/snippets-006.js ./demo/tensorflow-001.js ./demo/tensorflow-002.js ./demo/utilities.js

- §

  # Demo Tensorflow 002

  Tensorflow tfjs-models / body-pix experiment - model image output

- §

  Run code

      import * as scrawl from '../source/scrawl.js';

      import { reportSpeed } from './utilities.js';

- §

  #### Scene setup

      const canvas = scrawl.library.artefact.mycanvas;

      scrawl.makeFilter({
          name: 'grayscale',
          method: 'grayscale',
      }).clone({
          name: 'sepia',
          method: 'sepia',
      }).clone({
          name: 'invert',
          method: 'invert',
      }).clone({
          name: 'red',
          method: 'red',
      });

      scrawl.makeFilter({
          name: 'pixelate',
          method: 'pixelate',
          tileWidth: 20,
          tileHeight: 20,
          offsetX: 8,
          offsetY: 8,
      });

      scrawl.makeFilter({
          name: 'background-blur',
          method: 'gaussianBlur',
          radius: 20,
      });

      scrawl.makeFilter({
          name: 'body-blur',
          method: 'gaussianBlur',
          radius: 10,
      });

- §

  #### TensorFlow functionality

  We’ll handle everything in a raw asset object, which a Picture entity can then use as its source

      const myAsset = scrawl.makeRawAsset({

          name: 'tensorflow-model-interpreter',

          userAttributes: [{

- §

  We’re only interested in the pixel allocations generated by the tensorflow model in this demo

              key: 'data',
              defaultValue: [],
              setter: function (item) {

                  if (item && item.width && item.height && item.data) {

                      this.canvasWidth = item.width;
                      this.canvasHeight = item.height;
                      this.data = item.data;
                      this.dirtyData = true;
                  }
              },
          },{
              key: 'canvasWidth',
              defaultValue: 0,
              setter: () => {},
          },{
              key: 'canvasHeight',
              defaultValue: 0,
              setter: () => {},
          }],

          updateSource: function (assetWrapper) {

              const { element, engine, canvasWidth, canvasHeight, data } = assetWrapper;

              if (canvasWidth && canvasHeight && data) {

                  const segLength = canvasWidth * canvasHeight,
                      imageDataLen = segLength * 4,
                      imageArray = new Uint8ClampedArray(imageDataLen);

                  for (let i = 0, o = 0; i < segLength; i++) {

                      o = (i * 4) + 3;
                      if (data[i]) imageArray[o] = 255;
                  }

                  const iData = new ImageData(imageArray, canvasWidth, canvasHeight);

- §

  Clear the canvas, resizing it if required

                  element.width = canvasWidth;
                  element.height = canvasHeight;

                  engine.putImageData(iData, 0, 0);
              }
          },
      });

- §

  The forever loop function, which captures the tensorflow model’s output and passes it on to our raw asset for processing

      const perform = function (net) {

          net.segmentPerson(video.source)
          .then(data => {

              myAsset.set({data});
              perform(net);
          })
          .catch(e => console.log(e));
      };

- §

  ##### Import and use livestream

  convenience handle for the media stream asset

      let video, myBackground, myOutline;

- §

  Capture the media stream

      scrawl.importMediaStream({
          name: 'device-camera',
          audio: false,
      })
      .then(mycamera => {

          video = mycamera;

- §

  This fixes the issue in Firefox where the media stream will crash Tensorflow if the stream’s video element’s dimensions have not been set @ts-expect-error

          video.source.width = "1280";

- §

  @ts-expect-error

          video.source.height = "720";

- §

  Take the media stream and display it in our canvas element

          myBackground = scrawl.makePicture({

              name: 'background',
              asset: mycamera.name,
              order: 2,

              width: '100%',
              height: '100%',

              copyWidth: '80%',
              copyHeight: '80%',
              copyStartX: '10%',
              copyStartY: '10%',

              globalCompositeOperation: 'destination-over',
          });

          myBackground.clone({

              name: 'body',
              order: 1,
              globalCompositeOperation: 'source-in',
          });

- §

  Start the TensorFlow model

      /* eslint-disable */

- §

  @ts-expect-error

          bodyPix.load()
      /* eslint-enable */
          .then (net => {

- §

  Display the visual generated by our raw asset

              myOutline = scrawl.makePicture({

                  name: 'outline',
                  asset: 'tensorflow-model-interpreter',
                  order: 0,

                  width: '100%',
                  height: '100%',

                  copyWidth: '80%',
                  copyHeight: '80%',
                  copyStartX: '10%',
                  copyStartY: '10%',

- §

  We blur here to make the outline merge into the background

  - this does slow the demo down, but needs must.

                  filters: ['body-blur'],
              });

- §

  Invoke the forever loop

              perform(net);
          })
          .catch(e => console.log('ERROR: ', e));
      })
      .catch(err => console.log(err.message));

- §

  #### Scene animation

  Function to display frames-per-second data, and other information relevant to the demo

      const report = reportSpeed('#reportmessage');

- §

  Create the Display cycle animation

      scrawl.makeRender({

          name: 'demo-animation',
          target: canvas,
          afterShow: report,
      });

- §

  #### User interaction

  Event listeners

      scrawl.addNativeListener(['input', 'change'], (e) => {

          e.preventDefault();
          e.returnValue = false;

          if (e && e.target) {

              const id = e.target.id,
                  val = e.target.value;

              if ('backgroundFilter' === id) {

                  myBackground.clearFilters();
                  if (val) myBackground.addFilters(val);
              }
              else {

                  if ('1' === val) myOutline.addFilters('body-blur');
                  else myOutline.clearFilters();
              }
          }
      }, '.controlItem');

- §

  Set DOM form initial input values @ts-expect-error

      document.querySelector('#backgroundFilter').value = '';

- §

  @ts-expect-error

      document.querySelector('#outlineFilter').value = '1';

- §

  #### Development and testing

      console.log(scrawl.library);