mirror of
https://github.com/reanimate/reanimate.git
synced 2026-09-12 00:23:08 +00:00
Voice control (#75)
Add voice control via Reanimate.Voice. Include rtfd article. Former-commit-id: f80bb58031f6dfbb83bc8f188c17013afcb2dfd9
This commit is contained in:
David Himmelstrup
GitHub
parent
d92660fe18
commit
7e1c7ea511
16 changed files with 1896 additions and 46 deletions
153
examples/voice_advanced.hs
Normal file
153
examples/voice_advanced.hs
Normal file
|
|
@ -0,0 +1,153 @@
|
|||
#!/usr/bin/env stack
|
||||
-- stack --resolver lts-15.04 runghc --package reanimate
|
||||
{-# LANGUAGE OverloadedStrings #-}
|
||||
{-# LANGUAGE ApplicativeDo #-}
|
||||
module Main where
|
||||
|
||||
import Control.Monad
|
||||
import qualified Data.Text as T
|
||||
import Reanimate
|
||||
import Reanimate.Ease
|
||||
import Reanimate.Voice
|
||||
import Reanimate.Builtin.Documentation
|
||||
import Geom2D.CubicBezier ( QuadBezier(..)
|
||||
, evalBezier
|
||||
, Point(..)
|
||||
)
|
||||
import Graphics.SvgTree ( ElementRef(..) )
|
||||
|
||||
transcript :: Transcript
|
||||
transcript = loadTranscript "voice_advanced.txt"
|
||||
|
||||
main :: IO ()
|
||||
main = reanimate $ sceneAnimation $ do
|
||||
bg <- newSpriteSVG $ mkBackgroundPixel rtfdBackgroundColor
|
||||
spriteZ bg (-100)
|
||||
newSpriteSVG_ $ mkGroup
|
||||
[withStrokeColor "black" $ mkLine (-screenWidth, 0) (screenWidth, 0)]
|
||||
|
||||
centerTxt <- textHandler
|
||||
|
||||
flashEffect
|
||||
circleEffect
|
||||
squareEffect
|
||||
finalEffect
|
||||
|
||||
waitOn $ forM_ (transcriptWords transcript) $ \tword -> fork $ do
|
||||
wait (wordStart tword)
|
||||
writeVar centerTxt $ wordReference tword
|
||||
|
||||
wait 2
|
||||
|
||||
wordDuration :: TWord -> Double
|
||||
wordDuration tword = wordEnd tword - wordStart tword
|
||||
|
||||
--
|
||||
finalEffect :: Scene s ()
|
||||
finalEffect = fork $ do
|
||||
let begin = findWord transcript ["final"] "circles"
|
||||
ends = findWords transcript ["final"] "flash"
|
||||
path = QuadBezier (Point 6 (-radius)) (Point 0 6) (Point (-6) (-radius))
|
||||
radius = 0.3
|
||||
wait (wordStart begin)
|
||||
ss <- fork $ replicateM 3 (circleSprite radius path <* wait 0.2)
|
||||
mapM_ (flip spriteZ (-1)) ss
|
||||
forM_ (zip ss ends) $ \(s, end) -> fork $ do
|
||||
spriteMap s flipXAxis
|
||||
wait (wordStart end - wordStart begin)
|
||||
destroySprite s
|
||||
|
||||
-- square effect
|
||||
squareEffect :: Scene s ()
|
||||
squareEffect = fork $ do
|
||||
let
|
||||
begin = findWord transcript [] "square"
|
||||
end = findWord transcript ["middle"] "square"
|
||||
path =
|
||||
QuadBezier (Point 6 (-size / 2)) (Point 0 6) (Point (-6) (-size / 2))
|
||||
size = 1
|
||||
wait (wordStart begin)
|
||||
s <- squareSprite size path
|
||||
spriteMap s (rotate 180)
|
||||
spriteZ s (-1)
|
||||
wait (wordStart end + wordDuration end / 2 - wordStart begin)
|
||||
destroySprite s
|
||||
|
||||
-- circle effect
|
||||
circleEffect :: Scene s ()
|
||||
circleEffect = fork $ do
|
||||
let begin = findWord transcript [] "circle"
|
||||
end = findWord transcript ["middle"] "circle"
|
||||
path = QuadBezier (Point 6 (-radius)) (Point 0 6) (Point (-6) (-radius))
|
||||
radius = 0.3
|
||||
wait (wordStart begin)
|
||||
s <- circleSprite radius path
|
||||
spriteZ s (-1)
|
||||
wait (wordStart end + wordDuration end / 2 - wordStart begin)
|
||||
destroySprite s
|
||||
|
||||
-- flash effect
|
||||
flashEffect :: Scene s ()
|
||||
flashEffect = forM_ (findWords transcript [] "flash") $ \flashWord -> fork $ do
|
||||
wait (wordStart flashWord)
|
||||
flash <- newSpriteSVG $ mkBackground "black"
|
||||
spriteTween flash (wordDuration flashWord)
|
||||
$ \t -> withGroupOpacity (fromToS 0 0.7 $ (powerS 2 . reverseS) t)
|
||||
wait (wordDuration flashWord)
|
||||
destroySprite flash
|
||||
|
||||
--------------------------------------------------------------------------
|
||||
-- Helpers and sprites
|
||||
|
||||
textHandler :: Scene s (Var s T.Text)
|
||||
textHandler = simpleVar render T.empty
|
||||
where
|
||||
render txt =
|
||||
let txtSvg = translate 0 (-0.25) $ centerX $ latex txt
|
||||
activeWidth = svgWidth txtSvg + 0.5
|
||||
in mkGroup
|
||||
[ withStrokeWidth 0 $ withFillColorPixel rtfdBackgroundColor $ mkRect
|
||||
activeWidth
|
||||
1
|
||||
, txtSvg
|
||||
, withStrokeColor "black"
|
||||
$ mkLine (activeWidth / 2, 0.5) (activeWidth / 2, -0.5)
|
||||
, withStrokeColor "black"
|
||||
$ mkLine (-activeWidth / 2, 0.5) (-activeWidth / 2, -0.5)
|
||||
]
|
||||
|
||||
circleSprite :: Double -> QuadBezier Double -> Scene s (Sprite s)
|
||||
circleSprite radius path = newSprite $ do
|
||||
t <- spriteT
|
||||
d <- spriteDuration
|
||||
pure
|
||||
$ let Point x y = evalBezier path (t / d)
|
||||
in mkGroup
|
||||
[ mkClipPath "circle-mask"
|
||||
$ removeGroups
|
||||
$ translate 0 (screenHeight / 2)
|
||||
$ withFillColorPixel rtfdBackgroundColor
|
||||
$ mkRect screenWidth screenHeight
|
||||
, withClipPathRef (Ref "circle-mask") $ translate x y $ mkCircle
|
||||
radius
|
||||
]
|
||||
|
||||
squareSprite :: Double -> QuadBezier Double -> Scene s (Sprite s)
|
||||
squareSprite size path = newSprite $ do
|
||||
t <- spriteT
|
||||
d <- spriteDuration
|
||||
pure
|
||||
$ let Point x y = evalBezier path (t / d)
|
||||
in mkGroup
|
||||
[ mkClipPath "square-mask"
|
||||
$ removeGroups
|
||||
$ translate 0 (screenHeight / 2)
|
||||
$ withFillColorPixel rtfdBackgroundColor
|
||||
$ mkRect screenWidth screenHeight
|
||||
, withClipPathRef (Ref "square-mask")
|
||||
$ translate x y
|
||||
$ rotate (t / d * 360)
|
||||
$ withFillOpacity 0
|
||||
$ withStrokeColor "black"
|
||||
$ mkRect size size
|
||||
]
|
||||
20
examples/voice_advanced.txt
Normal file
20
examples/voice_advanced.txt
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
Everything in this animation is timed by my voice.
|
||||
Every flash, every circle, every square.
|
||||
|
||||
[middle]
|
||||
I decide when everything begins and ends.
|
||||
Time has run out for the square
|
||||
and now the circle
|
||||
|
||||
[final]
|
||||
Let's spawn three more circles
|
||||
and destroy them in a
|
||||
|
||||
|
||||
flash
|
||||
|
||||
|
||||
flash
|
||||
|
||||
|
||||
flash
|
||||
56
examples/voice_fake.hs
Normal file
56
examples/voice_fake.hs
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
#!/usr/bin/env stack
|
||||
-- stack --resolver lts-15.04 runghc --package reanimate
|
||||
{-# LANGUAGE OverloadedStrings #-}
|
||||
{-# LANGUAGE ApplicativeDo #-}
|
||||
module Main where
|
||||
|
||||
import Control.Monad
|
||||
import qualified Data.Text as T
|
||||
import Reanimate
|
||||
import Reanimate.Voice
|
||||
import Reanimate.Builtin.Documentation
|
||||
import Graphics.SvgTree ( ElementRef(..) )
|
||||
|
||||
transcript :: Transcript
|
||||
transcript =
|
||||
fakeTranscript
|
||||
"There is no audio\n\n\
|
||||
\for this transcript....\n\n\n\
|
||||
\Timings are fake,\n\n\
|
||||
\which is quite useful\n\n\
|
||||
\during development"
|
||||
|
||||
main :: IO ()
|
||||
main = reanimate $ sceneAnimation $ do
|
||||
newSpriteSVG_ $ mkBackgroundPixel rtfdBackgroundColor
|
||||
waitOn $ forM_ (splitTranscript transcript) $ \(svg, tword) -> do
|
||||
highlighted <- newVar 0
|
||||
void $ newSprite $ do
|
||||
v <- unVar highlighted
|
||||
pure $ centerUsing (latex $ transcriptText transcript) $ masked
|
||||
(wordKey tword)
|
||||
v
|
||||
svg
|
||||
(withFillColor "grey" $ mkRect 1 1)
|
||||
(withFillColor "black" $ mkRect 1 1)
|
||||
fork $ do
|
||||
wait (wordStart tword)
|
||||
let dur = wordEnd tword - wordStart tword
|
||||
tweenVar highlighted dur $ \v -> fromToS v 1
|
||||
wait 2
|
||||
where
|
||||
wordKey tword =
|
||||
T.unpack (wordReference tword) ++ show (wordStartOffset tword)
|
||||
|
||||
{-# INLINE masked #-}
|
||||
masked :: String -> Double -> SVG -> SVG -> SVG -> SVG
|
||||
masked key t maskSVG srcSVG dstSVG = mkGroup
|
||||
[ mkClipPath label $ removeGroups maskSVG
|
||||
, withClipPathRef (Ref label)
|
||||
$ translate (x - w / 2 + w * t) y (scaleToSize w screenHeight dstSVG)
|
||||
, withClipPathRef (Ref label)
|
||||
$ translate (x + w / 2 + w * t) y (scaleToSize w screenHeight srcSVG)
|
||||
]
|
||||
where
|
||||
label = "word-mask-" ++ key
|
||||
(x, y, w, _h) = boundingBox maskSVG
|
||||
49
examples/voice_transcript.hs
Normal file
49
examples/voice_transcript.hs
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
#!/usr/bin/env stack
|
||||
-- stack --resolver lts-15.04 runghc --package reanimate
|
||||
{-# LANGUAGE OverloadedStrings #-}
|
||||
{-# LANGUAGE ApplicativeDo #-}
|
||||
module Main where
|
||||
|
||||
import Control.Monad
|
||||
import qualified Data.Text as T
|
||||
import Reanimate
|
||||
import Reanimate.Voice
|
||||
import Reanimate.Builtin.Documentation
|
||||
import Graphics.SvgTree ( ElementRef(..) )
|
||||
|
||||
transcript :: Transcript
|
||||
transcript = loadTranscript "voice_transcript.txt"
|
||||
|
||||
main :: IO ()
|
||||
main = reanimate $ sceneAnimation $ do
|
||||
newSpriteSVG_ $ mkBackgroundPixel rtfdBackgroundColor
|
||||
waitOn $ forM_ (splitTranscript transcript) $ \(svg, tword) -> fork $ do
|
||||
highlighted <- newVar 0
|
||||
newSprite_ $ do
|
||||
v <- unVar highlighted
|
||||
pure $ centerUsing (latex $ transcriptText transcript) $ masked
|
||||
(wordKey tword)
|
||||
v
|
||||
svg
|
||||
(withFillColor "grey" $ mkRect 1 1)
|
||||
(withFillColor "black" $ mkRect 1 1)
|
||||
wait (wordStart tword)
|
||||
let dur = wordEnd tword - wordStart tword
|
||||
tweenVar highlighted dur $ \v -> fromToS v 1
|
||||
wait 2
|
||||
where
|
||||
wordKey tword =
|
||||
T.unpack (wordReference tword) ++ show (wordStartOffset tword)
|
||||
|
||||
{-# INLINE masked #-}
|
||||
masked :: String -> Double -> SVG -> SVG -> SVG -> SVG
|
||||
masked key t maskSVG srcSVG dstSVG = mkGroup
|
||||
[ mkClipPath label $ removeGroups maskSVG
|
||||
, withClipPathRef (Ref label)
|
||||
$ translate (x - w / 2 + w * t) y (scaleToSize w screenHeight dstSVG)
|
||||
, withClipPathRef (Ref label)
|
||||
$ translate (x + w / 2 + w * t) y (scaleToSize w screenHeight srcSVG)
|
||||
]
|
||||
where
|
||||
label = "word-mask-" ++ key
|
||||
(x, y, w, _h) = boundingBox maskSVG
|
||||
753
examples/voice_transcript.json
Normal file
753
examples/voice_transcript.json
Normal file
|
|
@ -0,0 +1,753 @@
|
|||
{
|
||||
"transcript": "This transcript is synced to\n\nmy voice automatically.\n\nThe beginning and end of each\n\nword is recorded precisely\n\nand can be used to trigger\n\nevents in the animation.\n",
|
||||
"words": [
|
||||
{
|
||||
"alignedWord": "this",
|
||||
"case": "success",
|
||||
"end": 0.62,
|
||||
"endOffset": 4,
|
||||
"phones": [
|
||||
{
|
||||
"duration": 0.08,
|
||||
"phone": "dh_B"
|
||||
},
|
||||
{
|
||||
"duration": 0.09,
|
||||
"phone": "ih_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.1,
|
||||
"phone": "s_E"
|
||||
}
|
||||
],
|
||||
"start": 0.35,
|
||||
"startOffset": 0,
|
||||
"word": "This"
|
||||
},
|
||||
{
|
||||
"alignedWord": "transcript",
|
||||
"case": "success",
|
||||
"end": 1.21,
|
||||
"endOffset": 15,
|
||||
"phones": [
|
||||
{
|
||||
"duration": 0.06,
|
||||
"phone": "t_B"
|
||||
},
|
||||
{
|
||||
"duration": 0.06,
|
||||
"phone": "r_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.06,
|
||||
"phone": "ae_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.08,
|
||||
"phone": "n_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.05,
|
||||
"phone": "s_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.06,
|
||||
"phone": "k_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.01,
|
||||
"phone": "r_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.07,
|
||||
"phone": "ih_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.07,
|
||||
"phone": "p_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.04,
|
||||
"phone": "t_E"
|
||||
}
|
||||
],
|
||||
"start": 0.65,
|
||||
"startOffset": 5,
|
||||
"word": "transcript"
|
||||
},
|
||||
{
|
||||
"alignedWord": "is",
|
||||
"case": "success",
|
||||
"end": 1.34,
|
||||
"endOffset": 18,
|
||||
"phones": [
|
||||
{
|
||||
"duration": 0.09,
|
||||
"phone": "ih_B"
|
||||
},
|
||||
{
|
||||
"duration": 0.01,
|
||||
"phone": "z_E"
|
||||
}
|
||||
],
|
||||
"start": 1.24,
|
||||
"startOffset": 16,
|
||||
"word": "is"
|
||||
},
|
||||
{
|
||||
"alignedWord": "<unk>",
|
||||
"case": "success",
|
||||
"end": 1.6400000000000001,
|
||||
"endOffset": 25,
|
||||
"phones": [
|
||||
{
|
||||
"duration": 0.3,
|
||||
"phone": "oov_S"
|
||||
}
|
||||
],
|
||||
"start": 1.34,
|
||||
"startOffset": 19,
|
||||
"word": "synced"
|
||||
},
|
||||
{
|
||||
"alignedWord": "to",
|
||||
"case": "success",
|
||||
"end": 1.81,
|
||||
"endOffset": 28,
|
||||
"phones": [
|
||||
{
|
||||
"duration": 0.09,
|
||||
"phone": "t_B"
|
||||
},
|
||||
{
|
||||
"duration": 0.05,
|
||||
"phone": "uw_E"
|
||||
}
|
||||
],
|
||||
"start": 1.67,
|
||||
"startOffset": 26,
|
||||
"word": "to"
|
||||
},
|
||||
{
|
||||
"alignedWord": "my",
|
||||
"case": "success",
|
||||
"end": 1.9500000000000002,
|
||||
"endOffset": 32,
|
||||
"phones": [
|
||||
{
|
||||
"duration": 0.06,
|
||||
"phone": "m_B"
|
||||
},
|
||||
{
|
||||
"duration": 0.08,
|
||||
"phone": "ay_E"
|
||||
}
|
||||
],
|
||||
"start": 1.81,
|
||||
"startOffset": 30,
|
||||
"word": "my"
|
||||
},
|
||||
{
|
||||
"alignedWord": "voice",
|
||||
"case": "success",
|
||||
"end": 2.3,
|
||||
"endOffset": 38,
|
||||
"phones": [
|
||||
{
|
||||
"duration": 0.09,
|
||||
"phone": "v_B"
|
||||
},
|
||||
{
|
||||
"duration": 0.17,
|
||||
"phone": "oy_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.09,
|
||||
"phone": "s_E"
|
||||
}
|
||||
],
|
||||
"start": 1.95,
|
||||
"startOffset": 33,
|
||||
"word": "voice"
|
||||
},
|
||||
{
|
||||
"alignedWord": "automatically",
|
||||
"case": "success",
|
||||
"end": 3.07,
|
||||
"endOffset": 52,
|
||||
"phones": [
|
||||
{
|
||||
"duration": 0.09,
|
||||
"phone": "ao_B"
|
||||
},
|
||||
{
|
||||
"duration": 0.07,
|
||||
"phone": "t_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.05,
|
||||
"phone": "ah_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.06,
|
||||
"phone": "m_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.05,
|
||||
"phone": "ae_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.06,
|
||||
"phone": "t_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.05,
|
||||
"phone": "ih_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.08,
|
||||
"phone": "k_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.07,
|
||||
"phone": "l_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.19,
|
||||
"phone": "iy_E"
|
||||
}
|
||||
],
|
||||
"start": 2.3,
|
||||
"startOffset": 39,
|
||||
"word": "automatically"
|
||||
},
|
||||
{
|
||||
"alignedWord": "the",
|
||||
"case": "success",
|
||||
"end": 3.9,
|
||||
"endOffset": 58,
|
||||
"phones": [
|
||||
{
|
||||
"duration": 0.07,
|
||||
"phone": "dh_B"
|
||||
},
|
||||
{
|
||||
"duration": 0.04,
|
||||
"phone": "ah_E"
|
||||
}
|
||||
],
|
||||
"start": 3.79,
|
||||
"startOffset": 55,
|
||||
"word": "The"
|
||||
},
|
||||
{
|
||||
"alignedWord": "beginning",
|
||||
"case": "success",
|
||||
"end": 4.43,
|
||||
"endOffset": 68,
|
||||
"phones": [
|
||||
{
|
||||
"duration": 0.05,
|
||||
"phone": "b_B"
|
||||
},
|
||||
{
|
||||
"duration": 0.06,
|
||||
"phone": "ih_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.07,
|
||||
"phone": "g_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.08,
|
||||
"phone": "ih_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.03,
|
||||
"phone": "n_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.1,
|
||||
"phone": "ih_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.14,
|
||||
"phone": "ng_E"
|
||||
}
|
||||
],
|
||||
"start": 3.9,
|
||||
"startOffset": 59,
|
||||
"word": "beginning"
|
||||
},
|
||||
{
|
||||
"alignedWord": "and",
|
||||
"case": "success",
|
||||
"end": 4.6499999999999995,
|
||||
"endOffset": 72,
|
||||
"phones": [
|
||||
{
|
||||
"duration": 0.08,
|
||||
"phone": "ae_B"
|
||||
},
|
||||
{
|
||||
"duration": 0.05,
|
||||
"phone": "n_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.05,
|
||||
"phone": "d_E"
|
||||
}
|
||||
],
|
||||
"start": 4.47,
|
||||
"startOffset": 69,
|
||||
"word": "and"
|
||||
},
|
||||
{
|
||||
"alignedWord": "end",
|
||||
"case": "success",
|
||||
"end": 5.069999999999999,
|
||||
"endOffset": 76,
|
||||
"phones": [
|
||||
{
|
||||
"duration": 0.19,
|
||||
"phone": "eh_B"
|
||||
},
|
||||
{
|
||||
"duration": 0.11,
|
||||
"phone": "n_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.09,
|
||||
"phone": "d_E"
|
||||
}
|
||||
],
|
||||
"start": 4.68,
|
||||
"startOffset": 73,
|
||||
"word": "end"
|
||||
},
|
||||
{
|
||||
"alignedWord": "of",
|
||||
"case": "success",
|
||||
"end": 5.22,
|
||||
"endOffset": 79,
|
||||
"phones": [
|
||||
{
|
||||
"duration": 0.07,
|
||||
"phone": "ah_B"
|
||||
},
|
||||
{
|
||||
"duration": 0.07,
|
||||
"phone": "v_E"
|
||||
}
|
||||
],
|
||||
"start": 5.08,
|
||||
"startOffset": 77,
|
||||
"word": "of"
|
||||
},
|
||||
{
|
||||
"alignedWord": "each",
|
||||
"case": "success",
|
||||
"end": 5.529999999999999,
|
||||
"endOffset": 84,
|
||||
"phones": [
|
||||
{
|
||||
"duration": 0.15,
|
||||
"phone": "iy_B"
|
||||
},
|
||||
{
|
||||
"duration": 0.12,
|
||||
"phone": "ch_E"
|
||||
}
|
||||
],
|
||||
"start": 5.26,
|
||||
"startOffset": 80,
|
||||
"word": "each"
|
||||
},
|
||||
{
|
||||
"alignedWord": "word",
|
||||
"case": "success",
|
||||
"end": 5.96,
|
||||
"endOffset": 90,
|
||||
"phones": [
|
||||
{
|
||||
"duration": 0.14,
|
||||
"phone": "w_B"
|
||||
},
|
||||
{
|
||||
"duration": 0.13,
|
||||
"phone": "er_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.16,
|
||||
"phone": "d_E"
|
||||
}
|
||||
],
|
||||
"start": 5.53,
|
||||
"startOffset": 86,
|
||||
"word": "word"
|
||||
},
|
||||
{
|
||||
"alignedWord": "is",
|
||||
"case": "success",
|
||||
"end": 6.24,
|
||||
"endOffset": 93,
|
||||
"phones": [
|
||||
{
|
||||
"duration": 0.09,
|
||||
"phone": "ih_B"
|
||||
},
|
||||
{
|
||||
"duration": 0.07,
|
||||
"phone": "z_E"
|
||||
}
|
||||
],
|
||||
"start": 6.08,
|
||||
"startOffset": 91,
|
||||
"word": "is"
|
||||
},
|
||||
{
|
||||
"alignedWord": "recorded",
|
||||
"case": "success",
|
||||
"end": 6.69,
|
||||
"endOffset": 102,
|
||||
"phones": [
|
||||
{
|
||||
"duration": 0.04,
|
||||
"phone": "r_B"
|
||||
},
|
||||
{
|
||||
"duration": 0.06,
|
||||
"phone": "ih_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.1,
|
||||
"phone": "k_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.09,
|
||||
"phone": "ao_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.01,
|
||||
"phone": "r_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.05,
|
||||
"phone": "d_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.06,
|
||||
"phone": "ih_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.04,
|
||||
"phone": "d_E"
|
||||
}
|
||||
],
|
||||
"start": 6.24,
|
||||
"startOffset": 94,
|
||||
"word": "recorded"
|
||||
},
|
||||
{
|
||||
"alignedWord": "precisely",
|
||||
"case": "success",
|
||||
"end": 7.44,
|
||||
"endOffset": 112,
|
||||
"phones": [
|
||||
{
|
||||
"duration": 0.02,
|
||||
"phone": "p_B"
|
||||
},
|
||||
{
|
||||
"duration": 0.02,
|
||||
"phone": "r_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.04,
|
||||
"phone": "iy_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.13,
|
||||
"phone": "s_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.12,
|
||||
"phone": "ay_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.11,
|
||||
"phone": "s_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.09,
|
||||
"phone": "l_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.22,
|
||||
"phone": "iy_E"
|
||||
}
|
||||
],
|
||||
"start": 6.69,
|
||||
"startOffset": 103,
|
||||
"word": "precisely"
|
||||
},
|
||||
{
|
||||
"alignedWord": "and",
|
||||
"case": "success",
|
||||
"end": 7.9799999999999995,
|
||||
"endOffset": 117,
|
||||
"phones": [
|
||||
{
|
||||
"duration": 0.05,
|
||||
"phone": "ae_B"
|
||||
},
|
||||
{
|
||||
"duration": 0.02,
|
||||
"phone": "n_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.06,
|
||||
"phone": "d_E"
|
||||
}
|
||||
],
|
||||
"start": 7.85,
|
||||
"startOffset": 114,
|
||||
"word": "and"
|
||||
},
|
||||
{
|
||||
"alignedWord": "can",
|
||||
"case": "success",
|
||||
"end": 8.120000000000001,
|
||||
"endOffset": 121,
|
||||
"phones": [
|
||||
{
|
||||
"duration": 0.05,
|
||||
"phone": "k_B"
|
||||
},
|
||||
{
|
||||
"duration": 0.04,
|
||||
"phone": "ae_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.05,
|
||||
"phone": "n_E"
|
||||
}
|
||||
],
|
||||
"start": 7.98,
|
||||
"startOffset": 118,
|
||||
"word": "can"
|
||||
},
|
||||
{
|
||||
"alignedWord": "be",
|
||||
"case": "success",
|
||||
"end": 8.219999999999999,
|
||||
"endOffset": 124,
|
||||
"phones": [
|
||||
{
|
||||
"duration": 0.06,
|
||||
"phone": "b_B"
|
||||
},
|
||||
{
|
||||
"duration": 0.04,
|
||||
"phone": "iy_E"
|
||||
}
|
||||
],
|
||||
"start": 8.12,
|
||||
"startOffset": 122,
|
||||
"word": "be"
|
||||
},
|
||||
{
|
||||
"alignedWord": "used",
|
||||
"case": "success",
|
||||
"end": 8.55,
|
||||
"endOffset": 129,
|
||||
"phones": [
|
||||
{
|
||||
"duration": 0.11,
|
||||
"phone": "y_B"
|
||||
},
|
||||
{
|
||||
"duration": 0.17,
|
||||
"phone": "uw_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.04,
|
||||
"phone": "z_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.01,
|
||||
"phone": "d_E"
|
||||
}
|
||||
],
|
||||
"start": 8.22,
|
||||
"startOffset": 125,
|
||||
"word": "used"
|
||||
},
|
||||
{
|
||||
"alignedWord": "to",
|
||||
"case": "success",
|
||||
"end": 8.659999,
|
||||
"endOffset": 132,
|
||||
"phones": [
|
||||
{
|
||||
"duration": 0.05,
|
||||
"phone": "t_B"
|
||||
},
|
||||
{
|
||||
"duration": 0.05,
|
||||
"phone": "ih_E"
|
||||
}
|
||||
],
|
||||
"start": 8.559999,
|
||||
"startOffset": 130,
|
||||
"word": "to"
|
||||
},
|
||||
{
|
||||
"alignedWord": "trigger",
|
||||
"case": "success",
|
||||
"end": 9.0,
|
||||
"endOffset": 140,
|
||||
"phones": [
|
||||
{
|
||||
"duration": 0.08,
|
||||
"phone": "t_B"
|
||||
},
|
||||
{
|
||||
"duration": 0.05,
|
||||
"phone": "r_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.05,
|
||||
"phone": "ih_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.09,
|
||||
"phone": "g_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.07,
|
||||
"phone": "er_E"
|
||||
}
|
||||
],
|
||||
"start": 8.66,
|
||||
"startOffset": 133,
|
||||
"word": "trigger"
|
||||
},
|
||||
{
|
||||
"alignedWord": "events",
|
||||
"case": "success",
|
||||
"end": 9.48,
|
||||
"endOffset": 148,
|
||||
"phones": [
|
||||
{
|
||||
"duration": 0.09,
|
||||
"phone": "iy_B"
|
||||
},
|
||||
{
|
||||
"duration": 0.08,
|
||||
"phone": "v_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.07,
|
||||
"phone": "eh_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.06,
|
||||
"phone": "n_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.06,
|
||||
"phone": "t_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.12,
|
||||
"phone": "s_E"
|
||||
}
|
||||
],
|
||||
"start": 9.0,
|
||||
"startOffset": 142,
|
||||
"word": "events"
|
||||
},
|
||||
{
|
||||
"alignedWord": "in",
|
||||
"case": "success",
|
||||
"end": 9.64,
|
||||
"endOffset": 151,
|
||||
"phones": [
|
||||
{
|
||||
"duration": 0.07,
|
||||
"phone": "ih_B"
|
||||
},
|
||||
{
|
||||
"duration": 0.07,
|
||||
"phone": "n_E"
|
||||
}
|
||||
],
|
||||
"start": 9.5,
|
||||
"startOffset": 149,
|
||||
"word": "in"
|
||||
},
|
||||
{
|
||||
"alignedWord": "the",
|
||||
"case": "success",
|
||||
"end": 9.729999,
|
||||
"endOffset": 155,
|
||||
"phones": [
|
||||
{
|
||||
"duration": 0.03,
|
||||
"phone": "dh_B"
|
||||
},
|
||||
{
|
||||
"duration": 0.06,
|
||||
"phone": "ah_E"
|
||||
}
|
||||
],
|
||||
"start": 9.639999,
|
||||
"startOffset": 152,
|
||||
"word": "the"
|
||||
},
|
||||
{
|
||||
"alignedWord": "animation",
|
||||
"case": "success",
|
||||
"end": 10.290000000000001,
|
||||
"endOffset": 165,
|
||||
"phones": [
|
||||
{
|
||||
"duration": 0.02,
|
||||
"phone": "ae_B"
|
||||
},
|
||||
{
|
||||
"duration": 0.1,
|
||||
"phone": "n_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.06,
|
||||
"phone": "ah_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.08,
|
||||
"phone": "m_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.07,
|
||||
"phone": "ey_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.1,
|
||||
"phone": "sh_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.07,
|
||||
"phone": "ah_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.06,
|
||||
"phone": "n_E"
|
||||
}
|
||||
],
|
||||
"start": 9.73,
|
||||
"startOffset": 156,
|
||||
"word": "animation"
|
||||
}
|
||||
]
|
||||
}
|
||||
11
examples/voice_transcript.txt
Normal file
11
examples/voice_transcript.txt
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
This transcript is synced to
|
||||
|
||||
my voice automatically.
|
||||
|
||||
The beginning and end of each
|
||||
|
||||
word is recorded precisely
|
||||
|
||||
and can be used to trigger
|
||||
|
||||
events in the animation.
|
||||
67
examples/voice_triggers.hs
Normal file
67
examples/voice_triggers.hs
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
#!/usr/bin/env stack
|
||||
-- stack --resolver lts-15.04 runghc --package reanimate
|
||||
{-# LANGUAGE OverloadedStrings #-}
|
||||
{-# LANGUAGE ApplicativeDo #-}
|
||||
module Main where
|
||||
|
||||
import Control.Monad
|
||||
import qualified Data.Text as T
|
||||
import Reanimate
|
||||
import Reanimate.Voice
|
||||
import Reanimate.Builtin.Documentation
|
||||
import Graphics.SvgTree ( ElementRef(..) )
|
||||
|
||||
transcript :: Transcript
|
||||
transcript = loadTranscript "voice_triggers.txt"
|
||||
|
||||
transformer :: SVG -> SVG
|
||||
transformer = scale 1 . translate (-4) 0 . centerUsing (latex $ transcriptText transcript)
|
||||
|
||||
main :: IO ()
|
||||
main = reanimate $ sceneAnimation $ do
|
||||
newSpriteSVG_ $ mkBackgroundPixel rtfdBackgroundColor
|
||||
waitOn $ forM_ (splitTranscript transcript) $ \(svg, tword) -> do
|
||||
highlighted <- newVar 0
|
||||
void $ newSprite $ do
|
||||
v <- unVar highlighted
|
||||
pure $ transformer $ masked (wordKey tword)
|
||||
v
|
||||
svg
|
||||
(withFillColor "grey" $ mkRect 1 1)
|
||||
(withFillColor "black" $ mkRect 1 1)
|
||||
let dur = wordEnd tword - wordStart tword
|
||||
fork $ do
|
||||
wait (wordStart tword)
|
||||
tweenVar highlighted dur $ \v -> fromToS v 1
|
||||
fork $ do
|
||||
wait (wordStart tword)
|
||||
case wordReference tword of
|
||||
"one" -> highlight dur $ latex "1"
|
||||
"two" -> highlight dur $ latex "2"
|
||||
"three" -> highlight dur $ latex "3"
|
||||
"red" -> highlight dur $ withFillColor "red" $ mkCircle 1
|
||||
"green" -> highlight dur $ withFillColor "green" $ mkCircle 1
|
||||
"blue" -> highlight dur $ withFillColor "blue" $ mkCircle 1
|
||||
_ -> return ()
|
||||
wait 2
|
||||
where
|
||||
wordKey tword = T.unpack (wordReference tword) ++ show (wordStartOffset tword)
|
||||
highlight dur img =
|
||||
play
|
||||
$ animate
|
||||
(\t -> translate (screenWidth / 4) 0 $ scale t $ scaleToHeight 4 $ center img)
|
||||
# signalA (bellS 2)
|
||||
# setDuration dur
|
||||
|
||||
{-# INLINE masked #-}
|
||||
masked :: String -> Double -> SVG -> SVG -> SVG -> SVG
|
||||
masked key t maskSVG srcSVG dstSVG = mkGroup
|
||||
[ mkClipPath label $ removeGroups maskSVG
|
||||
, withClipPathRef (Ref label)
|
||||
$ translate (x - w / 2 + w * t) y (scaleToSize w screenHeight dstSVG)
|
||||
, withClipPathRef (Ref label)
|
||||
$ translate (x + w / 2 + w * t) y (scaleToSize w screenHeight srcSVG)
|
||||
]
|
||||
where
|
||||
label = "word-mask-" ++ key
|
||||
(x, y, w, _h) = boundingBox maskSVG
|
||||
281
examples/voice_triggers.json
Normal file
281
examples/voice_triggers.json
Normal file
|
|
@ -0,0 +1,281 @@
|
|||
{
|
||||
"transcript": "one two three\n\none two three\n\nred green blue\n\nred green blue\n",
|
||||
"words": [
|
||||
{
|
||||
"alignedWord": "one",
|
||||
"case": "success",
|
||||
"end": 1.56,
|
||||
"endOffset": 3,
|
||||
"phones": [
|
||||
{
|
||||
"duration": 0.01,
|
||||
"phone": "w_B"
|
||||
},
|
||||
{
|
||||
"duration": 0.01,
|
||||
"phone": "ah_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.31,
|
||||
"phone": "n_E"
|
||||
}
|
||||
],
|
||||
"start": 0.60,
|
||||
"startOffset": 0,
|
||||
"word": "one"
|
||||
},
|
||||
{
|
||||
"alignedWord": "two",
|
||||
"case": "success",
|
||||
"end": 3.27,
|
||||
"endOffset": 7,
|
||||
"phones": [
|
||||
{
|
||||
"duration": 0.13,
|
||||
"phone": "t_B"
|
||||
},
|
||||
{
|
||||
"duration": 0.87,
|
||||
"phone": "uw_E"
|
||||
}
|
||||
],
|
||||
"start": 2.27,
|
||||
"startOffset": 4,
|
||||
"word": "two"
|
||||
},
|
||||
{
|
||||
"alignedWord": "three",
|
||||
"case": "success",
|
||||
"end": 5.0600000000000005,
|
||||
"endOffset": 13,
|
||||
"phones": [
|
||||
{
|
||||
"duration": 0.08,
|
||||
"phone": "th_B"
|
||||
},
|
||||
{
|
||||
"duration": 0.09,
|
||||
"phone": "r_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.73,
|
||||
"phone": "iy_E"
|
||||
}
|
||||
],
|
||||
"start": 4.16,
|
||||
"startOffset": 8,
|
||||
"word": "three"
|
||||
},
|
||||
{
|
||||
"alignedWord": "one",
|
||||
"case": "success",
|
||||
"end": 6.21,
|
||||
"endOffset": 18,
|
||||
"phones": [
|
||||
{
|
||||
"duration": 0.09,
|
||||
"phone": "w_B"
|
||||
},
|
||||
{
|
||||
"duration": 0.08,
|
||||
"phone": "ah_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.09,
|
||||
"phone": "n_E"
|
||||
}
|
||||
],
|
||||
"start": 5.95,
|
||||
"startOffset": 15,
|
||||
"word": "one"
|
||||
},
|
||||
{
|
||||
"alignedWord": "two",
|
||||
"case": "success",
|
||||
"end": 6.550000000000001,
|
||||
"endOffset": 22,
|
||||
"phones": [
|
||||
{
|
||||
"duration": 0.1,
|
||||
"phone": "t_B"
|
||||
},
|
||||
{
|
||||
"duration": 0.17,
|
||||
"phone": "uw_E"
|
||||
}
|
||||
],
|
||||
"start": 6.28,
|
||||
"startOffset": 19,
|
||||
"word": "two"
|
||||
},
|
||||
{
|
||||
"alignedWord": "three",
|
||||
"case": "success",
|
||||
"end": 6.96,
|
||||
"endOffset": 28,
|
||||
"phones": [
|
||||
{
|
||||
"duration": 0.09,
|
||||
"phone": "th_B"
|
||||
},
|
||||
{
|
||||
"duration": 0.07,
|
||||
"phone": "r_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.17,
|
||||
"phone": "iy_E"
|
||||
}
|
||||
],
|
||||
"start": 6.63,
|
||||
"startOffset": 23,
|
||||
"word": "three"
|
||||
},
|
||||
{
|
||||
"alignedWord": "red",
|
||||
"case": "success",
|
||||
"end": 9.58,
|
||||
"endOffset": 33,
|
||||
"phones": [
|
||||
{
|
||||
"duration": 0.16,
|
||||
"phone": "r_B"
|
||||
},
|
||||
{
|
||||
"duration": 0.49,
|
||||
"phone": "eh_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.24,
|
||||
"phone": "d_E"
|
||||
}
|
||||
],
|
||||
"start": 8.69,
|
||||
"startOffset": 30,
|
||||
"word": "red"
|
||||
},
|
||||
{
|
||||
"alignedWord": "green",
|
||||
"case": "success",
|
||||
"end": 11.4,
|
||||
"endOffset": 39,
|
||||
"phones": [
|
||||
{
|
||||
"duration": 0.12,
|
||||
"phone": "g_B"
|
||||
},
|
||||
{
|
||||
"duration": 0.14,
|
||||
"phone": "r_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.51,
|
||||
"phone": "iy_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.33,
|
||||
"phone": "n_E"
|
||||
}
|
||||
],
|
||||
"start": 10.3,
|
||||
"startOffset": 34,
|
||||
"word": "green"
|
||||
},
|
||||
{
|
||||
"alignedWord": "blue",
|
||||
"case": "success",
|
||||
"end": 13.199999,
|
||||
"endOffset": 44,
|
||||
"phones": [
|
||||
{
|
||||
"duration": 0.08,
|
||||
"phone": "b_B"
|
||||
},
|
||||
{
|
||||
"duration": 0.12,
|
||||
"phone": "l_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.78,
|
||||
"phone": "uw_E"
|
||||
}
|
||||
],
|
||||
"start": 12.219999,
|
||||
"startOffset": 40,
|
||||
"word": "blue"
|
||||
},
|
||||
{
|
||||
"alignedWord": "red",
|
||||
"case": "success",
|
||||
"end": 14.27,
|
||||
"endOffset": 49,
|
||||
"phones": [
|
||||
{
|
||||
"duration": 0.09,
|
||||
"phone": "r_B"
|
||||
},
|
||||
{
|
||||
"duration": 0.07,
|
||||
"phone": "eh_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.07,
|
||||
"phone": "d_E"
|
||||
}
|
||||
],
|
||||
"start": 14.04,
|
||||
"startOffset": 46,
|
||||
"word": "red"
|
||||
},
|
||||
{
|
||||
"alignedWord": "green",
|
||||
"case": "success",
|
||||
"end": 14.549999999999999,
|
||||
"endOffset": 55,
|
||||
"phones": [
|
||||
{
|
||||
"duration": 0.06,
|
||||
"phone": "g_B"
|
||||
},
|
||||
{
|
||||
"duration": 0.04,
|
||||
"phone": "r_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.07,
|
||||
"phone": "iy_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.11,
|
||||
"phone": "n_E"
|
||||
}
|
||||
],
|
||||
"start": 14.27,
|
||||
"startOffset": 50,
|
||||
"word": "green"
|
||||
},
|
||||
{
|
||||
"alignedWord": "blue",
|
||||
"case": "success",
|
||||
"end": 14.869999,
|
||||
"endOffset": 60,
|
||||
"phones": [
|
||||
{
|
||||
"duration": 0.07,
|
||||
"phone": "b_B"
|
||||
},
|
||||
{
|
||||
"duration": 0.06,
|
||||
"phone": "l_I"
|
||||
},
|
||||
{
|
||||
"duration": 0.18,
|
||||
"phone": "uw_E"
|
||||
}
|
||||
],
|
||||
"start": 14.559999,
|
||||
"startOffset": 56,
|
||||
"word": "blue"
|
||||
}
|
||||
]
|
||||
}
|
||||
7
examples/voice_triggers.txt
Normal file
7
examples/voice_triggers.txt
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
one two three
|
||||
|
||||
one two three
|
||||
|
||||
red green blue
|
||||
|
||||
red green blue
|
||||
Loading…
Reference in a new issue