Convert travis tests to scripts

Moves tests from .travis.yml to bash scripts in tests directory. Adds
the test scripts to tests/Makefile.am so that they are included when
running make check.

.gitignore

@@ -41,6 +41,7 @@ Makefile.in
 *.la
 *.lo
 *.o
+*.trs
 
 *.log
 .kdev4

.travis-install.bash

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Download FFmpeg and HM decoder and place them in $PATH.
+
+set -euvo pipefail
+
+mkdir -p "${HOME}/bin"
+
+wget http://ultravideo.cs.tut.fi/ffmpeg-release-32bit-static.tar.xz
+sha256sum -c - << EOF
+4d3302ba0415e08ca10ca578dcd1f0acc48fadc9b803718283c8c670350c903e  ffmpeg-release-32bit-static.tar.xz
+EOF
+tar xf ffmpeg-release-32bit-static.tar.xz
+cp ffmpeg-2.6.3-32bit-static/ffmpeg "${HOME}/bin/ffmpeg"
+chmod +x "${HOME}/bin/ffmpeg"
+
+wget http://ultravideo.cs.tut.fi/ubuntu-12.04-hmdec-16.10.tgz
+sha256sum -c - << EOF
+e00d61dd031a14aab1a03c0b23df315b8f6ec3fab66a0e2ae2162496153ccf92  ubuntu-12.04-hmdec-16.10.tgz
+EOF
+tar xf ubuntu-12.04-hmdec-16.10.tgz
+cp hmdec-16.10 "${HOME}/bin/TAppDecoderStatic"
+chmod +x "${HOME}/bin/TAppDecoderStatic"
+
+export PATH="${HOME}/bin:${PATH}"

.travis-install.sh

@@ -1,12 +0,0 @@
-#!/bin/sh
-set -ev
-
-if [ -n "$VALGRIND_TEST" ]; then
-  wget http://ultravideo.cs.tut.fi/ffmpeg-release-32bit-static.tar.xz
-  7z x ffmpeg-release-32bit-static.tar.xz
-  7z x ffmpeg-release-32bit-static.tar
-  chmod +x ./ffmpeg-2.6.3-32bit-static/ffmpeg
-  ./ffmpeg-2.6.3-32bit-static/ffmpeg -f lavfi -i "mandelbrot=size=${TEST_DIM}:end_pts=10" -vframes $TEST_FRAMES -pix_fmt yuv420p mandelbrot_${TEST_DIM}.yuv
-  wget http://ultravideo.cs.tut.fi/ubuntu-12.04-hmdec-16.10.tgz
-  tar -xzvf ubuntu-12.04-hmdec-16.10.tgz
-fi

.travis-script.sh

@@ -1,21 +0,0 @@
-#!/bin/sh
-set -ev
-
-./autogen.sh
-./configure $KVZ_CONFIGURE_ARGS
-make --jobs=2 V=1
-
-if [ -n "$VALGRIND_TEST" ]; then
-  libtool execute valgrind --leak-check=full --error-exitcode=1 -- \
-    src/kvazaar -i mandelbrot_${TEST_DIM}.yuv --input-res=${TEST_DIM} \
-    -o test.265 $VALGRIND_TEST
-    ./hmdec-16.10 -b test.265
-elif [ -n "$EXPECTED_STATUS" ]; then
-  set +e
-  libtool execute src/kvazaar $PARAMS
-  EXIT_STATUS=$?
-  set -e
-  [ "$EXIT_STATUS" = "$EXPECTED_STATUS" ]
-else
-  make check
-fi

.travis.yml

@@ -1,23 +1,16 @@
 language: c
 
-env:
+# Use container based infrastructure.
-  global:
-  - KVZ_CONFIGURE_ARGS=--enable-werror 
-  - TEST_DIM=264x130
-  - TEST_FRAMES=10
-
-# Use container based infrastructure
 sudo: false
 
-# Use this the global requirements list for valgrind tests, because those are the most numerous.
 addons:
   apt:
     sources:
       - ubuntu-toolchain-r-test
     packages:
       - autoconf
+      - gcc-4.8
       - libtool
-    - p7zip-full  # to uncompress our own ffmpeg binary
       - valgrind
       - yasm
 
@@ -26,113 +19,25 @@ matrix:
 
   include:
     - compiler: clang
-      addons:
-        apt:
-          sources:
-          - ubuntu-toolchain-r-test
-          packages:
-          - autoconf
-          - libtool
-          - yasm
-    
     - compiler: gcc-4.8
-      addons:
-        apt:
-          sources:
-          - ubuntu-toolchain-r-test
-          packages:
-          - autoconf
-          - gcc-4.8
-          - libtool
-          - yasm
 
     # We have some Mac specific code and Mac sometimes has odd build issues.
     - os: osx
       compiler: clang  # gcc is actually clang on Travis OS X
-
+      install: true
-    # Check for external symbols without kvz_ prefix.
-    - compiler: gcc-4.8
       script:
         - ./autogen.sh
-        - ./configure && make
+        - ./configure --enable-werror
-        - (! nm -go --defined-only src/.libs/libkvazaar.a | grep -v ' kvz_') || (echo 'ERROR Only symbols prefixed with kvz_ should be exported from libkvazaar.'; false)
+        - make --jobs=2 V=1
-      addons:
-        apt:
-          sources:
-          - ubuntu-toolchain-r-test
-          packages:
-          - autoconf
-          - gcc-4.8
-          - libtool
-          - yasm
 
-    # Tests trying to use invalid input dimensions
+install: bash .travis-install.bash
-    - env: EXPECTED_STATUS=1 PARAMS="-i src/kvazaar --input-res=1x65 -o /dev/null"
-      addons:
-        apt:
-          sources:
-          - ubuntu-toolchain-r-test
-          packages:
-          - autoconf
-          - libtool
-          - yasm
-
-    # These valgrind tests are slow, so they are performed with the minimum
-    # number of small frames and fast settings.
-    
-    # Tests for interlace
-    - env: VALGRIND_TEST="--source-scan-type=tff -p0 --preset=ultrafast --threads=2 --owf=1 --wpp"
-    
-    # Tests for owf, wpp and tiles. There is lots of separate branches of
-    # code related to owf=0 and owf!=0, which is why all permutations are
-    # tried.
-    - env: VALGRIND_TEST="-p4 -r1 --owf=1 --threads=0 --rd=0 --no-rdoq --no-deblock --no-sao --no-signhide --subme=0 --pu-depth-inter=1-3 --pu-depth-intra=2-3"
-    - env: VALGRIND_TEST="-p4 -r1 --owf=0 --threads=0 --rd=0 --no-rdoq --no-deblock --no-sao --no-signhide --subme=0 --pu-depth-inter=1-3 --pu-depth-intra=2-3"
-    - env: VALGRIND_TEST="-p4 -r2 --owf=1 --threads=2 --wpp --rd=0 --no-rdoq --no-deblock --no-sao --no-signhide --subme=0 --pu-depth-inter=1-3 --pu-depth-intra=2-3"
-    - env: VALGRIND_TEST="-p4 -r2 --owf=0 --threads=2 --rd=0 --no-rdoq --no-deblock --no-sao --no-signhide --subme=0 --pu-depth-inter=1-3 --pu-depth-intra=2-3"
-    - env: VALGRIND_TEST="-p4 -r2 --owf=1 --threads=2 --tiles-height-split=u2 --no-wpp --rd=0 --no-rdoq --no-deblock --no-sao --no-signhide --subme=0 --pu-depth-inter=1-3 --pu-depth-intra=2-3"
-    - env: VALGRIND_TEST="-p4 -r2 --owf=0 --threads=2 --tiles-height-split=u2 --no-wpp --rd=0 --no-rdoq --no-deblock --no-sao --no-signhide --subme=0 --pu-depth-inter=1-3 --pu-depth-intra=2-3"
-    
-    # Tests for rdoq, sao, deblock and signhide and subme.
-    - env: VALGRIND_TEST="-p0 -r1 --threads=2 --wpp --owf=1 --rd=0 --no-rdoq --no-deblock --no-sao --no-signhide --subme=1 --pu-depth-intra=2-3"
-    - env: VALGRIND_TEST="-p0 -r1 --threads=2 --wpp --owf=1 --rd=0 --no-rdoq --no-signhide --subme=0"
-    - env: VALGRIND_TEST="-p0 -r1 --threads=2 --wpp --owf=1 --rd=0 --no-deblock --no-sao --subme=0"
-    
-    # Tests for all-intra.
-    - env: VALGRIND_TEST="-p1 --threads=2 --owf=1 --rd=1 --no-rdoq --no-deblock --no-sao --no-signhide"
-    - env: VALGRIND_TEST="-p1 --threads=2 --owf=1 --rd=2 --no-rdoq --no-deblock --no-sao --no-signhide --no-transform-skip"
-
-    # Tests for SMP and AMP blocks.
-    - env: TEST_FRAMES=4 VALGRIND_TEST="--threads=2 --owf=1 --wpp --smp"
-    - env: TEST_FRAMES=4 VALGRIND_TEST="--threads=2 --owf=1 --wpp --amp"
-    - env: TEST_FRAMES=4 VALGRIND_TEST="--threads=2 --owf=1 --wpp --smp --amp"
-
-    # Tests for rate control
-    - env: VALGRIND_TEST="--bitrate=500000 -p0 -r1 --owf=1 --threads=2 --rd=0 --no-rdoq --no-deblock --no-sao --no-signhide --subme=0 --pu-depth-inter=1-3 --pu-depth-intra=2-3"
-    
-    # Tests for GOP, with and without OWF.
-    - env: TEST_FRAMES=20 VALGRIND_TEST="--gop=8 -p0 --threads=2 --wpp --owf=1 --rd=0 --no-rdoq --no-deblock --no-sao --no-signhide --subme=0 --pu-depth-inter=1-3 --pu-depth-intra=2-3"
-    - env: TEST_FRAMES=10 VALGRIND_TEST="--gop=8 -p0 --threads=2 --wpp --owf=4 --rd=0 --no-rdoq --no-deblock --no-sao --no-signhide --subme=0 --pu-depth-inter=1-3 --pu-depth-intra=2-3"
-    - env: TEST_FRAMES=20 VALGRIND_TEST="--gop=8 -p0 --threads=2 --wpp --owf=0 --rd=0 --no-rdoq --no-deblock --no-sao --no-signhide --subme=0 --pu-depth-inter=1-3 --pu-depth-intra=2-3"
-    
-    # Tests for --mv-constraint
-    - env: VALGRIND_TEST="--threads=2 --owf=1 --preset=ultrafast --pu-depth-inter=0-3 --mv-constraint=frametilemargin"
-    - env: VALGRIND_TEST="--threads=2 --owf=1 --preset=ultrafast --subme=4 --mv-constraint=frametilemargin"
-    
-    # Tests for --slices
-    - env: TEST_DIM=512x256 VALGRIND_TEST="--threads=2 --owf=1 --preset=ultrafast --tiles=2x2 --slices=tiles"
-    - env: VALGRIND_TEST="--threads=2 --owf=1 --preset=ultrafast --slices=wpp"
-    
-    # Test weird shapes.
-    - env: TEST_DIM=16x16 VALGRIND_TEST="--threads=2 --owf=1 --preset=veryslow"
-    - env: TEST_DIM=256x16 VALGRIND_TEST="--threads=2 --owf=1 --preset=veryslow"
-    - env: TEST_DIM=16x256 VALGRIND_TEST="--threads=2 --owf=1 --preset=veryslow"
-
-install:
-  - source .travis-install.sh
 
 script:
-  - source .travis-script.sh
+  - ./autogen.sh
+  - ./configure --enable-werror
+  - make --jobs=2 V=1
+  - make check VERBOSE=1
 
 after_script:
-  - set +e # Disable errors to work around Travis not knowing how to fix their stuff.
+  # Disable errors to work around Travis not knowing how to fix their stuff.
+  - set +e

tests/Makefile.am

@@ -1,5 +1,17 @@
 
-TESTS = $(check_PROGRAMS)
+TESTS = $(check_PROGRAMS) \
+    test_external_symbols.bash \
+    test_gop.bash \
+    test_interlace.bash \
+    test_intra.bash \
+    test_invalid_input.bash \
+    test_mv_constraint.bash \
+    test_owf_wpp_tiles.bash \
+    test_rate_control.bash \
+    test_slices.bash \
+    test_smp.bash \
+    test_tools.bash \
+    test_weird_shapes.bash
 
 check_PROGRAMS = kvazaar_tests
 
@@ -17,4 +29,3 @@ kvazaar_tests_SOURCES = \
 	test_strategies.h
 kvazaar_tests_CFLAGS = -I$(srcdir) -I$(top_srcdir) -I$(top_srcdir)/src
 kvazaar_tests_LDFLAGS = -static $(top_builddir)/src/libkvazaar.la $(LIBS)
-

tests/test_external_symbols.bash

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Check for external symbols without kvz_ prefix.
+
+set -euo pipefail
+
+if nm -go --defined-only ../src/.libs/libkvazaar.a | grep -v ' kvz_'; then
+    printf '%s\n' 'Only symbols prefixed with "kvz_" should be exported from libkvazaar.'
+    false
+fi

tests/test_gop.bash

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# Test GOP, with and without OWF.
+
+set -eu
+source util.bash
+
+common_args='--gop=8 -p0 --threads=2 --wpp --rd=0 --no-rdoq --no-deblock --no-sao --no-signhide --subme=0 --pu-depth-inter=1-3 --pu-depth-intra=2-3'
+valgrind_test 264x130 10 $common_args --owf=1
+valgrind_test 264x130 10 $common_args --owf=4
+valgrind_test 264x130 20 $common_args --owf=0

tests/test_interlace.bash

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -eu
+source util.bash
+
+valgrind_test 264x130 10 --source-scan-type=tff -p0 --preset=ultrafast --threads=2 --owf=1 --wpp

tests/test_intra.bash

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# Test all-intra coding.
+
+set -eu
+
+source util.bash
+
+common_args='264x130 10 -p1 --threads=2 --owf=1 --no-rdoq --no-deblock --no-sao --no-signhide'
+valgrind_test $common_args --rd=1
+valgrind_test $common_args --rd=2 --no-transform-skip

tests/test_invalid_input.bash

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# Test trying to use invalid input dimensions.
+
+set -eu
+source util.bash
+
+encode_test 1x65 1 1

tests/test_mv_constraint.bash

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -eu
+source util.bash
+
+valgrind_test 264x130 10 --threads=2 --owf=1 --preset=ultrafast --pu-depth-inter=0-3 --mv-constraint=frametilemargin
+valgrind_test 264x130 10 --threads=2 --owf=1 --preset=ultrafast --subme=4 --mv-constraint=frametilemargin

tests/test_owf_wpp_tiles.bash

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# Test OWF, WPP and tiles. There is lots of separate branches of code
+# related to owf == 0 and owf != 0, which is why all permutations are
+# tried.
+
+set -eu
+source util.bash
+
+common_args='-p4 --rd=0 --no-rdoq --no-deblock --no-sao --no-signhide --subme=0 --pu-depth-inter=1-3 --pu-depth-intra=2-3'
+valgrind_test 264x130 10 $common_args -r1 --owf=1 --threads=0 --no-wpp
+valgrind_test 264x130 10 $common_args -r1 --owf=0 --threads=0 --no-wpp
+valgrind_test 264x130 10 $common_args -r2 --owf=1 --threads=2 --wpp
+valgrind_test 264x130 10 $common_args -r2 --owf=0 --threads=2 --no-wpp
+valgrind_test 264x130 10 $common_args -r2 --owf=1 --threads=2 --tiles-height-split=u2 --no-wpp
+valgrind_test 264x130 10 $common_args -r2 --owf=0 --threads=2 --tiles-height-split=u2 --no-wpp

tests/test_rate_control.bash

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -eu
+source util.bash
+
+valgrind_test 264x130 10 --bitrate=500000 -p0 -r1 --owf=1 --threads=2 --rd=0 --no-rdoq --no-deblock --no-sao --no-signhide --subme=0 --pu-depth-inter=1-3 --pu-depth-intra=2-3

tests/test_slices.bash

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -eu
+source util.bash
+
+valgrind_test 512x256 10 --threads=2 --owf=1 --preset=ultrafast --tiles=2x2 --slices=tiles
+valgrind_test 264x130 10 --threads=2 --owf=1 --preset=ultrafast --slices=wpp

tests/test_smp.bash

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Test SMP and AMP blocks.
+
+set -eu
+source util.bash
+
+valgrind_test 264x130 4 --threads=2 --owf=1 --wpp --smp
+valgrind_test 264x130 4 --threads=2 --owf=1 --wpp --amp
+valgrind_test 264x130 4 --threads=2 --owf=1 --wpp --smp --amp

tests/test_tools.bash

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# Test RDOQ, SAO, deblock and signhide and subme.
+
+set -eu
+source util.bash
+
+common_args='264x130 10 -p0 -r1 --threads=2 --wpp --owf=1 --rd=0'
+
+valgrind_test $common_args --no-rdoq --no-deblock --no-sao --no-signhide --subme=1 --pu-depth-intra=2-3
+valgrind_test $common_args --no-rdoq --no-signhide --subme=0
+valgrind_test $common_args --rdoq --no-deblock --no-sao --subme=0

tests/test_weird_shapes.bash

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -eu
+source util.bash
+
+valgrind_test  16x16  10 --threads=2 --owf=1 --preset=veryslow
+valgrind_test 256x16  10 --threads=2 --owf=1 --preset=veryslow
+valgrind_test  16x256 10 --threads=2 --owf=1 --preset=veryslow

tests/util.bash

@@ -0,0 +1,65 @@
+#!/bin/bash
+
+# Helper functions for test scripts.
+
+set -euo pipefail
+
+# Temporary files for encoder input and output.
+yuvfile="$(mktemp --tmpdir tmp.XXXXXXXXXX.yuv)"
+hevcfile="$(mktemp --tmpdir tmp.XXXXXXXXXX.hevc)"
+
+cleanup() {
+    rm -rf ${yuvfile} ${hevcfile}
+}
+trap cleanup EXIT
+
+print_and_run() {
+    printf '\n\n$ %s\n' "$*"
+    "$@"
+}
+
+prepare() {
+    cleanup
+    print_and_run \
+        ffmpeg -f lavfi -i "mandelbrot=size=${1}" \
+            -vframes "${2}" -pix_fmt yuv420p \
+            "${yuvfile}"
+}
+
+valgrind_test() {
+    dimensions="$1"
+    shift
+    frames="$1"
+    shift
+
+    prepare "${dimensions}" "${frames}"
+
+    print_and_run \
+        libtool execute \
+            valgrind --leak-check=full --error-exitcode=1 -- \
+            ../src/kvazaar -i "${yuvfile}" "--input-res=${dimensions}" -o "${hevcfile}" "$@"
+
+    print_and_run \
+        TAppDecoderStatic -b "${hevcfile}"
+
+    cleanup
+}
+
+encode_test() {
+    dimensions="$1"
+    shift
+    frames="$1"
+    shift
+    expected_status="$1"
+    shift
+
+    prepare "${dimensions}" "${frames}"
+
+    set +e
+    print_and_run \
+        libtool execute \
+            ../src/kvazaar -i "${yuvfile}" "--input-res=${dimensions}" -o "${hevcfile}" "$@"
+    actual_status="$?"
+    set -e
+    [[ ${actual_status} = ${expected_status} ]]
+}