Add --hash=md5

Add md5 through extras/libmd5 taken from HM with BSD license. It's
implemented as a generic strategy using the same interface as checksum,
so we can write a SIMD version if it seems necessary.
This commit is contained in:
Ari Koivula 2016-03-17 18:12:43 +02:00
parent 883448b8fb
commit 4125218cfa
12 changed files with 403 additions and 18 deletions

View file

@ -14,9 +14,9 @@
<RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
<PreprocessorDefinitions>KVZ_DLL_EXPORTS;KVZ_COMPILE_ASM;WIN32_LEAN_AND_MEAN;WIN32;WIN64;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>$(SolutionDir)..\..\pthreads.2\include;$(SolutionDir)..\src;$(SolutionDir)..\src\extras;$(SolutionDir)..\;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<DisableSpecificWarnings>4244;4204;4206;4028;4152;4996;4018;4456;4389;4100;4131;4459</DisableSpecificWarnings>
<DisableSpecificWarnings>4244;4204;4206;4028;4152;4996;4018;4456;4389;4100;4131;4459;4706</DisableSpecificWarnings>
<OpenMPSupport>false</OpenMPSupport>
<TreatSpecificWarningsAsErrors>4013;4029;4047;4716;4700;4020;4021;4133;4090</TreatSpecificWarningsAsErrors>
<TreatSpecificWarningsAsErrors>4013;4029;4047;4716;4700;4020;4021;4133</TreatSpecificWarningsAsErrors>
</ClCompile>
<Link>
<AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>

View file

@ -125,6 +125,7 @@
</ClCompile>
</ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="..\..\src\extras\libmd5.c" />
<ClCompile Include="..\..\src\input_frame_buffer.c" />
<ClCompile Include="..\..\src\kvazaar.c" />
<ClCompile Include="..\..\src\bitstream.c" />
@ -176,6 +177,7 @@
<ClCompile Include="..\..\src\strategies\strategies-quant.c" />
<ClInclude Include="..\..\src\checkpoint.h" />
<ClInclude Include="..\..\src\cu.h" />
<ClInclude Include="..\..\src\extras\libmd5.h" />
<ClInclude Include="..\..\src\image.h" />
<ClInclude Include="..\..\src\imagelist.h" />
<ClCompile Include="..\..\src\strategies\altivec\picture-altivec.c" />

View file

@ -216,6 +216,7 @@
<ClCompile Include="..\..\src\strategies\avx2\sao-avx2.c">
<Filter>Optimization\strategies\avx2</Filter>
</ClCompile>
<ClCompile Include="..\..\src\extras\libmd5.c" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\..\src\bitstream.h">
@ -401,6 +402,7 @@
<ClInclude Include="..\..\src\strategies\avx2\sao-avx2.h">
<Filter>Optimization\strategies\avx2</Filter>
</ClInclude>
<ClInclude Include="..\..\src\extras\libmd5.h" />
</ItemGroup>
<ItemGroup>
<YASM Include="..\..\src\extras\x86inc.asm">

View file

@ -129,7 +129,9 @@ libkvazaar_la_SOURCES = \
strategies/x86_asm/picture-x86-asm.c \
strategies/x86_asm/picture-x86-asm.h \
strategyselector.c \
strategyselector.h
strategyselector.h \
extras/libmd5.c \
extras/libmd5.h
libkvazaar_la_CFLAGS =

View file

@ -786,26 +786,48 @@ static void add_checksum(encoder_state_t * const state)
bitstream_t * const stream = &state->stream;
const videoframe_t * const frame = state->tile->frame;
unsigned char checksum[3][SEI_HASH_MAX_LENGTH];
uint32_t checksum_val;
unsigned int i;
kvz_nal_write(stream, KVZ_NAL_SUFFIX_SEI_NUT, 0, 0);
WRITE_U(stream, 132, 8, "sei_type");
switch (state->encoder_control->cfg->hash)
{
case KVZ_HASH_CHECKSUM:
kvz_image_checksum(frame->rec, checksum, state->encoder_control->bitdepth);
WRITE_U(stream, 132, 8, "sei_type");
WRITE_U(stream, 13, 8, "size");
WRITE_U(stream, 1 + 3 * 4, 8, "size");
WRITE_U(stream, 2, 8, "hash_type"); // 2 = checksum
for (i = 0; i < 3; ++i) {
// Pack bits into a single 32 bit uint instead of pushing them one byte
// at a time.
checksum_val = (checksum[i][0] << 24) + (checksum[i][1] << 16) +
(checksum[i][2] << 8) + (checksum[i][3]);
for (int i = 0; i < 3; ++i) {
uint32_t checksum_val = (
(checksum[i][0] << 24) + (checksum[i][1] << 16) +
(checksum[i][2] << 8) + (checksum[i][3]));
WRITE_U(stream, checksum_val, 32, "picture_checksum");
CHECKPOINT("checksum[%d] = %u", i, checksum_val);
}
break;
case KVZ_HASH_MD5:
kvz_image_md5(frame->rec, checksum, state->encoder_control->bitdepth);
WRITE_U(stream, 1 + 3 * 16, 8, "size");
WRITE_U(stream, 0, 8, "hash_type"); // 0 = md5
for (int i = 0; i < 3; ++i) {
for (int b = 0; b < 16; ++b) {
WRITE_U(stream, checksum[i][b], 8, "picture_md5");
}
}
break;
case KVZ_HASH_NONE:
// Means we shouldn't be writing this SEI.
assert(0);
}
kvz_bitstream_align(stream);
// spec:sei_rbsp() rbsp_trailing_bits

256
src/extras/libmd5.c Normal file
View file

@ -0,0 +1,256 @@
/*
* This code implements the MD5 message-digest algorithm. The algorithm was
* written by Ron Rivest. This code was written by Colin Plumb in 1993, our
* understanding is that no copyright is claimed and that this code is in the
* public domain.
*
* Equivalent code is available from RSA Data Security, Inc.
* This code has been tested against that, and is functionally equivalent,
*
* To compute the message digest of a chunk of bytes, declare an MD5Context
* structure, pass it to MD5Init, call MD5Update as needed on buffers full of
* bytes, and then call MD5Final, which will fill a supplied 16-byte array with
* the digest.
*/
#include <stdint.h>
#include <string.h>
#include "libmd5.h"
//! \ingroup libMD5
//! \{
static void MD5Transform(uint32_t buf[4], uint32_t const in[16]);
#ifndef __BIG_ENDIAN__
# define byteReverse(buf, len) /* Nothing */
#else
void byteReverse(uint32_t *buf, unsigned len);
/*
* Note: this code is harmless on little-endian machines.
*/
void byteReverse(uint32_t *buf, unsigned len)
{
uint32_t t;
do {
char* bytes = (char *) buf;
t = ((unsigned) bytes[3] << 8 | bytes[2]) << 16 |
((unsigned) bytes[1] << 8 | bytes[0]);
*buf = t;
buf++;
} while (--len);
}
#endif
/*
* Start MD5 accumulation. Set bit count to 0 and buffer to mysterious
* initialization constants.
*/
void MD5Init(context_md5_t *ctx)
{
ctx->buf[0] = 0x67452301;
ctx->buf[1] = 0xefcdab89;
ctx->buf[2] = 0x98badcfe;
ctx->buf[3] = 0x10325476;
ctx->bits[0] = 0;
ctx->bits[1] = 0;
}
/*
* Update context to reflect the concatenation of another buffer full
* of bytes.
*/
void MD5Update(context_md5_t *ctx, const unsigned char *buf, unsigned len)
{
uint32_t t;
/* Update bitcount */
t = ctx->bits[0];
if ((ctx->bits[0] = t + ((uint32_t) len << 3)) < t)
ctx->bits[1]++; /* Carry from low to high */
ctx->bits[1] += len >> 29;
t = (t >> 3) & 0x3f; /* Bytes already in shsInfo->data */
/* Handle any leading odd-sized chunks */
if (t) {
unsigned char *p = ctx->in.b8 + t;
t = 64 - t;
if (len < t) {
memcpy(p, buf, len);
return;
}
memcpy(p, buf, t);
byteReverse(ctx->in.b32, 16);
MD5Transform(ctx->buf, ctx->in.b32);
buf += t;
len -= t;
}
/* Process data in 64-byte chunks */
while (len >= 64) {
memcpy(ctx->in.b8, buf, 64);
byteReverse(ctx->in.b32, 16);
MD5Transform(ctx->buf, ctx->in.b32);
buf += 64;
len -= 64;
}
/* Handle any remaining bytes of data. */
memcpy(ctx->in.b8, buf, len);
}
/*
* Final wrapup - pad to 64-byte boundary with the bit pattern
* 1 0* (64-bit count of bits processed, MSB-first)
*/
void MD5Final(unsigned char digest[16], context_md5_t *ctx)
{
unsigned count;
unsigned char *p;
/* Compute number of bytes mod 64 */
count = (ctx->bits[0] >> 3) & 0x3F;
/* Set the first char of padding to 0x80. This is safe since there is
always at least one byte free */
p = ctx->in.b8 + count;
*p++ = 0x80;
/* Bytes of padding needed to make 64 bytes */
count = 64 - 1 - count;
/* Pad out to 56 mod 64 */
if (count < 8) {
/* Two lots of padding: Pad the first block to 64 bytes */
memset(p, 0, count);
byteReverse(ctx->in.b32, 16);
MD5Transform(ctx->buf, ctx->in.b32);
/* Now fill the next block with 56 bytes */
memset(ctx->in.b8, 0, 56);
} else {
/* Pad block to 56 bytes */
memset(p, 0, count - 8);
}
byteReverse(ctx->in.b32, 14);
/* Append length in bits and transform */
ctx->in.b32[14] = ctx->bits[0];
ctx->in.b32[15] = ctx->bits[1];
MD5Transform(ctx->buf, ctx->in.b32);
byteReverse((uint32_t *) ctx->buf, 4);
memcpy(digest, ctx->buf, 16);
memset(ctx, 0, sizeof(* ctx)); /* In case it's sensitive */
/* The original version of this code omitted the asterisk. In
effect, only the first part of ctx was wiped with zeros, not
the whole thing. Bug found by Derek Jones. Original line: */
// memset(ctx, 0, sizeof(ctx)); /* In case it's sensitive */
}
/* The four core functions - F1 is optimized somewhat */
/* #define F1(x, y, z) (x & y | ~x & z) */
#define F1(x, y, z) (z ^ (x & (y ^ z)))
#define F2(x, y, z) F1(z, x, y)
#define F3(x, y, z) (x ^ y ^ z)
#define F4(x, y, z) (y ^ (x | ~z))
/* This is the central step in the MD5 algorithm. */
#define MD5STEP(f, w, x, y, z, data, s) \
( w += f(x, y, z) + data, w = w<<s | w>>(32-s), w += x )
/*
* The core of the MD5 algorithm, this alters an existing MD5 hash to
* reflect the addition of 16 longwords of new data. MD5Update blocks
* the data and converts bytes into longwords for this routine.
*/
static void MD5Transform(uint32_t buf[4], uint32_t const in[16])
{
register uint32_t a, b, c, d;
a = buf[0];
b = buf[1];
c = buf[2];
d = buf[3];
MD5STEP(F1, a, b, c, d, in[0] + 0xd76aa478, 7);
MD5STEP(F1, d, a, b, c, in[1] + 0xe8c7b756, 12);
MD5STEP(F1, c, d, a, b, in[2] + 0x242070db, 17);
MD5STEP(F1, b, c, d, a, in[3] + 0xc1bdceee, 22);
MD5STEP(F1, a, b, c, d, in[4] + 0xf57c0faf, 7);
MD5STEP(F1, d, a, b, c, in[5] + 0x4787c62a, 12);
MD5STEP(F1, c, d, a, b, in[6] + 0xa8304613, 17);
MD5STEP(F1, b, c, d, a, in[7] + 0xfd469501, 22);
MD5STEP(F1, a, b, c, d, in[8] + 0x698098d8, 7);
MD5STEP(F1, d, a, b, c, in[9] + 0x8b44f7af, 12);
MD5STEP(F1, c, d, a, b, in[10] + 0xffff5bb1, 17);
MD5STEP(F1, b, c, d, a, in[11] + 0x895cd7be, 22);
MD5STEP(F1, a, b, c, d, in[12] + 0x6b901122, 7);
MD5STEP(F1, d, a, b, c, in[13] + 0xfd987193, 12);
MD5STEP(F1, c, d, a, b, in[14] + 0xa679438e, 17);
MD5STEP(F1, b, c, d, a, in[15] + 0x49b40821, 22);
MD5STEP(F2, a, b, c, d, in[1] + 0xf61e2562, 5);
MD5STEP(F2, d, a, b, c, in[6] + 0xc040b340, 9);
MD5STEP(F2, c, d, a, b, in[11] + 0x265e5a51, 14);
MD5STEP(F2, b, c, d, a, in[0] + 0xe9b6c7aa, 20);
MD5STEP(F2, a, b, c, d, in[5] + 0xd62f105d, 5);
MD5STEP(F2, d, a, b, c, in[10] + 0x02441453, 9);
MD5STEP(F2, c, d, a, b, in[15] + 0xd8a1e681, 14);
MD5STEP(F2, b, c, d, a, in[4] + 0xe7d3fbc8, 20);
MD5STEP(F2, a, b, c, d, in[9] + 0x21e1cde6, 5);
MD5STEP(F2, d, a, b, c, in[14] + 0xc33707d6, 9);
MD5STEP(F2, c, d, a, b, in[3] + 0xf4d50d87, 14);
MD5STEP(F2, b, c, d, a, in[8] + 0x455a14ed, 20);
MD5STEP(F2, a, b, c, d, in[13] + 0xa9e3e905, 5);
MD5STEP(F2, d, a, b, c, in[2] + 0xfcefa3f8, 9);
MD5STEP(F2, c, d, a, b, in[7] + 0x676f02d9, 14);
MD5STEP(F2, b, c, d, a, in[12] + 0x8d2a4c8a, 20);
MD5STEP(F3, a, b, c, d, in[5] + 0xfffa3942, 4);
MD5STEP(F3, d, a, b, c, in[8] + 0x8771f681, 11);
MD5STEP(F3, c, d, a, b, in[11] + 0x6d9d6122, 16);
MD5STEP(F3, b, c, d, a, in[14] + 0xfde5380c, 23);
MD5STEP(F3, a, b, c, d, in[1] + 0xa4beea44, 4);
MD5STEP(F3, d, a, b, c, in[4] + 0x4bdecfa9, 11);
MD5STEP(F3, c, d, a, b, in[7] + 0xf6bb4b60, 16);
MD5STEP(F3, b, c, d, a, in[10] + 0xbebfbc70, 23);
MD5STEP(F3, a, b, c, d, in[13] + 0x289b7ec6, 4);
MD5STEP(F3, d, a, b, c, in[0] + 0xeaa127fa, 11);
MD5STEP(F3, c, d, a, b, in[3] + 0xd4ef3085, 16);
MD5STEP(F3, b, c, d, a, in[6] + 0x04881d05, 23);
MD5STEP(F3, a, b, c, d, in[9] + 0xd9d4d039, 4);
MD5STEP(F3, d, a, b, c, in[12] + 0xe6db99e5, 11);
MD5STEP(F3, c, d, a, b, in[15] + 0x1fa27cf8, 16);
MD5STEP(F3, b, c, d, a, in[2] + 0xc4ac5665, 23);
MD5STEP(F4, a, b, c, d, in[0] + 0xf4292244, 6);
MD5STEP(F4, d, a, b, c, in[7] + 0x432aff97, 10);
MD5STEP(F4, c, d, a, b, in[14] + 0xab9423a7, 15);
MD5STEP(F4, b, c, d, a, in[5] + 0xfc93a039, 21);
MD5STEP(F4, a, b, c, d, in[12] + 0x655b59c3, 6);
MD5STEP(F4, d, a, b, c, in[3] + 0x8f0ccc92, 10);
MD5STEP(F4, c, d, a, b, in[10] + 0xffeff47d, 15);
MD5STEP(F4, b, c, d, a, in[1] + 0x85845dd1, 21);
MD5STEP(F4, a, b, c, d, in[8] + 0x6fa87e4f, 6);
MD5STEP(F4, d, a, b, c, in[15] + 0xfe2ce6e0, 10);
MD5STEP(F4, c, d, a, b, in[6] + 0xa3014314, 15);
MD5STEP(F4, b, c, d, a, in[13] + 0x4e0811a1, 21);
MD5STEP(F4, a, b, c, d, in[4] + 0xf7537e82, 6);
MD5STEP(F4, d, a, b, c, in[11] + 0xbd3af235, 10);
MD5STEP(F4, c, d, a, b, in[2] + 0x2ad7d2bb, 15);
MD5STEP(F4, b, c, d, a, in[9] + 0xeb86d391, 21);
buf[0] += a;
buf[1] += b;
buf[2] += c;
buf[3] += d;
}

58
src/extras/libmd5.h Normal file
View file

@ -0,0 +1,58 @@
/* The copyright in this software is being made available under the BSD
* License, included below. This software may be subject to other third party
* and contributor rights, including patent rights, and no such rights are
* granted under this license.
*
* Copyright (c) 2010-2015, ITU/ISO/IEC
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* * Neither the name of the ITU/ISO/IEC nor the names of its contributors may
* be used to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
* THE POSSIBILITY OF SUCH DAMAGE.
*/
#pragma once
#include <stdint.h>
//! \ingroup libMD5
//! \{
typedef struct _context_md5_t {
uint32_t buf[4];
uint32_t bits[2];
union {
unsigned char b8[64];
uint32_t b32[16];
} in;
} context_md5_t;
#ifdef __cplusplus
extern "C" {
#endif
void MD5Init(context_md5_t *ctx);
void MD5Update(context_md5_t *ctx, const unsigned char *buf, unsigned len);
void MD5Final(unsigned char digest[16], context_md5_t *ctx);
#ifdef __cplusplus
}
#endif
//! \}

View file

@ -75,3 +75,18 @@ void kvz_image_checksum(const kvz_picture *im, unsigned char checksum_out[][SEI_
kvz_array_checksum(im->u, im->height >> 1, im->width >> 1, im->width >> 1, checksum_out[1], bitdepth);
kvz_array_checksum(im->v, im->height >> 1, im->width >> 1, im->width >> 1, checksum_out[2], bitdepth);
}
/*!
\brief Calculate md5 for all colors of the picture.
\param im The image that md5 is calculated for.
\param checksum_out Result of the calculation.
\returns Void
*/
void kvz_image_md5(const kvz_picture *im, unsigned char checksum_out[][SEI_HASH_MAX_LENGTH], const uint8_t bitdepth)
{
kvz_array_md5(im->y, im->height, im->width, im->width, checksum_out[0], bitdepth);
/* The number of chroma pixels is half that of luma. */
kvz_array_md5(im->u, im->height >> 1, im->width >> 1, im->width >> 1, checksum_out[1], bitdepth);
kvz_array_md5(im->v, im->height >> 1, im->width >> 1, im->width >> 1, checksum_out[2], bitdepth);
}

View file

@ -33,7 +33,7 @@
#include "image.h"
#include "bitstream.h"
#define SEI_HASH_MAX_LENGTH 4
#define SEI_HASH_MAX_LENGTH 16
//////////////////////////////////////////////////////////////////////////
// FUNCTIONS
@ -41,6 +41,9 @@ void kvz_nal_write(bitstream_t * const bitstream, const uint8_t nal_type,
const uint8_t temporal_id, const int long_start_code);
void kvz_image_checksum(const kvz_picture *im,
unsigned char checksum_out[][SEI_HASH_MAX_LENGTH], const uint8_t bitdepth);
void kvz_image_md5(const kvz_picture *im,
unsigned char checksum_out[][SEI_HASH_MAX_LENGTH],
const uint8_t bitdepth);

View file

@ -21,10 +21,28 @@
#include <stdlib.h>
#include <assert.h>
#include "extras/libmd5.h"
#include "strategyselector.h"
#include "nal.h"
static void array_md5_generic(const kvz_pixel* data,
const int height, const int width,
const int stride,
unsigned char checksum_out[SEI_HASH_MAX_LENGTH], const uint8_t bitdepth)
{
assert(SEI_HASH_MAX_LENGTH >= 16);
context_md5_t md5_ctx;
MD5Init(&md5_ctx);
unsigned bytes = width * height * sizeof(kvz_pixel);
MD5Update(&md5_ctx, (const unsigned char *)data, bytes);
MD5Final(checksum_out, &md5_ctx);
}
static void array_checksum_generic(const kvz_pixel* data,
const int height, const int width,
const int stride,
@ -150,6 +168,7 @@ static void array_checksum_generic8(const kvz_pixel* data,
int kvz_strategy_register_nal_generic(void* opaque, uint8_t bitdepth) {
bool success = true;
success &= kvz_strategyselector_register(opaque, "array_md5", "generic", 0, &array_md5_generic);
success &= kvz_strategyselector_register(opaque, "array_checksum", "generic", 0, &array_checksum_generic);
success &= kvz_strategyselector_register(opaque, "array_checksum", "generic4", 1, &array_checksum_generic4);
success &= kvz_strategyselector_register(opaque, "array_checksum", "generic8", 2, &array_checksum_generic8);

View file

@ -26,6 +26,10 @@ void (*kvz_array_checksum)(const kvz_pixel* data,
const int height, const int width,
const int stride,
unsigned char checksum_out[SEI_HASH_MAX_LENGTH], const uint8_t bitdepth);
void (*kvz_array_md5)(const kvz_pixel* data,
const int height, const int width,
const int stride,
unsigned char checksum_out[SEI_HASH_MAX_LENGTH], const uint8_t bitdepth);
int kvz_strategy_register_nal(void* opaque, uint8_t bitdepth) {

View file

@ -43,12 +43,14 @@ typedef void (*array_checksum_func)(const kvz_pixel* data,
const int stride,
unsigned char checksum_out[SEI_HASH_MAX_LENGTH], const uint8_t bitdepth);
extern array_checksum_func kvz_array_checksum;
extern array_checksum_func kvz_array_md5;
int kvz_strategy_register_nal(void* opaque, uint8_t bitdepth);
#define STRATEGIES_NAL_EXPORTS \
{"array_checksum", (void**) &kvz_array_checksum},
{"array_checksum", (void**) &kvz_array_checksum},\
{"array_md5", (void**) &kvz_array_md5},
#endif //STRATEGIES_NAL_H_