Public/VeraCrypt
1
0
Fork 0
mirror of https://github.com/veracrypt/VeraCrypt.git synced 2025-11-11 02:58:02 -06:00
Code Activity

Implement SHA256 acceleration on ARM64 platforms using CPU instructions

Browse Source
This commit is contained in:
Mounir IDRASSI
2025-01-26 16:21:13 +01:00
parent 5ff256a53d
commit 247c98d954
12 changed files with 267 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/Crypto/Crypto.vcxproj
View File

@@ -255,6 +255,10 @@
<ClCompile Include="SerpentFast.c" />
<ClCompile Include="SerpentFast_simd.cpp" />
<ClCompile Include="Sha2.c" />
<ClCompile Include="sha256_armv8.c">
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="Sha2Intel.c" />
<ClCompile Include="Streebog.c" />
<ClCompile Include="t1ha2.c" />

3
src/Crypto/Crypto.vcxproj.filters
View File

@@ -96,6 +96,9 @@
<ClCompile Include="Aes_hw_armv8.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="sha256_armv8.c">
<Filter>Source Files</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="Aes.h">

17
src/Crypto/Sha2.c
View File

@@ -315,6 +315,10 @@ extern "C"
void VC_CDECL sha256_compress_nayuki(uint_32t state[8], const uint_8t block[64]);
#endif
#if CRYPTOPP_ARM_SHA2_AVAILABLE
void sha256_compress_digest_armv8(const void* input_data, uint_32t digest[8], uint_64t num_blks);
#endif
#if defined(__cplusplus)
}
#endif
@@ -757,6 +761,13 @@ void SSE2Sha256Transform(sha256_ctx* ctx, void* mp, uint_64t num_blks)
}
#endif
#if CRYPTOPP_ARM_SHA2_AVAILABLE
void ArmSha256Transform(sha256_ctx* ctx, void* mp, uint_64t num_blks)
{
sha256_compress_digest_armv8(mp, ctx->hash, num_blks);
}
#endif
#if CRYPTOPP_BOOL_X86 || CRYPTOPP_BOOL_X32
void Sha256AsmTransform(sha256_ctx* ctx, void* mp, uint_64t num_blks)
{
@@ -805,6 +816,12 @@ void sha256_begin(sha256_ctx* ctx)
else
#endif
#if CRYPTOPP_ARM_SHA2_AVAILABLE
if (HasSHA256())
sha256transfunc = ArmSha256Transform;
else
#endif
#if CRYPTOPP_BOOL_X86 || CRYPTOPP_BOOL_X32
sha256transfunc = Sha256AsmTransform;
#else

13
src/Crypto/config.h
View File

@@ -240,6 +240,19 @@
# endif // Platforms
#endif
// ARMv8 and SHA-1, SHA-256. -march=armv8-a+crypto or above must be present
// Requires GCC 4.8, Clang 3.3 or Visual Studio 2017
#if !defined(CRYPTOPP_ARM_SHA_AVAILABLE) && !defined(CRYPTOPP_DISABLE_ARM_SHA)
# if defined(__aarch32__) || defined(__aarch64__) || defined(_M_ARM64)
# if defined(__ARM_FEATURE_CRYPTO) || (CRYPTOPP_GCC_VERSION >= 40800) || \
(CRYPTOPP_LLVM_CLANG_VERSION >= 30300) || (CRYPTOPP_APPLE_CLANG_VERSION >= 40300) || \
(CRYPTOPP_MSC_VERSION >= 1916)
# define CRYPTOPP_ARM_SHA1_AVAILABLE 1
# define CRYPTOPP_ARM_SHA2_AVAILABLE 1
# endif // Compilers
# endif // Platforms
#endif
// Undo the ASM and Intrinsic related defines due to X32.
#if CRYPTOPP_BOOL_X32
# undef CRYPTOPP_BOOL_X64

29
src/Crypto/cpu.c
View File

@@ -475,9 +475,13 @@ void DisableCPUExtendedFeatures ()
#ifndef HWCAP_AES
# define HWCAP_AES (1 << 3)
#endif
#ifndef HWCAP_SHA2
# define HWCAP_SHA2 (1 << 6)
#endif
#endif
volatile int g_hasAESARM = 0;
volatile int g_hasSHA256ARM = 0;
inline int CPU_QueryAES()
{
@@ -503,9 +507,34 @@ inline int CPU_QueryAES()
#endif
}
inline int CPU_QuerySHA2()
{
#if defined(CRYPTOPP_ARM_SHA2_AVAILABLE)
#if defined(__linux__) && defined(__aarch64__)
if ((getauxval(AT_HWCAP) & HWCAP_SHA2) != 0)
return 1;
#elif defined(__APPLE__) && defined(__aarch64__)
// Apple Sillcon (M1) and later
return 1;
#elif defined(_WIN32) && defined(_M_ARM64)
#ifdef TC_WINDOWS_DRIVER
if (ExIsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE) != 0)
return 1;
#else
if (IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE) != 0)
return 1;
#endif
#endif
return 0;
#else
return 0;
#endif
}
void DetectArmFeatures()
{
g_hasAESARM = CPU_QueryAES();
g_hasSHA256ARM = CPU_QuerySHA2();
}
#endif

2
src/Crypto/cpu.h
View File

@@ -298,9 +298,11 @@ extern "C" {
#endif
extern volatile int g_hasAESARM;
extern volatile int g_hasSHA256ARM;
void DetectArmFeatures();
#define HasAESNI() g_hasAESARM
#define HasSHA256() g_hasSHA256ARM
#if defined(__cplusplus)
}

184
src/Crypto/sha256_armv8.c Normal file
View File

@@ -0,0 +1,184 @@
/*
* SHA-256 using CPU instructions in ARMv8
*
* Contributed by Jeffrey Walton. Based on public domain code by
* Johannes Schneiders, Skip Hovsmith and Barry O'Rourke.
*
* Further changes (C) 2020 Jack Lloyd
*
* Botan is released under the Simplified BSD License (see license.txt)
*/
/* Modified and adapted for VeraCrypt */
#include "Common/Tcdefs.h"
#if !defined(_UEFI)
#include <memory.h>
#include <stdlib.h>
#endif
#include "cpu.h"
#include "misc.h"
#if CRYPTOPP_ARM_SHA2_AVAILABLE
#include <arm_neon.h>
CRYPTOPP_ALIGN_DATA(64) static const uint32 K[] = {
0x428A2F98, 0x71374491, 0xB5C0FBCF, 0xE9B5DBA5, 0x3956C25B, 0x59F111F1, 0x923F82A4, 0xAB1C5ED5,
0xD807AA98, 0x12835B01, 0x243185BE, 0x550C7DC3, 0x72BE5D74, 0x80DEB1FE, 0x9BDC06A7, 0xC19BF174,
0xE49B69C1, 0xEFBE4786, 0x0FC19DC6, 0x240CA1CC, 0x2DE92C6F, 0x4A7484AA, 0x5CB0A9DC, 0x76F988DA,
0x983E5152, 0xA831C66D, 0xB00327C8, 0xBF597FC7, 0xC6E00BF3, 0xD5A79147, 0x06CA6351, 0x14292967,
0x27B70A85, 0x2E1B2138, 0x4D2C6DFC, 0x53380D13, 0x650A7354, 0x766A0ABB, 0x81C2C92E, 0x92722C85,
0xA2BFE8A1, 0xA81A664B, 0xC24B8B70, 0xC76C51A3, 0xD192E819, 0xD6990624, 0xF40E3585, 0x106AA070,
0x19A4C116, 0x1E376C08, 0x2748774C, 0x34B0BCB5, 0x391C0CB3, 0x4ED8AA4A, 0x5B9CCA4F, 0x682E6FF3,
0x748F82EE, 0x78A5636F, 0x84C87814, 0x8CC70208, 0x90BEFFFA, 0xA4506CEB, 0xBEF9A3F7, 0xC67178F2,
};
void sha256_compress_digest_armv8(void* input_data, uint32 digest[8], uint64 num_blks) {
// Load initial values
uint32x4_t STATE0 = vld1q_u32(&digest[0]);
uint32x4_t STATE1 = vld1q_u32(&digest[4]);
// Intermediate void* cast due to https://llvm.org/bugs/show_bug.cgi?id=20670
const uint32* input32 = (const uint32*)(const void*)input_data;
while (num_blks > 0) {
// Save current state
const uint32x4_t ABCD_SAVE = STATE0;
const uint32x4_t EFGH_SAVE = STATE1;
uint32x4_t MSG0 = vld1q_u32(input32 + 0);
uint32x4_t MSG1 = vld1q_u32(input32 + 4);
uint32x4_t MSG2 = vld1q_u32(input32 + 8);
uint32x4_t MSG3 = vld1q_u32(input32 + 12);
MSG0 = vreinterpretq_u32_u8(vrev32q_u8(vreinterpretq_u8_u32(MSG0)));
MSG1 = vreinterpretq_u32_u8(vrev32q_u8(vreinterpretq_u8_u32(MSG1)));
MSG2 = vreinterpretq_u32_u8(vrev32q_u8(vreinterpretq_u8_u32(MSG2)));
MSG3 = vreinterpretq_u32_u8(vrev32q_u8(vreinterpretq_u8_u32(MSG3)));
uint32x4_t MSG_K, TSTATE;
// Rounds 0-3
MSG_K = vaddq_u32(MSG0, vld1q_u32(&K[4 * 0]));
TSTATE = vsha256hq_u32(STATE0, STATE1, MSG_K);
STATE1 = vsha256h2q_u32(STATE1, STATE0, MSG_K);
STATE0 = TSTATE;
MSG0 = vsha256su1q_u32(vsha256su0q_u32(MSG0, MSG1), MSG2, MSG3);
// Rounds 4-7
MSG_K = vaddq_u32(MSG1, vld1q_u32(&K[4 * 1]));
TSTATE = vsha256hq_u32(STATE0, STATE1, MSG_K);
STATE1 = vsha256h2q_u32(STATE1, STATE0, MSG_K);
STATE0 = TSTATE;
MSG1 = vsha256su1q_u32(vsha256su0q_u32(MSG1, MSG2), MSG3, MSG0);
// Rounds 8-11
MSG_K = vaddq_u32(MSG2, vld1q_u32(&K[4 * 2]));
TSTATE = vsha256hq_u32(STATE0, STATE1, MSG_K);
STATE1 = vsha256h2q_u32(STATE1, STATE0, MSG_K);
STATE0 = TSTATE;
MSG2 = vsha256su1q_u32(vsha256su0q_u32(MSG2, MSG3), MSG0, MSG1);
// Rounds 12-15
MSG_K = vaddq_u32(MSG3, vld1q_u32(&K[4 * 3]));
TSTATE = vsha256hq_u32(STATE0, STATE1, MSG_K);
STATE1 = vsha256h2q_u32(STATE1, STATE0, MSG_K);
STATE0 = TSTATE;
MSG3 = vsha256su1q_u32(vsha256su0q_u32(MSG3, MSG0), MSG1, MSG2);
// Rounds 16-19
MSG_K = vaddq_u32(MSG0, vld1q_u32(&K[4 * 4]));
TSTATE = vsha256hq_u32(STATE0, STATE1, MSG_K);
STATE1 = vsha256h2q_u32(STATE1, STATE0, MSG_K);
STATE0 = TSTATE;
MSG0 = vsha256su1q_u32(vsha256su0q_u32(MSG0, MSG1), MSG2, MSG3);
// Rounds 20-23
MSG_K = vaddq_u32(MSG1, vld1q_u32(&K[4 * 5]));
TSTATE = vsha256hq_u32(STATE0, STATE1, MSG_K);
STATE1 = vsha256h2q_u32(STATE1, STATE0, MSG_K);
STATE0 = TSTATE;
MSG1 = vsha256su1q_u32(vsha256su0q_u32(MSG1, MSG2), MSG3, MSG0);
// Rounds 24-27
MSG_K = vaddq_u32(MSG2, vld1q_u32(&K[4 * 6]));
TSTATE = vsha256hq_u32(STATE0, STATE1, MSG_K);
STATE1 = vsha256h2q_u32(STATE1, STATE0, MSG_K);
STATE0 = TSTATE;
MSG2 = vsha256su1q_u32(vsha256su0q_u32(MSG2, MSG3), MSG0, MSG1);
// Rounds 28-31
MSG_K = vaddq_u32(MSG3, vld1q_u32(&K[4 * 7]));
TSTATE = vsha256hq_u32(STATE0, STATE1, MSG_K);
STATE1 = vsha256h2q_u32(STATE1, STATE0, MSG_K);
STATE0 = TSTATE;
MSG3 = vsha256su1q_u32(vsha256su0q_u32(MSG3, MSG0), MSG1, MSG2);
// Rounds 32-35
MSG_K = vaddq_u32(MSG0, vld1q_u32(&K[4 * 8]));
TSTATE = vsha256hq_u32(STATE0, STATE1, MSG_K);
STATE1 = vsha256h2q_u32(STATE1, STATE0, MSG_K);
STATE0 = TSTATE;
MSG0 = vsha256su1q_u32(vsha256su0q_u32(MSG0, MSG1), MSG2, MSG3);
// Rounds 36-39
MSG_K = vaddq_u32(MSG1, vld1q_u32(&K[4 * 9]));
TSTATE = vsha256hq_u32(STATE0, STATE1, MSG_K);
STATE1 = vsha256h2q_u32(STATE1, STATE0, MSG_K);
STATE0 = TSTATE;
MSG1 = vsha256su1q_u32(vsha256su0q_u32(MSG1, MSG2), MSG3, MSG0);
// Rounds 40-43
MSG_K = vaddq_u32(MSG2, vld1q_u32(&K[4 * 10]));
TSTATE = vsha256hq_u32(STATE0, STATE1, MSG_K);
STATE1 = vsha256h2q_u32(STATE1, STATE0, MSG_K);
STATE0 = TSTATE;
MSG2 = vsha256su1q_u32(vsha256su0q_u32(MSG2, MSG3), MSG0, MSG1);
// Rounds 44-47
MSG_K = vaddq_u32(MSG3, vld1q_u32(&K[4 * 11]));
TSTATE = vsha256hq_u32(STATE0, STATE1, MSG_K);
STATE1 = vsha256h2q_u32(STATE1, STATE0, MSG_K);
STATE0 = TSTATE;
MSG3 = vsha256su1q_u32(vsha256su0q_u32(MSG3, MSG0), MSG1, MSG2);
// Rounds 48-51
MSG_K = vaddq_u32(MSG0, vld1q_u32(&K[4 * 12]));
TSTATE = vsha256hq_u32(STATE0, STATE1, MSG_K);
STATE1 = vsha256h2q_u32(STATE1, STATE0, MSG_K);
STATE0 = TSTATE;
// Rounds 52-55
MSG_K = vaddq_u32(MSG1, vld1q_u32(&K[4 * 13]));
TSTATE = vsha256hq_u32(STATE0, STATE1, MSG_K);
STATE1 = vsha256h2q_u32(STATE1, STATE0, MSG_K);
STATE0 = TSTATE;
// Rounds 56-59
MSG_K = vaddq_u32(MSG2, vld1q_u32(&K[4 * 14]));
TSTATE = vsha256hq_u32(STATE0, STATE1, MSG_K);
STATE1 = vsha256h2q_u32(STATE1, STATE0, MSG_K);
STATE0 = TSTATE;
// Rounds 60-63
MSG_K = vaddq_u32(MSG3, vld1q_u32(&K[4 * 15]));
TSTATE = vsha256hq_u32(STATE0, STATE1, MSG_K);
STATE1 = vsha256h2q_u32(STATE1, STATE0, MSG_K);
STATE0 = TSTATE;
// Add back to state
STATE0 = vaddq_u32(STATE0, ABCD_SAVE);
STATE1 = vaddq_u32(STATE1, EFGH_SAVE);
input32 += 64 / 4;
num_blks--;
}
// Save state
vst1q_u32(&digest[0], STATE0);
vst1q_u32(&digest[4], STATE1);
}
#endif

4
src/Driver/Driver.vcxproj
View File

@@ -272,6 +272,10 @@ copy $(OutDir)veracrypt.inf "$(SolutionDir)Debug\Setup Files\veracrypt.inf"</Com
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">false</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\Crypto\Sha2.c" />
<ClCompile Include="..\Crypto\sha256_armv8.c">
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\Crypto\Sha2Intel.c" />
<ClCompile Include="..\Crypto\Streebog.c" />
<ClCompile Include="..\Crypto\t1ha2.c" />

3
src/Driver/Driver.vcxproj.filters
View File

@@ -168,6 +168,9 @@
<ClCompile Include="..\Crypto\Aes_hw_armv8.c">
<Filter>Crypto\Source Files</Filter>
</ClCompile>
<ClCompile Include="..\Crypto\sha256_armv8.c">
<Filter>Crypto\Source Files</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\Common\Tcdefs.h">

BIN
src/Release/Setup Files/veracrypt-arm64.sys
View File

Binary file not shown.

BIN
src/Release/Setup Files/veracrypt-x64.sys
View File

Binary file not shown.

8
src/Volume/Volume.make
View File

@@ -50,6 +50,7 @@ ifneq "$(COMPILE_ASM)" "false"
OBJSEX += ../Crypto/Camellia_aesni_asm.oo
OBJSEX += ../Crypto/sha256-nayuki.oo
OBJSEX += ../Crypto/sha512-nayuki.oo
OBJSEX += ../Crypto/sha256_armv8.oo
OBJSEX += ../Crypto/sha256_avx1.oo
OBJSEX += ../Crypto/sha256_avx2.oo
OBJSEX += ../Crypto/sha256_sse4.oo
@@ -82,6 +83,7 @@ else ifeq "$(CPU_ARCH)" "x64"
else ifeq "$(CPU_ARCH)" "arm64"
OBJARMV8CRYPTO += ../Crypto/Aes_hw_armv8.oarmv8crypto
OBJS += ../Crypto/Aescrypt.o
OBJARMV8CRYPTO += ../Crypto/sha256_armv8.oarmv8crypto
else
OBJS += ../Crypto/Aescrypt.o
endif
@@ -150,6 +152,12 @@ ifneq "$(COMPILE_ASM)" "false"
$(CC) $(CFLAGS_X64) -c ../Crypto/Aes_hw_armv8.c -o ../Crypto/Aes_hw_armv8_x64.o
lipo -create ../Crypto/Aes_hw_armv8_arm64.o ../Crypto/Aes_hw_armv8_x64.o -output ../Crypto/Aes_hw_armv8.oo
rm -fr ../Crypto/Aes_hw_armv8_arm64.o ../Crypto/Aes_hw_armv8_x64.o
../Crypto/sha256_armv8.oo: ../Crypto/sha256_armv8.c
@echo Compiling $(<F)
$(CC) $(CFLAGS_ARM64) -c ../Crypto/sha256_armv8.c -o ../Crypto/sha256_armv8_arm64.o
$(CC) $(CFLAGS_X64) -c ../Crypto/sha256_armv8.c -o ../Crypto/sha256_armv8_x64.o
lipo -create ../Crypto/sha256_armv8_arm64.o ../Crypto/sha256_armv8_x64.o -output ../Crypto/sha256_armv8.oo
rm -fr ../Crypto/sha256_armv8_arm64.o ../Crypto/sha256_armv8_x64.o
../Crypto/Aes_asm.oo: ../Crypto/Aes_x86.asm ../Crypto/Aes_x64.asm
@echo Assembling $(<F)
$(AS) $(ASFLAGS32) -o ../Crypto/Aes_x86.o ../Crypto/Aes_x86.asm