new sha/sha1 digesting funcs

include/common/tcommon.h

@@ -125,9 +125,8 @@ enum {
   TMQ_MSG_TYPE__POLL_RAW_DATA_RSP,
 };
 
-static const char* const tmqMsgTypeStr[] = {
-    "data", "meta", "ask ep", "meta data", "wal info", "batch meta", "raw data"
-};
+static const char* const tmqMsgTypeStr[] = {"data",     "meta",       "ask ep",  "meta data",
+                                            "wal info", "batch meta", "raw data"};
 
 enum {
   STREAM_INPUT__DATA_SUBMIT = 1,
@@ -441,9 +440,9 @@ typedef struct STUidTagInfo {
 #define TABLE_NAME_COLUMN_INDEX         6
 #define PRIMARY_KEY_COLUMN_INDEX        7
 
-//steam get result block column
-#define DATA_TS_COLUMN_INDEX            0
-#define DATA_VERSION_COLUMN_INDEX       1
+// steam get result block column
+#define DATA_TS_COLUMN_INDEX      0
+#define DATA_VERSION_COLUMN_INDEX 1
 
 // stream create table block column
 #define UD_TABLE_NAME_COLUMN_INDEX 0
@@ -458,6 +457,7 @@ int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol, char* likePat
 
 #define TSMA_RES_STB_POSTFIX          "_tsma_res_stb_"
 #define MD5_OUTPUT_LEN                32
+#define SHA1_OUTPUT_LEN               40
 #define TSMA_RES_STB_EXTRA_COLUMN_NUM 4  // 3 columns: _wstart, _wend, _wduration, 1 tag: tbname
 
 static inline bool isTsmaResSTb(const char* stbName) {

include/libs/function/functionMgt.h

@@ -122,7 +122,7 @@ typedef enum EFunctionType {
   FUNCTION_TYPE_FIND_IN_SET,
   FUNCTION_TYPE_LIKE_IN_SET,
   FUNCTION_TYPE_REGEXP_IN_SET,
-
+  FUNCTION_TYPE_SHA1,
 
   // conversion function
   FUNCTION_TYPE_CAST = 2000,
@@ -174,19 +174,19 @@ typedef enum EFunctionType {
   FUNCTION_TYPE_IROWTS_ORIGIN,
   FUNCTION_TYPE_GROUP_ID,
   FUNCTION_TYPE_IS_WINDOW_FILLED,
-  FUNCTION_TYPE_TPREV_TS,           // _tprev_ts
-  FUNCTION_TYPE_TCURRENT_TS,        // _tcurrent_ts
-  FUNCTION_TYPE_TNEXT_TS,           // _tnext_ts
-  FUNCTION_TYPE_TWSTART,            // _twstart
-  FUNCTION_TYPE_TWEND,              // _twend
-  FUNCTION_TYPE_TWDURATION,         // _twduration
-  FUNCTION_TYPE_TWROWNUM,           // _twrownum
-  FUNCTION_TYPE_TPREV_LOCALTIME,    // _tprev_localtime
-  FUNCTION_TYPE_TNEXT_LOCALTIME,    // _tnext_localtime
-  FUNCTION_TYPE_TLOCALTIME,         // _tlocaltime
-  FUNCTION_TYPE_TGRPID,             // _tgrpid
-  FUNCTION_TYPE_PLACEHOLDER_COLUMN, // %%n
-  FUNCTION_TYPE_PLACEHOLDER_TBNAME, // %%tbname
+  FUNCTION_TYPE_TPREV_TS,            // _tprev_ts
+  FUNCTION_TYPE_TCURRENT_TS,         // _tcurrent_ts
+  FUNCTION_TYPE_TNEXT_TS,            // _tnext_ts
+  FUNCTION_TYPE_TWSTART,             // _twstart
+  FUNCTION_TYPE_TWEND,               // _twend
+  FUNCTION_TYPE_TWDURATION,          // _twduration
+  FUNCTION_TYPE_TWROWNUM,            // _twrownum
+  FUNCTION_TYPE_TPREV_LOCALTIME,     // _tprev_localtime
+  FUNCTION_TYPE_TNEXT_LOCALTIME,     // _tnext_localtime
+  FUNCTION_TYPE_TLOCALTIME,          // _tlocaltime
+  FUNCTION_TYPE_TGRPID,              // _tgrpid
+  FUNCTION_TYPE_PLACEHOLDER_COLUMN,  // %%n
+  FUNCTION_TYPE_PLACEHOLDER_TBNAME,  // %%tbname
   FUNCTION_TYPE_IMPUTATION_ROWTS,
   FUNCTION_TYPE_IMPUTATION_MARK,
   FUNCTION_TYPE_ANOMALY_MARK,
@@ -395,9 +395,10 @@ int32_t fmGetStreamPesudoFuncEnv(int32_t funcId, SNodeList* pParamNodes, SFuncEx
 
 const void* fmGetStreamPesudoFuncVal(int32_t funcId, const SStreamRuntimeFuncInfo* pStreamRuntimeFuncInfo);
 
-void fmGetStreamPesudoFuncValTbname(int32_t funcId, const SStreamRuntimeFuncInfo* pStreamRuntimeFuncInfo, void** data, int32_t* dataLen);
-int32_t fmSetStreamPseudoFuncParamVal(int32_t funcId, SNodeList* pParamNodes, const SStreamRuntimeFuncInfo* pStreamRuntimeInfo);
-
+void fmGetStreamPesudoFuncValTbname(int32_t funcId, const SStreamRuntimeFuncInfo* pStreamRuntimeFuncInfo, void** data,
+                                    int32_t* dataLen);
+int32_t fmSetStreamPseudoFuncParamVal(int32_t funcId, SNodeList* pParamNodes,
+                                      const SStreamRuntimeFuncInfo* pStreamRuntimeInfo);
 
 #ifdef __cplusplus
 }

include/libs/scalar/scalar.h

@@ -95,6 +95,7 @@ int32_t ltrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOut
 int32_t rtrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t md5Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t shaFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t charFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t asciiFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t positionFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);

include/util/tsha.h

@@ -0,0 +1,26 @@
+#ifndef _TD_UTIL_SHA_H
+#define _TD_UTIL_SHA_H
+
+#include "os.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+  uint32_t      state[5];
+  uint32_t      count[2];
+  unsigned char buffer[64];
+} T_SHA1_CTX;
+
+void tSHA1Transform(uint32_t state[5], const unsigned char buffer[64]);
+void tSHA1Init(T_SHA1_CTX *context);
+void tSHA1Update(T_SHA1_CTX *context, const unsigned char *data, uint32_t len);
+void tSHA1Final(unsigned char digest[20], T_SHA1_CTX *context);
+void tSHA1(char *hash_out, const char *str, uint32_t len);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_UTIL_SHA_H*/

include/util/tutil.h

@@ -20,6 +20,7 @@
 #include "tdef.h"
 #include "thash.h"
 #include "tmd5.h"
+#include "tsha.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -215,17 +216,28 @@ static FORCE_INLINE int32_t taosHashBinary(char *pBuf, int32_t len) {
 
 static FORCE_INLINE int32_t taosCreateMD5Hash(char *pBuf, int32_t len) {
   T_MD5_CTX ctx;
+
   tMD5Init(&ctx);
   tMD5Update(&ctx, (uint8_t *)pBuf, len);
   tMD5Final(&ctx);
-  char   *p = pBuf;
-  int32_t resLen = 0;
+
   return sprintf(pBuf, "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", ctx.digest[0], ctx.digest[1],
                  ctx.digest[2], ctx.digest[3], ctx.digest[4], ctx.digest[5], ctx.digest[6], ctx.digest[7],
                  ctx.digest[8], ctx.digest[9], ctx.digest[10], ctx.digest[11], ctx.digest[12], ctx.digest[13],
                  ctx.digest[14], ctx.digest[15]);
 }
 
+static FORCE_INLINE int32_t taosCreateSHA1Hash(char *pBuf, int32_t len) {
+  uint8_t result[21] = {0};
+
+  tSHA1(result, pBuf, len);
+
+  return sprintf(pBuf, "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", result[0],
+                 result[1], result[2], result[3], result[4], result[5], result[6], result[7], result[8], result[9],
+                 result[10], result[11], result[12], result[13], result[14], result[15], result[16], result[17],
+                 result[18], result[19]);
+}
+
 static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, int32_t method, int32_t prefix,
                                              int32_t suffix) {
   if ((prefix == 0 && suffix == 0) || (tblen <= (prefix + suffix)) || (tblen <= -1 * (prefix + suffix)) ||
@@ -301,7 +313,7 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen,
 
 #define VND_CHECK_CODE(CODE, LINO, LABEL) TSDB_CHECK_CODE(CODE, LINO, LABEL)
 
-#define TCONTAINER_OF(ptr, type, member) ((type *)((char *)(ptr) - offsetof(type, member)))
+#define TCONTAINER_OF(ptr, type, member) ((type *)((char *)(ptr)-offsetof(type, member)))
 
 #define TAOS_GET_TERRNO(code) (terrno == 0 ? code : terrno)
 

source/libs/function/src/builtins.c

@@ -1859,6 +1859,9 @@ static int32_t translateOutVarchar(SFunctionNode* pFunc, char* pErrBuf, int32_t
     case FUNCTION_TYPE_MD5:
       bytes = MD5_OUTPUT_LEN + VARSTR_HEADER_SIZE;
       break;
+    case FUNCTION_TYPE_SHA1:
+      bytes = SHA1_OUTPUT_LEN + VARSTR_HEADER_SIZE;
+      break;
     case FUNCTION_TYPE_USER:
     case FUNCTION_TYPE_CURRENT_USER:
       bytes = TSDB_USER_LEN;
@@ -6690,6 +6693,48 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .sprocessFunc = base64FunctionFrom,
     .finalizeFunc = NULL
   },
+  {
+    .name = "sha",
+    .type = FUNCTION_TYPE_SHA1,
+    .classification = FUNC_MGT_SCALAR_FUNC,
+    .parameters = {.minParamNum = 1,
+                   .maxParamNum = 1,
+                   .paramInfoPattern = 1,
+                   .inputParaInfo[0][0] = {.isLastParam = true,
+                                           .startParam = 1,
+                                           .endParam = 1,
+                                           .validDataType = FUNC_PARAM_SUPPORT_VARCHAR_TYPE,
+                                           .validNodeType = FUNC_PARAM_SUPPORT_EXPR_NODE,
+                                           .paramAttribute = FUNC_PARAM_NO_SPECIFIC_ATTRIBUTE,
+                                           .valueRangeFlag = FUNC_PARAM_NO_SPECIFIC_VALUE,},
+                   .outputParaInfo = {.validDataType = FUNC_PARAM_SUPPORT_VARCHAR_TYPE}},
+    .translateFunc = translateOutVarchar,
+    .getEnvFunc = NULL,
+    .initFunc = NULL,
+    .sprocessFunc = shaFunction,
+    .finalizeFunc = NULL
+  },
+  {
+    .name = "sha1",
+    .type = FUNCTION_TYPE_SHA1,
+    .classification = FUNC_MGT_SCALAR_FUNC,
+    .parameters = {.minParamNum = 1,
+                   .maxParamNum = 1,
+                   .paramInfoPattern = 1,
+                   .inputParaInfo[0][0] = {.isLastParam = true,
+                                           .startParam = 1,
+                                           .endParam = 1,
+                                           .validDataType = FUNC_PARAM_SUPPORT_VARCHAR_TYPE,
+                                           .validNodeType = FUNC_PARAM_SUPPORT_EXPR_NODE,
+                                           .paramAttribute = FUNC_PARAM_NO_SPECIFIC_ATTRIBUTE,
+                                           .valueRangeFlag = FUNC_PARAM_NO_SPECIFIC_VALUE,},
+                   .outputParaInfo = {.validDataType = FUNC_PARAM_SUPPORT_VARCHAR_TYPE}},
+    .translateFunc = translateOutVarchar,
+    .getEnvFunc = NULL,
+    .initFunc = NULL,
+    .sprocessFunc = shaFunction,
+    .finalizeFunc = NULL
+  },
 };
 // clang-format on
 

source/libs/parser/src/parInsertSql.c

@@ -636,6 +636,45 @@ static int32_t parseBinary(SInsertParseContext* pCxt, const char** ppSql, SToken
       (void)memcpy(*pData, input, inputBytes);
       int32_t len = taosCreateMD5Hash(*pData, inputBytes);
       *nData = len;
+    } else if (0 == strncasecmp(pToken->z, "sha(", 4) || 0 == strncasecmp(pToken->z, "sha1(", 5)) {
+      NEXT_VALID_TOKEN(*ppSql, *pToken);
+      if (TK_NK_LP != pToken->type) {
+        return buildSyntaxErrMsg(&pCxt->msg, "( expected", pToken->z);
+      }
+
+      NEXT_VALID_TOKEN(*ppSql, *pToken);
+      if (TK_NULL == pToken->type) {
+        NEXT_VALID_TOKEN(*ppSql, *pToken);
+        if (TK_NK_RP == pToken->type) {
+          pVal->flag = CV_FLAG_NULL;
+
+          return TSDB_CODE_SUCCESS;
+        }
+      } else if (TK_NK_STRING != pToken->type) {
+        return buildSyntaxErrMsg(&pCxt->msg, "string expected", pToken->z);
+      }
+
+      inputBytes = trimString(pToken->z, pToken->n, tmpTokenBuf, TSDB_MAX_BYTES_PER_ROW);
+      input = tmpTokenBuf;
+
+      NEXT_VALID_TOKEN(*ppSql, *pToken);
+      if (TK_NK_RP != pToken->type) {
+        return buildSyntaxErrMsg(&pCxt->msg, ") expected", pToken->z);
+      }
+
+      if (SHA1_OUTPUT_LEN + VARSTR_HEADER_SIZE > bytes) {
+        return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name);
+      }
+
+      int32_t bufLen = TMAX(SHA1_OUTPUT_LEN + VARSTR_HEADER_SIZE + 1, inputBytes);
+      *pData = taosMemoryMalloc(bufLen);
+      if (NULL == *pData) {
+        return terrno;
+      }
+
+      (void)memcpy(*pData, input, inputBytes);
+      int32_t len = taosCreateSHA1Hash(*pData, inputBytes);
+      *nData = len;
     } else {
       return buildSyntaxErrMsg(&pCxt->msg, "invalid identifier", pToken->z);
     }

source/libs/scalar/src/sclfunc.c

@@ -1761,6 +1761,44 @@ int32_t md5Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t shaFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
+  SColumnInfoData *pInputData = pInput->columnData;
+  SColumnInfoData *pOutputData = pOutput->columnData;
+  int32_t          bufLen = TMAX(SHA1_OUTPUT_LEN + VARSTR_HEADER_SIZE + 1, pInputData->info.bytes);
+  char            *pOutputBuf = taosMemoryMalloc(bufLen);
+  if (!pOutputBuf) {
+    qError("md5 function alloc memory failed");
+    return terrno;
+  }
+  for (int32_t i = 0; i < pInput->numOfRows; ++i) {
+    if (colDataIsNull_s(pInputData, i)) {
+      colDataSetNULL(pOutputData, i);
+      continue;
+    }
+    char *input = colDataGetData(pInput[0].columnData, i);
+    if (bufLen < varDataLen(input) + VARSTR_HEADER_SIZE) {
+      bufLen = varDataLen(input) + VARSTR_HEADER_SIZE;
+      pOutputBuf = taosMemoryRealloc(pOutputBuf, bufLen);
+      if (!pOutputBuf) {
+        qError("md5 function alloc memory failed");
+        return terrno;
+      }
+    }
+    char *output = pOutputBuf;
+    (void)memcpy(varDataVal(output), varDataVal(input), varDataLen(input));
+    int32_t len = taosCreateSHA1Hash(varDataVal(output), varDataLen(input));
+    varDataSetLen(output, len);
+    int32_t code = colDataSetVal(pOutputData, i, output, false);
+    if (TSDB_CODE_SUCCESS != code) {
+      taosMemoryFree(pOutputBuf);
+      SCL_ERR_RET(code);
+    }
+  }
+  pOutput->numOfRows = pInput->numOfRows;
+  taosMemoryFree(pOutputBuf);
+  return TSDB_CODE_SUCCESS;
+}
+
 /* pre-calculated table for 32-bit CRC */
 static uint32_t crc32_table[] = {
     0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA, 0x076DC419, 0x706AF48F, 0xE963A535, 0x9E6495A3, 0x0EDB8832,