Skip to content

feat(md5): support md5 in inserting #33416

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 8 commits into
base: 3.0
Choose a base branch
from
+876 −230
Open

feat(md5): support md5 in inserting #33416

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
8b3608b
feat(encoding): new security funcs
stephenkgu Oct 27, 2025
36b1751
Merge branch '3.0' into fix/TS-7235
stephenkgu Oct 27, 2025
c154911
fix null with {from, to}_base64
stephenkgu Oct 27, 2025
25876aa
feat(md5): support md5 in inserting
stephenkgu Oct 29, 2025
096c37a
fix failed decoding cases
stephenkgu Oct 29, 2025
d6c20c2
new sha/sha1 digesting funcs
stephenkgu Oct 30, 2025
1174581
fix CI compiling issue
stephenkgu Oct 30, 2025
da093d8
Merge branch '3.0' into fix/TS-7235-md5
stephenkgu Oct 31, 2025
File filter

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 21 additions & 15 deletions include/libs/scalar/scalar.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,18 +38,22 @@ int32_t scalarConvertOpValueNodeTs(SOperatorNode *node);
/*
pDst need to freed in caller
*/
int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst, const void* pExtraParam, void* streamTsRange);
int32_t scalarCalculateInRange(SNode *pNode, SArray *pBlockList, SScalarParam *pDst, int32_t rowStartIdx, int32_t rowEndIdx, const void* pExtraParam, void* streamTsRange);
void sclFreeParam(SScalarParam* param);
int32_t scalarAssignPlaceHolderRes(SColumnInfoData* pResColData, int64_t offset, int64_t rows, int16_t funcId, const void* pExtraParams);
int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst, const void *pExtraParam,
void *streamTsRange);
int32_t scalarCalculateInRange(SNode *pNode, SArray *pBlockList, SScalarParam *pDst, int32_t rowStartIdx,
int32_t rowEndIdx, const void *pExtraParam, void *streamTsRange);
void sclFreeParam(SScalarParam *param);
int32_t scalarAssignPlaceHolderRes(SColumnInfoData *pResColData, int64_t offset, int64_t rows, int16_t funcId,
const void *pExtraParams);
int32_t scalarGetOperatorParamNum(EOperatorType type);
int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type, STypeMod typeMod, int8_t processType);

int32_t vectorGetConvertType(int32_t type1, int32_t type2);
int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, int32_t *overflow, int32_t startIndex, int32_t numOfRows);
int32_t vectorConvertSingleCol(SScalarParam *input, SScalarParam *output, int32_t type, STypeMod typeMod, int32_t startIndex, int32_t numOfRows);
int32_t vectorGetConvertType(int32_t type1, int32_t type2);
int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, int32_t *overflow, int32_t startIndex,
int32_t numOfRows);
int32_t vectorConvertSingleCol(SScalarParam *input, SScalarParam *output, int32_t type, STypeMod typeMod,
int32_t startIndex, int32_t numOfRows);
STypeMod getConvertTypeMod(int32_t type, const SColumnInfo *pCol1, const SColumnInfo *pCol2);
uint32_t base64BufSize(size_t inputLenBytes);

/* Math functions */
int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
Expand Down Expand Up @@ -90,19 +94,20 @@ int32_t upperFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOut
int32_t ltrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t rtrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t md5Function(SScalarParam* pInput, int32_t inputNum, SScalarParam* pOutput);
int32_t md5Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t charFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t asciiFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t positionFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t trimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t replaceFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t repeatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t substrIdxFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t base64Function(SScalarParam* pInput, int32_t inputNum, SScalarParam* pOutput);
int32_t crc32Function(SScalarParam* pInput, int32_t inputNum, SScalarParam* pOutput);
int32_t findInSetFunction(SScalarParam* pInput, int32_t inputNum, SScalarParam* pOutput);
int32_t likeInSetFunction(SScalarParam* pInput, int32_t inputNum, SScalarParam* pOutput);
int32_t regexpInSetFunction(SScalarParam* pInput, int32_t inputNum, SScalarParam* pOutput);
int32_t base64Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t base64FunctionFrom(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t crc32Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t findInSetFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t likeInSetFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t regexpInSetFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);

/* Conversion functions */
int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
Expand Down Expand Up @@ -170,7 +175,8 @@ int32_t modeScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam
// stream pseudo functions
int32_t streamPseudoScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);

int32_t streamCalcCurrWinTimeRange(STimeRangeNode* node, void* pStRtFuncInfo, STimeWindow* pWinRange, bool* winRangeValid, int32_t type);
int32_t streamCalcCurrWinTimeRange(STimeRangeNode *node, void *pStRtFuncInfo, STimeWindow *pWinRange,
bool *winRangeValid, int32_t type);
int32_t scalarCalculateExtWinsTimeRange(STimeRangeNode *pNode, const void *pExtraParam, SExtWinTimeWindow *pWins);

#ifdef __cplusplus
Expand Down
8 changes: 7 additions & 1 deletion include/util/tbase64.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
#define _TD_UTIL_BASE64_H_

#include "os.h"
#include "types.h"

#ifdef __cplusplus
extern "C" {
Expand All @@ -25,8 +26,13 @@ extern "C" {
int32_t base64_decode(const char *value, int32_t inlen, int32_t *outlen, uint8_t **result);
int32_t base64_encode(const uint8_t *value, int32_t vlen, char **result);

void tbase64_encode(uint8_t *out, const uint8_t *input, size_t in_len, VarDataLenT out_len);
void tbase64_decode(uint8_t *out, const uint8_t *input, size_t in_len, VarDataLenT out_len);
uint32_t tbase64_encode_len(size_t in_len);
uint32_t tbase64_decode_len(size_t in_len);

#ifdef __cplusplus
}
#endif

#endif /*_TD_UTIL_BASE64_H_*/
#endif /*_TD_UTIL_BASE64_H_*/
34 changes: 33 additions & 1 deletion source/libs/function/src/builtins.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
#include "scalar.h"
#include "tanalytics.h"
#include "taoserror.h"
#include "tbase64.h"
#include "tglobal.h"
#include "ttypes.h"

Expand Down Expand Up @@ -1024,7 +1025,17 @@ static int32_t translateBase64(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
FUNC_ERR_RET(validateParam(pFunc, pErrBuf, len));

SDataType* pRestType1 = getSDataTypeFromNode(nodesListGetNode(pFunc->pParameterList, 0));
int32_t outputLength = base64BufSize(pRestType1->bytes);
int32_t outputLength = tbase64_encode_len(pRestType1->bytes);

pFunc->node.resType = (SDataType){.bytes = outputLength, .type = TSDB_DATA_TYPE_VARCHAR};
return TSDB_CODE_SUCCESS;
}

static int32_t translateBase64From(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
FUNC_ERR_RET(validateParam(pFunc, pErrBuf, len));

SDataType* pRestType1 = getSDataTypeFromNode(nodesListGetNode(pFunc->pParameterList, 0));
int32_t outputLength = tbase64_decode_len(pRestType1->bytes);

pFunc->node.resType = (SDataType){.bytes = outputLength, .type = TSDB_DATA_TYPE_VARCHAR};
return TSDB_CODE_SUCCESS;
Expand Down Expand Up @@ -6658,6 +6669,27 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.sprocessFunc = anomalyCheckMaskFunction,
.finalizeFunc = NULL
},
{
.name = "from_base64",
.type = FUNCTION_TYPE_BASE64,
.classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC,
.parameters = {.minParamNum = 1,
.maxParamNum = 1,
.paramInfoPattern = 1,
.inputParaInfo[0][0] = {.isLastParam = true,
.startParam = 1,
.endParam = 1,
.validDataType = FUNC_PARAM_SUPPORT_VARCHAR_TYPE | FUNC_PARAM_SUPPORT_NCHAR_TYPE | FUNC_PARAM_SUPPORT_NULL_TYPE,
.validNodeType = FUNC_PARAM_SUPPORT_EXPR_NODE,
.paramAttribute = FUNC_PARAM_NO_SPECIFIC_ATTRIBUTE,
.valueRangeFlag = FUNC_PARAM_NO_SPECIFIC_VALUE,},
.outputParaInfo = {.validDataType = FUNC_PARAM_SUPPORT_VARCHAR_TYPE}},
.translateFunc = translateBase64From,
.getEnvFunc = NULL,
.initFunc = NULL,
.sprocessFunc = base64FunctionFrom,
.finalizeFunc = NULL
},
};
// clang-format on

Expand Down
Loading
Loading