Codeunit 2003 ML Prediction Management

namespace System.AI;

using System;
using System.IO;
using System.Reflection;
using System.Text;
using System.Utilities;

codeunit 2003 "ML Prediction Management"
{

    trigger OnRun()
    begin
    end;

    var
        AzureMLConnector: Codeunit "Azure ML Connector";
        NotInitializedErr: Label 'The request has not been properly initialized.';
        RecordVar: Variant;
        LabelNo: Integer;
        ConfidenceNo: Integer;
        FeatureNumbers: array[25] of Integer;
        LastFeatureIndex: Integer;
        UsingKeyvaultCredentials: Boolean;
        NotRecordVariantErr: Label 'The variant must be a record variant.';
        FieldDoesNotExistErr: Label 'A field with the ID %1 does not exist.', Comment = '%1 = field ID';
        TooManyFeaturesErr: Label 'Cannot train or predict because you have added more than %1 features to the model.', Comment = '%1 = max number of features';
        LabelCannotBeFeatureErr: Label 'You have used the same field as the feature and as the label. A field can be either the label or feature, but not both.';
        FeatureRepeatedErr: Label 'You can add a field as a feature only one time.';
        TrainingPercent: Decimal;
        ApiUri: Text[250];
        ApiKey: SecretText;
        ApiTimeout: Integer;
        TrainingPercentageErr: Label 'The training percentage must be a decimal number between 0 and 1.';
        SomethingWentWrongErr: Label 'Oops, something went wrong when connecting to the Azure Machine Learning endpoint. Please contact your system administrator. %1.', Comment = '%1 = detailed error';
        DetailedErrorErr: Label 'Details: ';
        ErrorResponseTxt: Label 'Error code: ', Locked = true;
        AzureMachineLearningLimitReachedErr: Label 'The Microsoft Azure Machine Learning limit has been reached. Please contact your system administrator.';
        DownloadModelPlotLbl: Label 'Download model visualization in pdf format.';
        MachineLearningSecretNameTxt: Label 'machinelearning', Locked = true;
        NoCredentialsInKeyVaultErr: Label 'There were no machine learning credentials in the key vault or the credentials were corrupted.';

    procedure DefaultInitialize()
    begin
        LabelNo := 0;
        ConfidenceNo := 0;
        LastFeatureIndex := 0;
        TrainingPercent := 0.8;
    end;

    procedure Initialize(Uri: Text[250]; "Key": SecretText; TimeOutSeconds: Integer)
    begin
        ApiUri := Uri;
        ApiKey := Key;
        ApiTimeout := TimeOutSeconds;
        UsingKeyvaultCredentials := false;
        DefaultInitialize();
    end;

    [NonDebuggable]
    procedure InitializeWithKeyVaultCredentials(TimeOutSeconds: Integer)
    var
        AzureAIUsage: Codeunit "Azure AI Usage";
        AzureAIService: Enum "Azure AI Service";
        LimitType: Option;
        LimitValue: Decimal;
    begin
        AzureAIService := AzureAIService::"Machine Learning";

        if AzureAIUsage.GetTotalProcessingTime(AzureAIService) > AzureAIUsage.GetResourceLimit(AzureAIService) then
            Error(AzureMachineLearningLimitReachedErr);

        GetMachineLearningCredentials(ApiUri, ApiKey, LimitType, LimitValue);
        ApiTimeout := TimeOutSeconds;
        UsingKeyvaultCredentials := true;
        DefaultInitialize();
    end;

    [Scope('OnPrem')]
    procedure SetMessageHandler(MessageHandler: DotNet HttpMessageHandler)
    begin
        AzureMLConnector.SetMessageHandler(MessageHandler);
    end;

    procedure SetRecord(RecordVariant: Variant)
    var
        DataTypeManagement: Codeunit "Data Type Management";
        RecRef: RecordRef;
    begin
        if not DataTypeManagement.GetRecordRef(RecordVariant, RecRef) then
            Error(NotRecordVariantErr);
        RecordVar := RecordVariant;
    end;

    procedure GetRecord(var RecordVariant: Variant)
    begin
        RecordVariant := RecordVar;
    end;

    procedure AddFeature(FeatureFieldNo: Integer)
    var
        FeatureIndex: Integer;
    begin
        if LastFeatureIndex >= MaxNoFeatures() then
            Error(TooManyFeaturesErr, MaxNoFeatures());

        if FeatureFieldNo = LabelNo then
            Error(LabelCannotBeFeatureErr);

        TestFieldNumber(FeatureFieldNo);

        for FeatureIndex := 1 to LastFeatureIndex do
            if FeatureNumbers[FeatureIndex] = FeatureFieldNo then
                Error(FeatureRepeatedErr);

        LastFeatureIndex := LastFeatureIndex + 1;
        FeatureNumbers[LastFeatureIndex] := FeatureFieldNo;
    end;

    procedure SetLabel(LabelFieldNo: Integer)
    var
        DataTypeManagement: Codeunit "Data Type Management";
        RecRef: RecordRef;
    begin
        DataTypeManagement.GetRecordRef(RecordVar, RecRef);
        TestFieldNumber(LabelFieldNo);
        LabelNo := LabelFieldNo;
    end;

    procedure SetConfidence(ConfidenceFieldNo: Integer)
    var
        DataTypeManagement: Codeunit "Data Type Management";
        RecRef: RecordRef;
    begin
        DataTypeManagement.GetRecordRef(RecordVar, RecRef);
        TestFieldNumber(ConfidenceFieldNo);
        ConfidenceNo := ConfidenceFieldNo;
    end;

    procedure SetTrainingPercent(TrainingPercentValue: Decimal)
    begin
        if (TrainingPercentValue <= 0) or (TrainingPercentValue >= 1) then
            Error(TrainingPercentageErr);

        TrainingPercent := TrainingPercentValue;
    end;

    procedure GetTrainingPercent(): Decimal
    begin
        exit(TrainingPercent);
    end;

    procedure Train(var Model: Text; var Quality: Decimal)
    var
        OutputValue: Text;
        CallAzureEndPoint: Boolean;
    begin
        CallAzureEndPoint := true;
        OnBeforeTrain(Model, Quality, CallAzureEndPoint);
        if not CallAzureEndPoint then
            exit;

        InitializeAzureMLConnector();

        TestFeatureLabelInitialized();
        AzureMLConnector.AddParameter('method', 'train');
        AzureMLConnector.AddParameter('train_percent', Format(TrainingPercent, 0, 9));
        CreateInput();
        if not AzureMLConnector.SendToAzureMLInternal(UsingKeyvaultCredentials) then
            Error(SomethingWentWrongErr, GetLastDetailedError());

        AzureMLConnector.GetOutput(1, 1, OutputValue);
        Model := OutputValue;
        AzureMLConnector.GetOutput(1, 2, OutputValue);
        SYSTEM.Evaluate(Quality, OutputValue, 9);
    end;

    procedure Predict(Model: Text)
    var
        CallAzureEndPoint: Boolean;
    begin
        CallAzureEndPoint := true;
        OnBeforePredict(RecordVar, CallAzureEndPoint);
        if not CallAzureEndPoint then
            exit;

        InitializeAzureMLConnector();

        TestFeatureLabelInitialized();
        AzureMLConnector.AddParameter('method', 'predict');
        AzureMLConnector.AddParameter('model', Model);
        CreateInput();
        if not AzureMLConnector.SendToAzureMLInternal(UsingKeyvaultCredentials) then
            Error(SomethingWentWrongErr, GetLastDetailedError());

        LoadPrediction();
    end;

    procedure Evaluate(Model: Text; var Quality: Decimal)
    var
        OutputValue: Text;
        CallAzureEndPoint: Boolean;
    begin
        CallAzureEndPoint := true;
        OnBeforeEvaluate(Model, Quality, RecordVar, CallAzureEndPoint);
        if not CallAzureEndPoint then
            exit;

        InitializeAzureMLConnector();

        TestFeatureLabelInitialized();
        AzureMLConnector.AddParameter('method', 'evaluate');
        AzureMLConnector.AddParameter('model', Model);
        if CreateInput() then begin
            if not AzureMLConnector.SendToAzureMLInternal(UsingKeyvaultCredentials) then
                Error(SomethingWentWrongErr, GetLastDetailedError());

            AzureMLConnector.GetOutput(1, 1, OutputValue);
            SYSTEM.Evaluate(Quality, OutputValue);
        end;
    end;

    procedure PlotModel(Model: Text; Features: Text; Labels: Text): Text
    var
        Result: Text;
    begin
        InitializeAzureMLConnector();

        AzureMLConnector.AddParameter('method', 'plotmodel');
        AzureMLConnector.AddParameter('model', Model);
        if Features <> '' then
            AzureMLConnector.AddParameter('captions', StrSubstNo('"%1"', Features));
        if Labels <> '' then
            AzureMLConnector.AddParameter('labels', StrSubstNo('"%1"', Labels));

        CreateDummyInput();

        if not AzureMLConnector.SendToAzureMLInternal(UsingKeyvaultCredentials) then
            Error(SomethingWentWrongErr, GetLastDetailedError());

        AzureMLConnector.GetOutput(1, 1, Result);
        exit(Result);
    end;

    procedure DownloadPlot(PdfDataBase64: Text; ModelName: Text)
    var
        Base64Convert: Codeunit "Base64 Convert";
        TempBlob: Codeunit "Temp Blob";
        OutStream: OutStream;
        InStr: InStream;
    begin
        TempBlob.CreateOutStream(OutStream);
        Base64Convert.FromBase64(PdfDataBase64, OutStream);
        TempBlob.CreateInStream(InStr);
        ModelName := StrSubstNo('%1.pdf', ModelName);
        DownloadFromStream(InStr, DownloadModelPlotLbl, '', '*.pdf', ModelName);
    end;

    procedure IsDataSufficientForClassification(): Boolean
    var
        TypeHelper: Codeunit "Type Helper";
        RecRef: RecordRef;
        FieldRef: FieldRef;
        LabelDict: DotNet GenericDictionary2;
        TotalRecordCount: Integer;
        LabelDictKey: Text;
        LabelDictValue: Integer;
        Element: Integer;
        MinLabelCount: Integer;
        MinLabelCountInitialized: Boolean;
    begin
        if not IsDataSufficientBase(RecRef) then
            exit(false);

        CreateLabelDictionary(LabelDict);
        repeat
            FieldRef := RecRef.Field(LabelNo);
            LabelDictKey := Format(FieldRef.Value, 0, 9);
            if LabelDict.TryGetValue(LabelDictKey, LabelDictValue) then begin
                LabelDict.Remove(LabelDictKey);
                LabelDict.Add(LabelDictKey, LabelDictValue + 1);
            end else
                LabelDict.Add(LabelDictKey, 1);
            TotalRecordCount += 1;
        until RecRef.Next() = 0;

        MinLabelCountInitialized := false;
        foreach Element in LabelDict.Values do
            if not MinLabelCountInitialized then begin
                MinLabelCount := Element;
                MinLabelCountInitialized := true;
            end else
                if Element < MinLabelCount then
                    MinLabelCount := Element;

        if MinLabelCount = TotalRecordCount then
            exit(false); // there is only one label in the dataset

        // The certainity needed for at least one of the labels to be in the training set should be 99%
        exit(TotalRecordCount >= (TypeHelper.CalculateLog(1 - 0.99) /
                                  TypeHelper.CalculateLog(1 - (MinLabelCount / TotalRecordCount)) /
                                  TrainingPercent));
    end;

    procedure IsDataSufficientForRegression(): Boolean
    var
        RecRef: RecordRef;
    begin
        exit(IsDataSufficientBase(RecRef));
    end;

    procedure IsDataSufficientBase(var RecRef: RecordRef): Boolean
    var
        DataTypeManagement: Codeunit "Data Type Management";
    begin
        TestFeatureLabelInitialized();
        DataTypeManagement.GetRecordRef(RecordVar, RecRef);

        if not RecRef.FindSet() then
            exit(false);

        // 20 is the minimum number of data points for which the decision
        // tree algorithm we are using will create a model with multiple nodes
        exit(TrainingPercent * RecRef.Count >= 20);
    end;

    [NonDebuggable]
    local procedure InitializeAzureMLConnector()
    begin
        if not AzureMLConnector.Initialize(ApiKey, ApiUri, ApiTimeout) then
            Error(NotInitializedErr, GetLastDetailedError());
    end;

    local procedure CreateLabelDictionary(var LabelDict: DotNet GenericDictionary2)
    var
        Type: DotNet Type;
        Activator: DotNet Activator;
        Arr: DotNet Array;
        DummyString: DotNet String;
        DummyInt: Integer;
    begin
        // A new object of type dictionary<String, int> is created. This object holds the Label as the key and the
        // value as the number of records in the dataset with that label.
        Arr := Arr.CreateInstance(GetDotNetType(Type), 2);
        Arr.SetValue(GetDotNetType(DummyString), 0);
        Arr.SetValue(GetDotNetType(DummyInt), 1);

        Type := GetDotNetType(LabelDict);
        Type := Type.MakeGenericType(Arr);

        LabelDict := Activator.CreateInstance(Type);
    end;

    local procedure TestFieldNumber(FieldNumber: Integer)
    var
        DataTypeManagement: Codeunit "Data Type Management";
        RecRef: RecordRef;
    begin
        DataTypeManagement.GetRecordRef(RecordVar, RecRef);
        if not RecRef.FieldExist(FieldNumber) then
            Error(FieldDoesNotExistErr, FieldNumber);
    end;

    local procedure TestFeatureLabelInitialized()
    var
        Initialized: Boolean;
    begin
        Initialized := true;
        if Format(RecordVar) = '' then
            Initialized := false;
        if LabelNo = 0 then
            Initialized := false;
        if LastFeatureIndex = 0 then
            Initialized := false;
        if not Initialized then
            Error(NotInitializedErr);
    end;

    local procedure CreateInput(): Boolean
    var
        DataTypeManagement: Codeunit "Data Type Management";
        RecRef: RecordRef;
        FieldRef: FieldRef;
        ColumnNo: Integer;
    begin
        for ColumnNo := 1 to MaxNoFeatures() do
            AzureMLConnector.AddInputColumnName(StrSubstNo('feature%1', ColumnNo));
        AzureMLConnector.AddInputColumnName('label');

        DataTypeManagement.GetRecordRef(RecordVar, RecRef);
        if not RecRef.FindSet() then
            exit(false);

        repeat
            AzureMLConnector.AddInputRow();
            for ColumnNo := 1 to MaxNoFeatures() do
                if ColumnNo <= LastFeatureIndex then begin
                    FieldRef := RecRef.Field(FeatureNumbers[ColumnNo]);
                    if FieldRef.Class = FieldClass::FlowField then
                        FieldRef.CalcField();
                    AzureMLConnector.AddInputValue(Format(FieldRef.Value, 0, 9));
                    AzureMLConnector.AddParameter(StrSubstNo('featuretype%1', ColumnNo), Format(FieldRef.Type));
                end else
                    AzureMLConnector.AddInputValue('');

            FieldRef := RecRef.Field(LabelNo);
            if FieldRef.Class = FieldClass::FlowField then
                FieldRef.CalcField();
            AzureMLConnector.AddInputValue(Format(FieldRef.Value, 0, 9));
            AzureMLConnector.AddParameter('labeltype', Format(FieldRef.Type));
        until RecRef.Next() = 0;

        exit(true);
    end;

    local procedure CreateDummyInput()
    var
        ColumnNo: Integer;
    begin
        for ColumnNo := 1 to MaxNoFeatures() do
            AzureMLConnector.AddInputColumnName(StrSubstNo('feature%1', ColumnNo));
        AzureMLConnector.AddInputColumnName('label');

        AzureMLConnector.AddInputRow();
        for ColumnNo := 1 to MaxNoFeatures() + 1 do
            AzureMLConnector.AddInputValue('0');
    end;

    procedure GetParameter(Name: Text): Text
    var
        ParameterValue: Text;
    begin
        AzureMLConnector.GetParameter(Name, ParameterValue);
        exit(ParameterValue);
    end;

    procedure GetInput(RowNo: Integer; ColumnNo: Integer): Text
    var
        InputValue: Text;
    begin
        AzureMLConnector.GetInput(RowNo, ColumnNo, InputValue);
        exit(InputValue);
    end;

    procedure GetInputLength(): Integer
    var
        Length: Integer;
    begin
        AzureMLConnector.GetInputLength(Length);
        exit(Length);
    end;

    local procedure GetLastDetailedError(): Text
    var
        DetailedErrorTxt: Text;
        DetailedErrorStart: Integer;
    begin
        DetailedErrorStart := StrPos(GetLastErrorText, ErrorResponseTxt);
        if DetailedErrorStart > 0 then begin
            DetailedErrorTxt := CopyStr(GetLastErrorText, StrPos(GetLastErrorText, ErrorResponseTxt));
            DetailedErrorTxt := CopyStr(DetailedErrorTxt, 1, StrPos(DetailedErrorTxt, '\') - 1);
            exit('\' + DetailedErrorErr + DetailedErrorTxt);
        end;
        exit('\' + DetailedErrorErr + GetLastErrorText);
    end;

    local procedure LoadPrediction()
    var
        DataTypeManagement: Codeunit "Data Type Management";
        ConfigValidateManagement: Codeunit "Config. Validate Management";
        RecRef: RecordRef;
        FieldRef: FieldRef;
        LabelColumnNumber: Integer;
        ConfidenceColumnNumber: Integer;
        RowNumber: Integer;
        Label: Text[250];
        Confidence: Text[250];
        OutputValue: Text;
    begin
        RowNumber := 1;
        LabelColumnNumber := MaxNoFeatures() + 1;
        ConfidenceColumnNumber := LabelColumnNumber + 1;
        DataTypeManagement.GetRecordRef(RecordVar, RecRef);
        if RecRef.FindSet() then
            repeat
                if ConfidenceNo <> 0 then begin
                    AzureMLConnector.GetOutput(RowNumber, ConfidenceColumnNumber, OutputValue);
                    Confidence := CopyStr(OutputValue, 1, MaxStrLen(Confidence));
                    FieldRef := RecRef.Field(ConfidenceNo);
                    ConfigValidateManagement.EvaluateValueWithValidate(FieldRef, Confidence, true);
                end;

                AzureMLConnector.GetOutput(RowNumber, LabelColumnNumber, OutputValue);
                Label := CopyStr(OutputValue, 1, MaxStrLen(Label));
                FieldRef := RecRef.Field(LabelNo);
                ConfigValidateManagement.EvaluateValueWithValidate(FieldRef, Label, true);
                RecRef.Modify(true);
                RowNumber := RowNumber + 1;
            until RecRef.Next() = 0;
        if RecRef.FindSet() then
            RecRef.SetTable(RecordVar);
    end;

    procedure MaxNoFeatures(): Integer
    begin
        exit(ArrayLen(FeatureNumbers));
    end;

    [TryFunction]
    [Scope('OnPrem')]
    procedure GetMachineLearningCredentials(var MLApiUri: Text[250]; var MLApiKey: SecretText; var LimitType: Option; var Limit: Decimal)
    var
        MachineLearningKeyVaultMgmt: Codeunit "Machine Learning KeyVaultMgmt.";
    begin
        MachineLearningKeyVaultMgmt.GetMachineLearningCredentials(MachineLearningSecretNameTxt, MLApiUri, MLApiKey, LimitType, Limit);

        if MLApiUri = '' then
            Error(NoCredentialsInKeyVaultErr);
    end;

    [IntegrationEvent(false, false)]
    local procedure OnBeforeTrain(var Model: Text; var Quality: Decimal; var CallAzureEndPoint: Boolean)
    begin
    end;

    [IntegrationEvent(false, false)]
    local procedure OnBeforePredict(var RecordVariant: Variant; var CallAzureEndPoint: Boolean)
    begin
    end;

    [IntegrationEvent(false, false)]
    local procedure OnBeforeEvaluate(Model: Text; var Quality: Decimal; var RecordVariant: Variant; var CallAzureEndPoint: Boolean)
    begin
    end;
}