Codeunit 2003 ML Prediction Management
src/Layers/W1/BaseApp/System/AI/MLPredictionManagement.Codeunit.al536 lines, Copyright (c) Microsoft Corporation. MIT
namespace System.AI;using System;using System.IO;using System.Reflection;using System.Text;using System.Utilities;codeunit 2003 "ML Prediction Management"{ trigger OnRun() begin end; var AzureMLConnector: Codeunit "Azure ML Connector"; NotInitializedErr: Label 'The request has not been properly initialized.'; RecordVar: Variant; LabelNo: Integer; ConfidenceNo: Integer; FeatureNumbers: array[25] of Integer; LastFeatureIndex: Integer; UsingKeyvaultCredentials: Boolean; NotRecordVariantErr: Label 'The variant must be a record variant.'; FieldDoesNotExistErr: Label 'A field with the ID %1 does not exist.', Comment = '%1 = field ID'; TooManyFeaturesErr: Label 'Cannot train or predict because you have added more than %1 features to the model.', Comment = '%1 = max number of features'; LabelCannotBeFeatureErr: Label 'You have used the same field as the feature and as the label. A field can be either the label or feature, but not both.'; FeatureRepeatedErr: Label 'You can add a field as a feature only one time.'; TrainingPercent: Decimal; ApiUri: Text[250]; ApiKey: SecretText; ApiTimeout: Integer; TrainingPercentageErr: Label 'The training percentage must be a decimal number between 0 and 1.'; SomethingWentWrongErr: Label 'Oops, something went wrong when connecting to the Azure Machine Learning endpoint. Please contact your system administrator. %1.', Comment = '%1 = detailed error'; DetailedErrorErr: Label 'Details: '; ErrorResponseTxt: Label 'Error code: ', Locked = true; AzureMachineLearningLimitReachedErr: Label 'The Microsoft Azure Machine Learning limit has been reached. Please contact your system administrator.'; DownloadModelPlotLbl: Label 'Download model visualization in pdf format.'; MachineLearningSecretNameTxt: Label 'machinelearning', Locked = true; NoCredentialsInKeyVaultErr: Label 'There were no machine learning credentials in the key vault or the credentials were corrupted.'; procedure DefaultInitialize() begin LabelNo := 0; ConfidenceNo := 0; LastFeatureIndex := 0; TrainingPercent := 0.8; end; procedure Initialize(Uri: Text[250]; "Key": SecretText; TimeOutSeconds: Integer) begin ApiUri := Uri; ApiKey := Key; ApiTimeout := TimeOutSeconds; UsingKeyvaultCredentials := false; DefaultInitialize(); end; [NonDebuggable] procedure InitializeWithKeyVaultCredentials(TimeOutSeconds: Integer) var AzureAIUsage: Codeunit "Azure AI Usage"; AzureAIService: Enum "Azure AI Service"; LimitType: Option; LimitValue: Decimal; begin AzureAIService := AzureAIService::"Machine Learning"; if AzureAIUsage.GetTotalProcessingTime(AzureAIService) > AzureAIUsage.GetResourceLimit(AzureAIService) then Error(AzureMachineLearningLimitReachedErr); GetMachineLearningCredentials(ApiUri, ApiKey, LimitType, LimitValue); ApiTimeout := TimeOutSeconds; UsingKeyvaultCredentials := true; DefaultInitialize(); end; [Scope('OnPrem')] procedure SetMessageHandler(MessageHandler: DotNet HttpMessageHandler) begin AzureMLConnector.SetMessageHandler(MessageHandler); end; procedure SetRecord(RecordVariant: Variant) var DataTypeManagement: Codeunit "Data Type Management"; RecRef: RecordRef; begin if not DataTypeManagement.GetRecordRef(RecordVariant, RecRef) then Error(NotRecordVariantErr); RecordVar := RecordVariant; end; procedure GetRecord(var RecordVariant: Variant) begin RecordVariant := RecordVar; end; procedure AddFeature(FeatureFieldNo: Integer) var FeatureIndex: Integer; begin if LastFeatureIndex >= MaxNoFeatures() then Error(TooManyFeaturesErr, MaxNoFeatures()); if FeatureFieldNo = LabelNo then Error(LabelCannotBeFeatureErr); TestFieldNumber(FeatureFieldNo); for FeatureIndex := 1 to LastFeatureIndex do if FeatureNumbers[FeatureIndex] = FeatureFieldNo then Error(FeatureRepeatedErr); LastFeatureIndex := LastFeatureIndex + 1; FeatureNumbers[LastFeatureIndex] := FeatureFieldNo; end; procedure SetLabel(LabelFieldNo: Integer) var DataTypeManagement: Codeunit "Data Type Management"; RecRef: RecordRef; begin DataTypeManagement.GetRecordRef(RecordVar, RecRef); TestFieldNumber(LabelFieldNo); LabelNo := LabelFieldNo; end; procedure SetConfidence(ConfidenceFieldNo: Integer) var DataTypeManagement: Codeunit "Data Type Management"; RecRef: RecordRef; begin DataTypeManagement.GetRecordRef(RecordVar, RecRef); TestFieldNumber(ConfidenceFieldNo); ConfidenceNo := ConfidenceFieldNo; end; procedure SetTrainingPercent(TrainingPercentValue: Decimal) begin if (TrainingPercentValue <= 0) or (TrainingPercentValue >= 1) then Error(TrainingPercentageErr); TrainingPercent := TrainingPercentValue; end; procedure GetTrainingPercent(): Decimal begin exit(TrainingPercent); end; procedure Train(var Model: Text; var Quality: Decimal) var OutputValue: Text; CallAzureEndPoint: Boolean; begin CallAzureEndPoint := true; OnBeforeTrain(Model, Quality, CallAzureEndPoint); if not CallAzureEndPoint then exit; InitializeAzureMLConnector(); TestFeatureLabelInitialized(); AzureMLConnector.AddParameter('method', 'train'); AzureMLConnector.AddParameter('train_percent', Format(TrainingPercent, 0, 9)); CreateInput(); if not AzureMLConnector.SendToAzureMLInternal(UsingKeyvaultCredentials) then Error(SomethingWentWrongErr, GetLastDetailedError()); AzureMLConnector.GetOutput(1, 1, OutputValue); Model := OutputValue; AzureMLConnector.GetOutput(1, 2, OutputValue); SYSTEM.Evaluate(Quality, OutputValue, 9); end; procedure Predict(Model: Text) var CallAzureEndPoint: Boolean; begin CallAzureEndPoint := true; OnBeforePredict(RecordVar, CallAzureEndPoint); if not CallAzureEndPoint then exit; InitializeAzureMLConnector(); TestFeatureLabelInitialized(); AzureMLConnector.AddParameter('method', 'predict'); AzureMLConnector.AddParameter('model', Model); CreateInput(); if not AzureMLConnector.SendToAzureMLInternal(UsingKeyvaultCredentials) then Error(SomethingWentWrongErr, GetLastDetailedError()); LoadPrediction(); end; procedure Evaluate(Model: Text; var Quality: Decimal) var OutputValue: Text; CallAzureEndPoint: Boolean; begin CallAzureEndPoint := true; OnBeforeEvaluate(Model, Quality, RecordVar, CallAzureEndPoint); if not CallAzureEndPoint then exit; InitializeAzureMLConnector(); TestFeatureLabelInitialized(); AzureMLConnector.AddParameter('method', 'evaluate'); AzureMLConnector.AddParameter('model', Model); if CreateInput() then begin if not AzureMLConnector.SendToAzureMLInternal(UsingKeyvaultCredentials) then Error(SomethingWentWrongErr, GetLastDetailedError()); AzureMLConnector.GetOutput(1, 1, OutputValue); SYSTEM.Evaluate(Quality, OutputValue); end; end; procedure PlotModel(Model: Text; Features: Text; Labels: Text): Text var Result: Text; begin InitializeAzureMLConnector(); AzureMLConnector.AddParameter('method', 'plotmodel'); AzureMLConnector.AddParameter('model', Model); if Features <> '' then AzureMLConnector.AddParameter('captions', StrSubstNo('"%1"', Features)); if Labels <> '' then AzureMLConnector.AddParameter('labels', StrSubstNo('"%1"', Labels)); CreateDummyInput(); if not AzureMLConnector.SendToAzureMLInternal(UsingKeyvaultCredentials) then Error(SomethingWentWrongErr, GetLastDetailedError()); AzureMLConnector.GetOutput(1, 1, Result); exit(Result); end; procedure DownloadPlot(PdfDataBase64: Text; ModelName: Text) var Base64Convert: Codeunit "Base64 Convert"; TempBlob: Codeunit "Temp Blob"; OutStream: OutStream; InStr: InStream; begin TempBlob.CreateOutStream(OutStream); Base64Convert.FromBase64(PdfDataBase64, OutStream); TempBlob.CreateInStream(InStr); ModelName := StrSubstNo('%1.pdf', ModelName); DownloadFromStream(InStr, DownloadModelPlotLbl, '', '*.pdf', ModelName); end; procedure IsDataSufficientForClassification(): Boolean var TypeHelper: Codeunit "Type Helper"; RecRef: RecordRef; FieldRef: FieldRef; LabelDict: DotNet GenericDictionary2; TotalRecordCount: Integer; LabelDictKey: Text; LabelDictValue: Integer; Element: Integer; MinLabelCount: Integer; MinLabelCountInitialized: Boolean; begin if not IsDataSufficientBase(RecRef) then exit(false); CreateLabelDictionary(LabelDict); repeat FieldRef := RecRef.Field(LabelNo); LabelDictKey := Format(FieldRef.Value, 0, 9); if LabelDict.TryGetValue(LabelDictKey, LabelDictValue) then begin LabelDict.Remove(LabelDictKey); LabelDict.Add(LabelDictKey, LabelDictValue + 1); end else LabelDict.Add(LabelDictKey, 1); TotalRecordCount += 1; until RecRef.Next() = 0; MinLabelCountInitialized := false; foreach Element in LabelDict.Values do if not MinLabelCountInitialized then begin MinLabelCount := Element; MinLabelCountInitialized := true; end else if Element < MinLabelCount then MinLabelCount := Element; if MinLabelCount = TotalRecordCount then exit(false); // there is only one label in the dataset // The certainity needed for at least one of the labels to be in the training set should be 99% exit(TotalRecordCount >= (TypeHelper.CalculateLog(1 - 0.99) / TypeHelper.CalculateLog(1 - (MinLabelCount / TotalRecordCount)) / TrainingPercent)); end; procedure IsDataSufficientForRegression(): Boolean var RecRef: RecordRef; begin exit(IsDataSufficientBase(RecRef)); end; procedure IsDataSufficientBase(var RecRef: RecordRef): Boolean var DataTypeManagement: Codeunit "Data Type Management"; begin TestFeatureLabelInitialized(); DataTypeManagement.GetRecordRef(RecordVar, RecRef); if not RecRef.FindSet() then exit(false); // 20 is the minimum number of data points for which the decision // tree algorithm we are using will create a model with multiple nodes exit(TrainingPercent * RecRef.Count >= 20); end; [NonDebuggable] local procedure InitializeAzureMLConnector() begin if not AzureMLConnector.Initialize(ApiKey, ApiUri, ApiTimeout) then Error(NotInitializedErr, GetLastDetailedError()); end; local procedure CreateLabelDictionary(var LabelDict: DotNet GenericDictionary2) var Type: DotNet Type; Activator: DotNet Activator; Arr: DotNet Array; DummyString: DotNet String; DummyInt: Integer; begin // A new object of type dictionary<String, int> is created. This object holds the Label as the key and the // value as the number of records in the dataset with that label. Arr := Arr.CreateInstance(GetDotNetType(Type), 2); Arr.SetValue(GetDotNetType(DummyString), 0); Arr.SetValue(GetDotNetType(DummyInt), 1); Type := GetDotNetType(LabelDict); Type := Type.MakeGenericType(Arr); LabelDict := Activator.CreateInstance(Type); end; local procedure TestFieldNumber(FieldNumber: Integer) var DataTypeManagement: Codeunit "Data Type Management"; RecRef: RecordRef; begin DataTypeManagement.GetRecordRef(RecordVar, RecRef); if not RecRef.FieldExist(FieldNumber) then Error(FieldDoesNotExistErr, FieldNumber); end; local procedure TestFeatureLabelInitialized() var Initialized: Boolean; begin Initialized := true; if Format(RecordVar) = '' then Initialized := false; if LabelNo = 0 then Initialized := false; if LastFeatureIndex = 0 then Initialized := false; if not Initialized then Error(NotInitializedErr); end; local procedure CreateInput(): Boolean var DataTypeManagement: Codeunit "Data Type Management"; RecRef: RecordRef; FieldRef: FieldRef; ColumnNo: Integer; begin for ColumnNo := 1 to MaxNoFeatures() do AzureMLConnector.AddInputColumnName(StrSubstNo('feature%1', ColumnNo)); AzureMLConnector.AddInputColumnName('label'); DataTypeManagement.GetRecordRef(RecordVar, RecRef); if not RecRef.FindSet() then exit(false); repeat AzureMLConnector.AddInputRow(); for ColumnNo := 1 to MaxNoFeatures() do if ColumnNo <= LastFeatureIndex then begin FieldRef := RecRef.Field(FeatureNumbers[ColumnNo]); if FieldRef.Class = FieldClass::FlowField then FieldRef.CalcField(); AzureMLConnector.AddInputValue(Format(FieldRef.Value, 0, 9)); AzureMLConnector.AddParameter(StrSubstNo('featuretype%1', ColumnNo), Format(FieldRef.Type)); end else AzureMLConnector.AddInputValue(''); FieldRef := RecRef.Field(LabelNo); if FieldRef.Class = FieldClass::FlowField then FieldRef.CalcField(); AzureMLConnector.AddInputValue(Format(FieldRef.Value, 0, 9)); AzureMLConnector.AddParameter('labeltype', Format(FieldRef.Type)); until RecRef.Next() = 0; exit(true); end; local procedure CreateDummyInput() var ColumnNo: Integer; begin for ColumnNo := 1 to MaxNoFeatures() do AzureMLConnector.AddInputColumnName(StrSubstNo('feature%1', ColumnNo)); AzureMLConnector.AddInputColumnName('label'); AzureMLConnector.AddInputRow(); for ColumnNo := 1 to MaxNoFeatures() + 1 do AzureMLConnector.AddInputValue('0'); end; procedure GetParameter(Name: Text): Text var ParameterValue: Text; begin AzureMLConnector.GetParameter(Name, ParameterValue); exit(ParameterValue); end; procedure GetInput(RowNo: Integer; ColumnNo: Integer): Text var InputValue: Text; begin AzureMLConnector.GetInput(RowNo, ColumnNo, InputValue); exit(InputValue); end; procedure GetInputLength(): Integer var Length: Integer; begin AzureMLConnector.GetInputLength(Length); exit(Length); end; local procedure GetLastDetailedError(): Text var DetailedErrorTxt: Text; DetailedErrorStart: Integer; begin DetailedErrorStart := StrPos(GetLastErrorText, ErrorResponseTxt); if DetailedErrorStart > 0 then begin DetailedErrorTxt := CopyStr(GetLastErrorText, StrPos(GetLastErrorText, ErrorResponseTxt)); DetailedErrorTxt := CopyStr(DetailedErrorTxt, 1, StrPos(DetailedErrorTxt, '\') - 1); exit('\' + DetailedErrorErr + DetailedErrorTxt); end; exit('\' + DetailedErrorErr + GetLastErrorText); end; local procedure LoadPrediction() var DataTypeManagement: Codeunit "Data Type Management"; ConfigValidateManagement: Codeunit "Config. Validate Management"; RecRef: RecordRef; FieldRef: FieldRef; LabelColumnNumber: Integer; ConfidenceColumnNumber: Integer; RowNumber: Integer; Label: Text[250]; Confidence: Text[250]; OutputValue: Text; begin RowNumber := 1; LabelColumnNumber := MaxNoFeatures() + 1; ConfidenceColumnNumber := LabelColumnNumber + 1; DataTypeManagement.GetRecordRef(RecordVar, RecRef); if RecRef.FindSet() then repeat if ConfidenceNo <> 0 then begin AzureMLConnector.GetOutput(RowNumber, ConfidenceColumnNumber, OutputValue); Confidence := CopyStr(OutputValue, 1, MaxStrLen(Confidence)); FieldRef := RecRef.Field(ConfidenceNo); ConfigValidateManagement.EvaluateValueWithValidate(FieldRef, Confidence, true); end; AzureMLConnector.GetOutput(RowNumber, LabelColumnNumber, OutputValue); Label := CopyStr(OutputValue, 1, MaxStrLen(Label)); FieldRef := RecRef.Field(LabelNo); ConfigValidateManagement.EvaluateValueWithValidate(FieldRef, Label, true); RecRef.Modify(true); RowNumber := RowNumber + 1; until RecRef.Next() = 0; if RecRef.FindSet() then RecRef.SetTable(RecordVar); end; procedure MaxNoFeatures(): Integer begin exit(ArrayLen(FeatureNumbers)); end; [TryFunction] [Scope('OnPrem')] procedure GetMachineLearningCredentials(var MLApiUri: Text[250]; var MLApiKey: SecretText; var LimitType: Option; var Limit: Decimal) var MachineLearningKeyVaultMgmt: Codeunit "Machine Learning KeyVaultMgmt."; begin MachineLearningKeyVaultMgmt.GetMachineLearningCredentials(MachineLearningSecretNameTxt, MLApiUri, MLApiKey, LimitType, Limit); if MLApiUri = '' then Error(NoCredentialsInKeyVaultErr); end; [IntegrationEvent(false, false)] local procedure OnBeforeTrain(var Model: Text; var Quality: Decimal; var CallAzureEndPoint: Boolean) begin end; [IntegrationEvent(false, false)] local procedure OnBeforePredict(var RecordVariant: Variant; var CallAzureEndPoint: Boolean) begin end; [IntegrationEvent(false, false)] local procedure OnBeforeEvaluate(Model: Text; var Quality: Decimal; var RecordVariant: Variant; var CallAzureEndPoint: Boolean) begin end;}