Codeunit 2012 Entity Text Impl.
src/System Application/App/Entity Text/src/EntityTextImpl.Codeunit.al430 lines, Copyright (c) Microsoft Corporation. MIT
// ------------------------------------------------------------------------------------------------// Copyright (c) Microsoft Corporation. All rights reserved.// Licensed under the MIT License. See License.txt in the project root for license information.// ------------------------------------------------------------------------------------------------namespace System.Text;using System;using System.AI;using System.Telemetry;using System.Utilities;/// <summary>/// Implements functionality to handle text suggestions./// </summary>codeunit 2012 "Entity Text Impl."{ Access = Internal; InherentEntitlements = X; InherentPermissions = X; /// <summary> /// Decides visibility on pages /// </summary> procedure IsEnabled(Silent: Boolean): Boolean var AzureOpenAI: Codeunit "Azure OpenAI"; begin exit(AzureOpenAI.IsEnabled(Enum::"Copilot Capability"::"Entity Text", Silent)); end; internal procedure GetFeatureName(): Text begin exit('Entity Text'); end; procedure CanSuggest(): Boolean var EntityTextPrompts: Codeunit "Entity Text Prompts"; EntityTextAOAISettings: Codeunit "Entity Text AOAI Settings"; begin if not EntityTextAOAISettings.IsEnabled(true) then exit(false); exit(EntityTextPrompts.HasPromptInfo()); end; [NonDebuggable] procedure GenerateSuggestion(Facts: Dictionary of [Text, Text]; Tone: Enum "Entity Text Tone"; TextFormat: Enum "Entity Text Format"; TextEmphasis: Enum "Entity Text Emphasis"; CallerModuleInfo: ModuleInfo): Text var EntityTextPrompts: Codeunit "Entity Text Prompts"; SystemPrompt: Text; UserPrompt: Text; Suggestion: Text; FactsList: Text; Category: Text; begin if not IsEnabled(true) then Error(CapabilityDisabledErr); if not CanSuggest() then Error(CannotGenerateErr); FactsList := BuildFacts(Facts, Category, TextFormat); EntityTextPrompts.BuildPrompts(FactsList, Category, Tone, TextFormat, TextEmphasis, SystemPrompt, UserPrompt); Session.LogMessage('0000JVG', TelemetryGenerationRequestedTxt, Verbosity::Normal, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', GetFeatureName()); Suggestion := GenerateAndReviewCompletion(SystemPrompt, UserPrompt, TextFormat, Facts, CallerModuleInfo); exit(Suggestion); end; procedure InsertSuggestion(SourceTableId: Integer; SourceSystemId: Guid; SourceScenario: Enum "Entity Text Scenario"; SuggestedText: Text) var EntityText: Record "Entity Text"; begin InsertSuggestion(SourceTableId, SourceSystemId, SourceScenario, SuggestedText, EntityText); end; procedure InsertSuggestion(SourceTableId: Integer; SourceSystemId: Guid; SourceScenario: Enum "Entity Text Scenario"; SuggestedText: Text; var EntityText: Record "Entity Text") begin EntityText.Init(); EntityText.Company := CopyStr(CompanyName(), 1, MaxStrLen(EntityText.Company)); EntityText."Source Table Id" := SourceTableId; EntityText."Source System Id" := SourceSystemId; EntityText.Scenario := SourceScenario; SetText(EntityText, SuggestedText); if not EntityText.Insert() then EntityText.Modify(); Session.LogMessage('0000JVH', StrSubstNo(TelemetrySuggestionCreatedTxt, Format(SourceTableId), Format(SourceScenario)), Verbosity::Normal, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', GetFeatureName()); end; procedure SetText(var EntityText: Record "Entity Text"; Content: Text) var Regex: Codeunit Regex; HttpUtility: DotNet HttpUtility; OutStr: OutStream; ContentNoTags: Text; ContentLines: List of [Text]; ContentLine: Text; begin Clear(EntityText.Text); EntityText.Text.CreateOutStream(OutStr, TextEncoding::UTF8); OutStr.WriteText(Content); // naively remove tags ContentLines := Regex.Replace(Content, '<br */?>', '\').Split('\'); foreach ContentLine in ContentLines do begin ContentLine := Regex.Replace(ContentLine, '<[^>]+>', ''); ContentLine := HttpUtility.HtmlDecode(ContentLine); if ContentLine <> '' then ContentNoTags += ContentLine + '\'; end; ContentNoTags := DelChr(ContentNoTags, '>', ' \'); EntityText."Preview Text" := CopyStr(ContentNoTags, 1, MaxStrLen(EntityText."Preview Text")); end; procedure GetText(var EntityText: Record "Entity Text"): Text var InStr: InStream; Result: Text; begin EntityText.CalcFields(Text); EntityText.Text.CreateInStream(InStr, TextEncoding::UTF8); InStr.ReadText(Result); exit(Result); end; procedure GetText(TableId: Integer; SystemId: Guid; EntityTextScenario: Enum "Entity Text Scenario"): Text var EntityText: Record "Entity Text"; Result: Text; begin if EntityText.Get(CompanyName(), TableId, SystemId, EntityTextScenario) then Result := GetText(EntityText); exit(Result); end; procedure SetEntityTextAuthorization(NewEndpoint: Text; NewDeployment: Text; NewApiKey: SecretText) begin Endpoint := NewEndpoint; Deployment := NewDeployment; ApiKey := NewApiKey; end; [NonDebuggable] local procedure BuildFacts(var Facts: Dictionary of [Text, Text]; var Category: Text; TextFormat: Enum "Entity Text Format"): Text var FactKey: Text; FactValue: Text; FactsList: Text; NewLineChar: Char; MaxFacts: Integer; TotalFacts: Integer; MaxFactLength: Integer; begin NewLineChar := 10; TotalFacts := Facts.Count(); if TotalFacts = 0 then Error(NoFactsErr); if TotalFacts < 2 then Error(MinFactsErr); if (TotalFacts < 4) and (TextFormat <> TextFormat::Tagline) then Error(NotEnoughFactsForFormatErr); MaxFacts := 20; MaxFactLength := 250; if TotalFacts > MaxFacts then Session.LogMessage('0000JWA', StrSubstNo(TelemetryPromptManyFactsTxt, Format(Facts.Count()), MaxFacts), Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', GetFeatureName()); TotalFacts := 0; foreach FactKey in Facts.Keys() do begin if TotalFacts < MaxFacts then begin Facts.Get(FactKey, FactValue); FactKey := FactKey.Replace(NewLineChar, '').Trim(); FactValue := FactValue.Replace(NewLineChar, '').Trim(); if (Category = '') and FactKey.Contains('Category') then Category := FactValue else FactsList += StrSubstNo(FactTemplateTxt, CopyStr(FactKey, 1, MaxFactLength), CopyStr(FactValue, 1, MaxFactLength), NewLineChar); end; TotalFacts += 1; end; exit(FactsList); end; [NonDebuggable] local procedure GenerateAndReviewCompletion(SystemPrompt: Text; UserPrompt: Text; TextFormat: Enum "Entity Text Format"; Facts: Dictionary of [Text, Text]; CallerModuleInfo: ModuleInfo): Text var MagicFunction: Codeunit "Magic Function"; EmptyArguments: JsonObject; Completion: Text; MaxAttempts: Integer; Attempt: Integer; begin MaxAttempts := 5; for Attempt := 0 to MaxAttempts do begin Completion := GenerateCompletion(TextFormat, SystemPrompt, UserPrompt, CallerModuleInfo); if (not CompletionContainsPrompt(Completion, SystemPrompt)) and IsGoodCompletion(Completion, TextFormat, Facts) then exit(Completion); Sleep(500); Session.LogMessage('0000LVP', StrSubstNo(TelemetryGenerationRetryTxt, Attempt + 1), Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', GetFeatureName()); end; // this completion is of low quality Session.LogMessage('0000JYB', TelemetryLowQualityCompletionTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', GetFeatureName()); Error(MagicFunction.Execute(EmptyArguments)); end; [NonDebuggable] local procedure CompletionContainsPrompt(Completion: Text; Prompt: Text): Boolean var PromptSentences: List of [Text]; PromptSentence: Text; begin PromptSentences := Prompt.Split('.'); Completion := Completion.ToLower(); foreach PromptSentence in PromptSentences do begin PromptSentence := PromptSentence.Trim().ToLower(); if PromptSentence <> '' then if Completion.Contains(PromptSentence) then begin Session.LogMessage('0000JZG', StrSubstNo(TelemetryCompletionHasPromptTxt, PromptSentence), Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', GetFeatureName()); exit(true); end; end; exit(false); end; [NonDebuggable] local procedure IsGoodCompletion(var Completion: Text; TextFormat: Enum "Entity Text Format"; Facts: Dictionary of [Text, Text]): Boolean var TempMatches: Record Matches temporary; Regex: Codeunit Regex; SplitCompletion: List of [Text]; FactKey: Text; FactValue: Text; CandidateNumber: Text; FoundNumber: Boolean; FormatValid: Boolean; begin if Completion = '' then begin Session.LogMessage('0000JWJ', TelemetryCompletionEmptyTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', GetFeatureName()); exit(false); end; if Completion.ToLower().StartsWith('tagline:') then begin Session.LogMessage('0000JYD', TelemetryTaglineCleanedTxt, Verbosity::Normal, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', GetFeatureName()); Completion := CopyStr(Completion, 9).Trim(); end; FormatValid := true; case TextFormat of TextFormat::TaglineParagraph: begin SplitCompletion := Completion.Split(EncodedNewlineTok + EncodedNewlineTok); FormatValid := SplitCompletion.Count() = 2; // a tagline + paragraph must contain an empty line end; TextFormat::Paragraph: FormatValid := (not Completion.Contains(EncodedNewlineTok + EncodedNewlineTok)); // multiple paragraphs should be avoided TextFormat::Tagline: FormatValid := not Completion.Contains(EncodedNewlineTok); // a tagline should not have any newline TextFormat::Brief: FormatValid := Completion.Contains(EncodedNewlineTok + EncodedNewlineTok) and (Completion.Contains(EncodedNewlineTok + '-') or Completion.Contains(EncodedNewlineTok + '•') or Completion.Contains(EncodedNewlineTok + '*')); // the brief should contain a pargraph and a list end; if not FormatValid then begin Session.LogMessage('0000JYC', StrSubstNo(TelemetryCompletionInvalidFormatTxt, TextFormat), Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', GetFeatureName()); exit(false); end; // check the facts Regex.Match(Completion, '(?<!\S)(\d*\.?\d+)(?!\S)', TempMatches); // extract numbers if not TempMatches.FindSet() then exit(true); // no numbers to validate repeat CandidateNumber := TempMatches.ReadValue(); FoundNumber := false; foreach FactKey in Facts.Keys() do if not FoundNumber then if FactKey.Contains(CandidateNumber) then FoundNumber := true else begin FactValue := Facts.Get(FactKey); if FactValue.Contains(CandidateNumber) then FoundNumber := true; end; if not FoundNumber then begin Session.LogMessage('0000JYE', StrSubstNo(TelemetryCompletionInvalidNumberTxt, CandidateNumber), Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', GetFeatureName()); exit(false); // made up number end; until TempMatches.Next() = 0; if Completion.Contains(StrSubstNo(NoteParagraphTxt, EncodedNewlineTok)) or Completion.Contains(StrSubstNo(TranslationParagraphTxt, EncodedNewlineTok)) then begin Session.LogMessage('0000LHZ', TelemetryCompletionExtraTextTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', GetFeatureName()); exit(false); end; exit(true); end; [NonDebuggable] local procedure GenerateCompletion(TextFormat: Enum "Entity Text Format"; SystemPrompt: Text; UserPrompt: Text; CallerModuleInfo: ModuleInfo): Text var AzureOpenAI: Codeunit "Azure OpenAI"; EntityTextAOAISettings: Codeunit "Entity Text AOAI Settings"; AOAIDeployments: Codeunit "AOAI Deployments"; AOAIOperationResponse: Codeunit "AOAI Operation Response"; AOAICompletionParams: Codeunit "AOAI Chat Completion Params"; AOAIChatMessages: Codeunit "AOAI Chat Messages"; MagicFunction: Codeunit "Magic Function"; GenerateProdMktAdFunction: Codeunit "Generate Prod Mkt Ad Function"; AOAIFunctionResponse: Codeunit "AOAI Function Response"; FeatureTelemetry: Codeunit "Feature Telemetry"; AOAIFunctionResponses: List of [Codeunit "AOAI Function Response"]; TelemetryCD: Dictionary of [Text, Text]; StartDateTime: DateTime; DurationAsBigInt: BigInteger; Result: Text; EntityTextModuleInfo: ModuleInfo; ResponseErr: Label 'AOAI Operation failed, response error code: %1', Comment = '%1 = Error code', Locked = true; begin NavApp.GetCurrentModuleInfo(EntityTextModuleInfo); if (not (Endpoint = '')) and (not (Deployment = '')) then AzureOpenAI.SetAuthorization(Enum::"AOAI Model Type"::"Chat Completions", Endpoint, Deployment, ApiKey) else if (not IsNullGuid(CallerModuleInfo.Id())) and (CallerModuleInfo.Publisher() = EntityTextModuleInfo.Publisher()) then AzureOpenAI.SetAuthorization(Enum::"AOAI Model Type"::"Chat Completions", AOAIDeployments.GetGPT56CeresLatest()) else begin TelemetryCD.Add('CallerModuleInfo', Format(CallerModuleInfo.Publisher())); TelemetryCD.Add('EntityTextModuleInfo', Format(EntityTextModuleInfo.Publisher())); FeatureTelemetry.LogError('0000LJB', GetFeatureName(), 'Entity Text Authorization', TelemetryNoAuthorizationHandlerTxt, '', TelemetryCD); Error(NoAuthorizationHandlerErr); end; AzureOpenAI.SetCopilotCapability(Enum::"Copilot Capability"::"Entity Text"); AOAICompletionParams.SetMaxTokens(2500); AOAICompletionParams.SetTemperature(0.7); AOAIChatMessages.AddTool(MagicFunction); GenerateProdMktAdFunction.SetTextFormat(TextFormat); AOAIChatMessages.AddTool(GenerateProdMktAdFunction); AOAIChatMessages.SetToolChoice('auto'); AOAIChatMessages.SetPrimarySystemMessage(SystemPrompt); AOAIChatMessages.AddUserMessage(UserPrompt); StartDateTime := CurrentDateTime(); AzureOpenAI.GenerateChatCompletion(AOAIChatMessages, AOAICompletionParams, AOAIOperationResponse); DurationAsBigInt := CurrentDateTime() - StartDateTime; TelemetryCD.Add('Response time', Format(DurationAsBigInt)); if AOAIOperationResponse.IsSuccess() then if AOAIOperationResponse.IsFunctionCall() then begin AOAIFunctionResponses := AOAIOperationResponse.GetFunctionResponses(); FeatureTelemetry.LogUsage('0000N5C', GetFeatureName(), 'Call Chat Completion API', TelemetryCD); foreach AOAIFunctionResponse in AOAIFunctionResponses do begin if not AOAIFunctionResponse.IsSuccess() then begin FeatureTelemetry.LogError('0000N5Z', GetFeatureName(), 'Call Chat Completion API', 'AOAI Function response is not sucessfull', '', TelemetryCD); exit(''); end; Result := AOAIFunctionResponse.GetResult(); if AOAIFunctionResponse.GetFunctionName() = MagicFunction.GetName() then Error(Result); end; end else begin Clear(Result); FeatureTelemetry.LogError('0000N5A', GetFeatureName(), 'Call Chat Completion API', 'AOAI response is not a function call', '', TelemetryCD); end else begin Clear(Result); FeatureTelemetry.LogError('0000N5B', GetFeatureName(), 'Call Chat Completion API', StrSubstNo(ResponseErr, AOAIOperationResponse.GetStatusCode()), '', TelemetryCD); end; if EntityTextAOAISettings.ContainsWordsInDenyList(Result) then Clear(Result); exit(Result); end; var Endpoint: Text; Deployment: Text; ApiKey: SecretText; FactTemplateTxt: Label '- %1: %2%3', Locked = true; EncodedNewlineTok: Label '<br />', Locked = true; NoteParagraphTxt: Label '%1Note:%1', Locked = true, Comment = 'This constant is used to limit the cases when the model goes out of format and must stay in English only.'; TranslationParagraphTxt: Label 'Translation:%1', Locked = true, Comment = 'This constant is used to limit the cases when the model goes out of format and must stay in English only.'; NoFactsErr: Label 'There''s no information available to draft a text from.'; CannotGenerateErr: Label 'Text cannot be generated. Please check your configuration and contact your partner.'; CapabilityDisabledErr: Label 'Sorry, your Copilot isn''t activated for Entity Text. Contact the system administrator.'; MinFactsErr: Label 'There''s not enough information available to draft a text. Please provide more.'; NotEnoughFactsForFormatErr: Label 'There''s not enough information available to draft a text for the chosen format. Please provide more, or choose another format.'; NoAuthorizationHandlerErr: Label 'There was no handler to provide authorization information for the suggestion. Contact your partner.'; TelemetryGenerationRequestedTxt: Label 'New suggestion requested.', Locked = true; TelemetrySuggestionCreatedTxt: Label 'A new suggestion was generated for table %1, scenario %2', Locked = true; TelemetryCompletionEmptyTxt: Label 'The returned completion was empty.', Locked = true; TelemetryLowQualityCompletionTxt: Label 'Failed to generate a good quality completion, returning a low quality one.', Locked = true; TelemetryCompletionInvalidFormatTxt: Label 'The format %1 was requested, but the completion format did not pass validation.', Locked = true; TelemetryCompletionHasPromptTxt: Label 'The completion contains this sentence from the prompt: %1', Locked = true; TelemetryTaglineCleanedTxt: Label 'Trimmed a completion', Locked = true; TelemetryCompletionInvalidNumberTxt: Label 'A number was found in the completion (%1) that did not exist in the facts.', Locked = true; TelemetryCompletionExtraTextTxt: Label 'The completion contains a Translation or Note section.', Locked = true; TelemetryPromptManyFactsTxt: Label 'There are %1 facts defined, they will be limited to %2.', Locked = true; TelemetryNoAuthorizationHandlerTxt: Label 'Entity Text authorization was not set.', Locked = true; TelemetryGenerationRetryTxt: Label 'Retrying text generation, attempt: %1', Locked = true;}