[GenAI] Add LLaMA support#7220

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
LittleLittleCloud merged 25 commits into from
Aug 28, 2024
Merged

[GenAI] Add LLaMA support #7220

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
7b17731
add llama
LittleLittleCloud Aug 13, 2024
996548b
add test for tokenizer
LittleLittleCloud Aug 14, 2024
023c9f7
make llama 3.1 working
LittleLittleCloud Aug 14, 2024
63be655
update
LittleLittleCloud Aug 14, 2024
44b3302
add shape test for 70b and 405b
LittleLittleCloud Aug 15, 2024
3d4482d
clean up
LittleLittleCloud Aug 15, 2024
d3681dd
Merge branch 'u/xiaoyun/AddLLaMA' of https://github.com/LittleLittleC…
LittleLittleCloud Aug 16, 2024
c77b518
add tests
LittleLittleCloud Aug 16, 2024
6498e2e
update
LittleLittleCloud Aug 16, 2024
6557eac
fix error
LittleLittleCloud Aug 16, 2024
fd61cff
calculate rotary embedding in model layer
LittleLittleCloud Aug 16, 2024
a3f2e41
remove rotary_emb from attention
LittleLittleCloud Aug 16, 2024
9a03acc
update feed
LittleLittleCloud Aug 17, 2024
0542bf7
update .csproj
LittleLittleCloud Aug 17, 2024
485f9af
Update NuGet.config
LittleLittleCloud Aug 19, 2024
896a21b
fix test
LittleLittleCloud Aug 19, 2024
d3d5b61
pass device
LittleLittleCloud Aug 21, 2024
d161b49
fix test
LittleLittleCloud Aug 21, 2024
32b74e3
update constructor
LittleLittleCloud Aug 21, 2024
b2b155c
disable 405b test
LittleLittleCloud Aug 21, 2024
2b48080
update
LittleLittleCloud Aug 22, 2024
e0ba719
disable 70b test
LittleLittleCloud Aug 22, 2024
093a38e
use windows only fact
LittleLittleCloud Aug 26, 2024
51bdcc6
revert change
LittleLittleCloud Aug 28, 2024
19a2d7e
rename test to LLaMA3_1
LittleLittleCloud Aug 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 23 additions & 1 deletion Microsoft.ML.sln
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -184,7 +184,11 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.Phi.Test
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.Samples", "docs\samples\Microsoft.ML.GenAI.Samples\Microsoft.ML.GenAI.Samples.csproj", "{1D4AD9A3-19AF-432B-889D-A63FE6D7BD47}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.ML.GenAI.Core.Tests", "test\Microsoft.ML.GenAI.Core.Tests\Microsoft.ML.GenAI.Core.Tests.csproj", "{14AB0804-D4CE-4634-B544-5A8587620783}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.Core.Tests", "test\Microsoft.ML.GenAI.Core.Tests\Microsoft.ML.GenAI.Core.Tests.csproj", "{14AB0804-D4CE-4634-B544-5A8587620783}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.LLaMA", "src\Microsoft.ML.GenAI.LLaMA\Microsoft.ML.GenAI.LLaMA.csproj", "{0AA6D5CB-195F-457A-8792-4221E76E6C44}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.ML.GenAI.LLaMA.Tests", "test\Microsoft.ML.GenAI.LLaMA.Tests\Microsoft.ML.GenAI.LLaMA.Tests.csproj", "{D202353D-6FAF-4263-9A01-BDCFBC92391F}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Expand DownExpand Up@@ -878,6 +882,22 @@ Global
{14AB0804-D4CE-4634-B544-5A8587620783}.Release|Any CPU.Build.0 = Release|Any CPU
{14AB0804-D4CE-4634-B544-5A8587620783}.Release|x64.ActiveCfg = Release|Any CPU
{14AB0804-D4CE-4634-B544-5A8587620783}.Release|x64.Build.0 = Release|Any CPU
{0AA6D5CB-195F-457A-8792-4221E76E6C44}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{0AA6D5CB-195F-457A-8792-4221E76E6C44}.Debug|Any CPU.Build.0 = Debug|Any CPU
{0AA6D5CB-195F-457A-8792-4221E76E6C44}.Debug|x64.ActiveCfg = Debug|Any CPU
{0AA6D5CB-195F-457A-8792-4221E76E6C44}.Debug|x64.Build.0 = Debug|Any CPU
{0AA6D5CB-195F-457A-8792-4221E76E6C44}.Release|Any CPU.ActiveCfg = Release|Any CPU
{0AA6D5CB-195F-457A-8792-4221E76E6C44}.Release|Any CPU.Build.0 = Release|Any CPU
{0AA6D5CB-195F-457A-8792-4221E76E6C44}.Release|x64.ActiveCfg = Release|Any CPU
{0AA6D5CB-195F-457A-8792-4221E76E6C44}.Release|x64.Build.0 = Release|Any CPU
{D202353D-6FAF-4263-9A01-BDCFBC92391F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{D202353D-6FAF-4263-9A01-BDCFBC92391F}.Debug|Any CPU.Build.0 = Debug|Any CPU
{D202353D-6FAF-4263-9A01-BDCFBC92391F}.Debug|x64.ActiveCfg = Debug|Any CPU
{D202353D-6FAF-4263-9A01-BDCFBC92391F}.Debug|x64.Build.0 = Debug|Any CPU
{D202353D-6FAF-4263-9A01-BDCFBC92391F}.Release|Any CPU.ActiveCfg = Release|Any CPU
{D202353D-6FAF-4263-9A01-BDCFBC92391F}.Release|Any CPU.Build.0 = Release|Any CPU
{D202353D-6FAF-4263-9A01-BDCFBC92391F}.Release|x64.ActiveCfg = Release|Any CPU
{D202353D-6FAF-4263-9A01-BDCFBC92391F}.Release|x64.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand DownExpand Up@@ -969,6 +989,8 @@ Global
{867FFC34-DFA7-400F-B9BB-85158326CE08} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
{1D4AD9A3-19AF-432B-889D-A63FE6D7BD47} = {DA452A53-2E94-4433-B08C-041EDEC729E6}
{14AB0804-D4CE-4634-B544-5A8587620783} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
{0AA6D5CB-195F-457A-8792-4221E76E6C44} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
{D202353D-6FAF-4263-9A01-BDCFBC92391F} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {41165AF1-35BB-4832-A189-73060F82B01D}
Expand Down
4 changes: 4 additions & 0 deletions NuGet.config
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,7 @@
<add key="dotnet5-roslyn" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json" />
<add key="mlnet-daily" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/MachineLearning/nuget/v3/index.json" />
<add key="mlnet-assets" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/machinelearning-assets/nuget/v3/index.json" />
<add key="dotnet-libraries-transport" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-libraries-transport/nuget/v3/index.json" />
<add key="dotnet8" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet8/nuget/v3/index.json" />
</packageSources>
<packageSourceMapping>
Expand DownExpand Up@@ -40,6 +41,9 @@
<packageSource key="mlnet-assets">
<package pattern="*" />
</packageSource>
<packageSource key="dotnet-libraries-transport">
<package pattern="*" />
</packageSource>
<packageSource key="dotnet8">
<package pattern="*" />
</packageSource>
Expand Down
51 changes: 51 additions & 0 deletions docs/samples/Microsoft.ML.GenAI.Samples/Llama/LLaMA3_1.cs
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.Json;
using System.Threading.Tasks;
using AutoGen.Core;
using Microsoft.ML.GenAI.Core;
using Microsoft.ML.GenAI.Core.Extension;
using Microsoft.ML.GenAI.LLaMA;
using Microsoft.ML.Tokenizers;
using TorchSharp;
using static TorchSharp.torch;

namespace Microsoft.ML.GenAI.Samples.Llama;

internal class LlamaSample
{
public static async void Run()
{
var device = "cuda";
if (device == "cuda")
{
torch.InitializeDeviceType(DeviceType.CUDA);
}

var defaultType = ScalarType.Float16;
torch.manual_seed(1);
torch.set_default_dtype(defaultType);
var weightFolder = @"C:\Users\xiaoyuz\source\repos\Meta-Llama-3.1-8B-Instruct";
var configName = "config.json";
var originalWeightFolder = Path.Combine(weightFolder, "original");

Console.WriteLine("Loading Llama from huggingface model weight folder");
var stopWatch = System.Diagnostics.Stopwatch.StartNew();
stopWatch.Start();
var tokenizer = LlamaTokenizerHelper.FromPretrained(originalWeightFolder);
var model = LlamaForCausalLM.FromPretrained(weightFolder, configName, layersOnTargetDevice: -1);

var pipeline = new CausalLMPipeline<TiktokenTokenizer, LlamaForCausalLM>(tokenizer, model, device);

var agent = new LlamaCausalLMAgent(pipeline, "assistant")
.RegisterPrintMessage();

var task = """
Write a C# program to print the sum of two numbers. Use top-level statement, put code between ```csharp and ```.
""";

await agent.SendAsync(task);
}
}
1 change: 1 addition & 0 deletions docs/samples/Microsoft.ML.GenAI.Samples/Microsoft.ML.GenAI.Samples.csproj
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -9,6 +9,7 @@

<ItemGroup>
<ProjectReference Include="..\..\..\src\Microsoft.ML.GenAI.Core\Microsoft.ML.GenAI.Core.csproj" />
<ProjectReference Include="..\..\..\src\Microsoft.ML.GenAI.LLaMA\Microsoft.ML.GenAI.LLaMA.csproj" />
<ProjectReference Include="..\..\..\src\Microsoft.ML.GenAI.Phi\Microsoft.ML.GenAI.Phi.csproj" />
</ItemGroup>

Expand Down
2 changes: 1 addition & 1 deletion docs/samples/Microsoft.ML.GenAI.Samples/Phi3Mini/AutoGenSample.cs
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,7 +26,7 @@ public static async Task RunAsync()
torch.manual_seed(1);
torch.set_default_dtype(defaultType);
var weightFolder = @"C:\Users\xiaoyuz\source\repos\Phi-3-mini-4k-instruct";
var pipeline = Utils.LoadPhi3Mini4KFromFolder(weightFolder, device: device);
var pipeline = Utils.LoadPhi3Mini4KFromFolder(weightFolder, device: device, quantizeToInt8: false);

// agent
var agent = new Phi3Agent(pipeline, "assistant")
Expand Down
2 changes: 1 addition & 1 deletion docs/samples/Microsoft.ML.GenAI.Samples/Phi3Mini/Utils.cs
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -20,7 +20,7 @@ public static ICausalLMPipeline<Tokenizer, Phi3ForCasualLM> LoadPhi3Mini4KFromFo
string weightFolder,
string configName = "config.json",
string device = "cuda",
int modelSizeOnCudaInGB = 16,
Comment thread
LittleLittleCloud marked this conversation as resolved.
int modelSizeOnCudaInGB = 55,
int modelSizeOnMemoryInGB = 64,
int modelSizeOnDiskInGB = 200,
bool quantizeToInt8 = false,
Expand Down
2 changes: 1 addition & 1 deletion eng/Versions.props
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -96,7 +96,7 @@
<MicrosoftMLTensorFlowTestModelsVersion>0.0.13-test</MicrosoftMLTensorFlowTestModelsVersion>
<MicrosoftMLTestDatabasesVersion>0.0.6-test</MicrosoftMLTestDatabasesVersion>
<MicrosoftMLTestModelsVersion>0.0.7-test</MicrosoftMLTestModelsVersion>
<MicrosoftMLTestTokenizersVersion>2.0.0-beta.24219.1</MicrosoftMLTestTokenizersVersion>
<MicrosoftMLTestTokenizersVersion>2.0.0-beta.24415.1</MicrosoftMLTestTokenizersVersion>
<SystemDataSqlClientVersion>4.8.6</SystemDataSqlClientVersion>
<SystemDataSQLiteCoreVersion>1.0.118</SystemDataSQLiteCoreVersion>
<XunitCombinatorialVersion>1.6.24</XunitCombinatorialVersion>
Expand Down
51 changes: 51 additions & 0 deletions src/Microsoft.ML.GenAI.Core/Extension/ModuleExtension.cs
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -197,6 +197,57 @@ public static Dictionary<string, string> InferDeviceMapForEachLayer(
return deviceMap;
}

/// <summary>
/// Infer the device map for each layer in the model.
/// The device map is a dictionary where the key is the device id (e.g. "cuda:0") and the value is the memory size in bytes of the device.
/// When inferring the device map, each layer in the model will be placed on the device in the order of the devices list.
/// </summary>
/// <param name="model"></param>
/// <param name="numberOfLayerToBePlaced">a list of key-value pairs where the key is the device id (e.g. "cuda:0") and the value is the number of layers to be placed on the device.
/// If you want to place all remaining layers on the device, set that value to -1.
/// e.g. [{"cuda:0", 2}, {"cpu", -1}], the first 2 layers will be placed on "cuda:0" and the rest will be placed on "cpu".
/// </param>
/// <returns></returns>
public static Dictionary<string, string> InferDeviceMapForEachLayer(
this nn.Module model,
IEnumerable<KeyValuePair<string, int>> numberOfLayerToBePlaced)
{
var layerSizeMap = model.GetSizeForEachDynamicLayerInBytes()
.OrderByDescending(x => x.Value)
.ToList();

var deviceMap = new Dictionary<string, string>();
foreach (var (device, count) in numberOfLayerToBePlaced)
{
if (count != -1)
{
var topK = layerSizeMap.Take(count).ToList();
layerSizeMap = layerSizeMap.Skip(count).ToList();
foreach (var (key, value) in topK)
{
deviceMap[key] = device;
}
}
else
{
foreach (var (key, value) in layerSizeMap)
{
deviceMap[key] = device;
}

layerSizeMap.Clear();
break;
}
}

if (layerSizeMap.Count > 0)
{
throw new ArgumentException("The layer count is not enough to cover all layers, did you forget to set the last layer count to -1?");
}

return deviceMap;
}

internal static string Peek(this nn.Module model)
{
var sb = new StringBuilder();
Expand Down
11 changes: 4 additions & 7 deletions src/Microsoft.ML.GenAI.Core/Microsoft.ML.GenAI.Core.csproj
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -8,23 +8,20 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="AutoGen.Core" Version="$(AutoGenVersion)" />
<PackageReference Include="Microsoft.SemanticKernel.Abstractions" Version="$(SemanticKernelVersion)" />
<PackageReference Include="System.Memory" Version="$(SystemMemoryVersion)" />
<PackageReference Include="TorchSharp" Version="$(TorchSharpVersion)" />
</ItemGroup>
<!--

<ItemGroup Condition="'$(Configuration)' == 'Debug'">
<PackageReference Include="libtorch-cpu-win-x64" Version="$(LibTorchVersion)" Condition="$([MSBuild]::IsOSPlatform('Windows'))" PrivateAssets="all" />
<PackageReference Include="libtorch-cpu-linux-x64" Version="$(LibTorchVersion)" Condition="$([MSBuild]::IsOSPlatform('Linux'))" PrivateAssets="all" />
<PackageReference Include="libtorch-cpu-osx-x64" Version="$(LibTorchVersion)" Condition="$([MSBuild]::IsOSPlatform('OSX'))" PrivateAssets="all" />
</ItemGroup> -->

<ItemGroup>
<ProjectReference Include="..\Microsoft.ML.Tokenizers\Microsoft.ML.Tokenizers.csproj" />
</ItemGroup>

<ItemGroup>
<InternalsVisibleTo Include="Microsoft.ML.GenAI.Phi" />
<InternalsVisibleTo Include="Microsoft.ML.GenAI.LLaMA" />
<InternalsVisibleTo Include="Microsoft.ML.GenAI.LLaMA.Tests" />
<InternalsVisibleTo Include="Microsoft.ML.GenAI.Phi.Tests" />
<InternalsVisibleTo Include="Microsoft.ML.GenAI.Core.Tests" />
</ItemGroup>
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
[GenAI] Add LLaMA support#7220

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

LittleLittleCloud
merged 25 commits into

from

Aug 28, 2024
 Merged


[GenAI] Add LLaMA support
#7220
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


25 commits


Select commit
Hold shift + click to select a range
7b17731

add llama


LittleLittleCloud Aug 13, 2024
996548b

add test for tokenizer


LittleLittleCloud Aug 14, 2024
023c9f7

make llama 3.1 working


LittleLittleCloud Aug 14, 2024
63be655

update


LittleLittleCloud Aug 14, 2024
44b3302

add shape test for 70b and 405b


LittleLittleCloud Aug 15, 2024
3d4482d

clean up


LittleLittleCloud Aug 15, 2024
d3681dd

Merge branch 'u/xiaoyun/AddLLaMA' of https://github.com/LittleLittleC…


LittleLittleCloud Aug 16, 2024
c77b518

add tests


LittleLittleCloud Aug 16, 2024
6498e2e

update


LittleLittleCloud Aug 16, 2024
6557eac

fix error


LittleLittleCloud Aug 16, 2024
fd61cff

calculate rotary embedding in model layer


LittleLittleCloud Aug 16, 2024
a3f2e41

remove rotary_emb from attention


LittleLittleCloud Aug 16, 2024
9a03acc

update feed


LittleLittleCloud Aug 17, 2024
0542bf7

update .csproj


LittleLittleCloud Aug 17, 2024
485f9af

Update NuGet.config


LittleLittleCloud Aug 19, 2024
896a21b

fix test


LittleLittleCloud Aug 19, 2024
d3d5b61

pass device


LittleLittleCloud Aug 21, 2024
d161b49

fix test


LittleLittleCloud Aug 21, 2024
32b74e3

update constructor


LittleLittleCloud Aug 21, 2024
b2b155c

disable 405b test


LittleLittleCloud Aug 21, 2024
2b48080

update


LittleLittleCloud Aug 22, 2024
e0ba719

disable 70b test


LittleLittleCloud Aug 22, 2024
093a38e

use windows only fact


LittleLittleCloud Aug 26, 2024
51bdcc6

revert change


LittleLittleCloud Aug 28, 2024
19a2d7e

rename test to LLaMA3_1


LittleLittleCloud Aug 28, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


24 changes: 23 additions & 1 deletion
Microsoft.ML.sln

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -184,7 +184,11 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.Phi.Test
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.Samples", "docs\samples\Microsoft.ML.GenAI.Samples\Microsoft.ML.GenAI.Samples.csproj", "{1D4AD9A3-19AF-432B-889D-A63FE6D7BD47}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.ML.GenAI.Core.Tests", "test\Microsoft.ML.GenAI.Core.Tests\Microsoft.ML.GenAI.Core.Tests.csproj", "{14AB0804-D4CE-4634-B544-5A8587620783}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.Core.Tests", "test\Microsoft.ML.GenAI.Core.Tests\Microsoft.ML.GenAI.Core.Tests.csproj", "{14AB0804-D4CE-4634-B544-5A8587620783}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.LLaMA", "src\Microsoft.ML.GenAI.LLaMA\Microsoft.ML.GenAI.LLaMA.csproj", "{0AA6D5CB-195F-457A-8792-4221E76E6C44}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.ML.GenAI.LLaMA.Tests", "test\Microsoft.ML.GenAI.LLaMA.Tests\Microsoft.ML.GenAI.LLaMA.Tests.csproj", "{D202353D-6FAF-4263-9A01-BDCFBC92391F}"
EndProject
Global
	GlobalSection(SolutionConfigurationPlatforms) = preSolution
Expand DownExpand Up@@ -878,6 +882,22 @@ Global
 {14AB0804-D4CE-4634-B544-5A8587620783}.Release|Any CPU.Build.0 = Release|Any CPU
 {14AB0804-D4CE-4634-B544-5A8587620783}.Release|x64.ActiveCfg = Release|Any CPU
 {14AB0804-D4CE-4634-B544-5A8587620783}.Release|x64.Build.0 = Release|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Debug|Any CPU.Build.0 = Debug|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Debug|x64.ActiveCfg = Debug|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Debug|x64.Build.0 = Debug|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Release|Any CPU.ActiveCfg = Release|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Release|Any CPU.Build.0 = Release|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Release|x64.ActiveCfg = Release|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Release|x64.Build.0 = Release|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Debug|Any CPU.Build.0 = Debug|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Debug|x64.ActiveCfg = Debug|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Debug|x64.Build.0 = Debug|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Release|Any CPU.ActiveCfg = Release|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Release|Any CPU.Build.0 = Release|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Release|x64.ActiveCfg = Release|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Release|x64.Build.0 = Release|Any CPU
	EndGlobalSection
	GlobalSection(SolutionProperties) = preSolution
 HideSolutionNode = FALSE
Expand DownExpand Up@@ -969,6 +989,8 @@ Global
 {867FFC34-DFA7-400F-B9BB-85158326CE08} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
 {1D4AD9A3-19AF-432B-889D-A63FE6D7BD47} = {DA452A53-2E94-4433-B08C-041EDEC729E6}
 {14AB0804-D4CE-4634-B544-5A8587620783} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
 {0AA6D5CB-195F-457A-8792-4221E76E6C44} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
 {D202353D-6FAF-4263-9A01-BDCFBC92391F} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
	EndGlobalSection
	GlobalSection(ExtensibilityGlobals) = postSolution
 SolutionGuid = {41165AF1-35BB-4832-A189-73060F82B01D}
Expand Down

4 changes: 4 additions & 0 deletions
NuGet.config

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,7 @@
 <add key="dotnet5-roslyn" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json" />
 <add key="mlnet-daily" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/MachineLearning/nuget/v3/index.json" />
 <add key="mlnet-assets" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/machinelearning-assets/nuget/v3/index.json" />
 <add key="dotnet-libraries-transport" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-libraries-transport/nuget/v3/index.json" />
 <add key="dotnet8" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet8/nuget/v3/index.json" />
 </packageSources>
 <packageSourceMapping>
Expand DownExpand Up@@ -40,6 +41,9 @@
 <packageSource key="mlnet-assets">
 <package pattern="*" />
 </packageSource>
 <packageSource key="dotnet-libraries-transport">
 <package pattern="*" />
 </packageSource>
 <packageSource key="dotnet8">
 <package pattern="*" />
 </packageSource>
Expand Down

51 changes: 51 additions & 0 deletions
docs/samples/Microsoft.ML.GenAI.Samples/Llama/LLaMA3_1.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.Json;
using System.Threading.Tasks;
using AutoGen.Core;
using Microsoft.ML.GenAI.Core;
using Microsoft.ML.GenAI.Core.Extension;
using Microsoft.ML.GenAI.LLaMA;
using Microsoft.ML.Tokenizers;
using TorchSharp;
using static TorchSharp.torch;

namespace Microsoft.ML.GenAI.Samples.Llama;

internal class LlamaSample
{
 public static async void Run()
 {
 var device = "cuda";
 if (device == "cuda")
 {
 torch.InitializeDeviceType(DeviceType.CUDA);
 }

 var defaultType = ScalarType.Float16;
 torch.manual_seed(1);
 torch.set_default_dtype(defaultType);
 var weightFolder = @"C:\Users\xiaoyuz\source\repos\Meta-Llama-3.1-8B-Instruct";
 var configName = "config.json";
 var originalWeightFolder = Path.Combine(weightFolder, "original");

 Console.WriteLine("Loading Llama from huggingface model weight folder");
 var stopWatch = System.Diagnostics.Stopwatch.StartNew();
 stopWatch.Start();
 var tokenizer = LlamaTokenizerHelper.FromPretrained(originalWeightFolder);
 var model = LlamaForCausalLM.FromPretrained(weightFolder, configName, layersOnTargetDevice: -1);

 var pipeline = new CausalLMPipeline<TiktokenTokenizer, LlamaForCausalLM>(tokenizer, model, device);

 var agent = new LlamaCausalLMAgent(pipeline, "assistant")
 .RegisterPrintMessage();

 var task = """
 Write a C# program to print the sum of two numbers. Use top-level statement, put code between ```csharp and ```.
 """;

 await agent.SendAsync(task);
 }
}

1 change: 1 addition & 0 deletions
docs/samples/Microsoft.ML.GenAI.Samples/Microsoft.ML.GenAI.Samples.csproj

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -9,6 +9,7 @@

 <ItemGroup>
 <ProjectReference Include="..\..\..\src\Microsoft.ML.GenAI.Core\Microsoft.ML.GenAI.Core.csproj" />
 <ProjectReference Include="..\..\..\src\Microsoft.ML.GenAI.LLaMA\Microsoft.ML.GenAI.LLaMA.csproj" />
 <ProjectReference Include="..\..\..\src\Microsoft.ML.GenAI.Phi\Microsoft.ML.GenAI.Phi.csproj" />
 </ItemGroup>

Expand Down

2 changes: 1 addition & 1 deletion
docs/samples/Microsoft.ML.GenAI.Samples/Phi3Mini/AutoGenSample.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,7 +26,7 @@ public static async Task RunAsync()
 torch.manual_seed(1);
 torch.set_default_dtype(defaultType);
 var weightFolder = @"C:\Users\xiaoyuz\source\repos\Phi-3-mini-4k-instruct";
 var pipeline = Utils.LoadPhi3Mini4KFromFolder(weightFolder, device: device);
 var pipeline = Utils.LoadPhi3Mini4KFromFolder(weightFolder, device: device, quantizeToInt8: false);

 // agent
 var agent = new Phi3Agent(pipeline, "assistant")
Expand Down

2 changes: 1 addition & 1 deletion
docs/samples/Microsoft.ML.GenAI.Samples/Phi3Mini/Utils.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,7 +20,7 @@ public static ICausalLMPipeline<Tokenizer, Phi3ForCasualLM> LoadPhi3Mini4KFromFo
 string weightFolder,
 string configName = "config.json",
 string device = "cuda",
 int modelSizeOnCudaInGB = 16,
Comment thread
LittleLittleCloud marked this conversation as resolved.

 int modelSizeOnCudaInGB = 55,
 int modelSizeOnMemoryInGB = 64,
 int modelSizeOnDiskInGB = 200,
 bool quantizeToInt8 = false,
Expand Down

2 changes: 1 addition & 1 deletion
eng/Versions.props

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -96,7 +96,7 @@
 <MicrosoftMLTensorFlowTestModelsVersion>0.0.13-test</MicrosoftMLTensorFlowTestModelsVersion>
 <MicrosoftMLTestDatabasesVersion>0.0.6-test</MicrosoftMLTestDatabasesVersion>
 <MicrosoftMLTestModelsVersion>0.0.7-test</MicrosoftMLTestModelsVersion>
 <MicrosoftMLTestTokenizersVersion>2.0.0-beta.24219.1</MicrosoftMLTestTokenizersVersion>
 <MicrosoftMLTestTokenizersVersion>2.0.0-beta.24415.1</MicrosoftMLTestTokenizersVersion>
 <SystemDataSqlClientVersion>4.8.6</SystemDataSqlClientVersion>
 <SystemDataSQLiteCoreVersion>1.0.118</SystemDataSQLiteCoreVersion>
 <XunitCombinatorialVersion>1.6.24</XunitCombinatorialVersion>
Expand Down

51 changes: 51 additions & 0 deletions
src/Microsoft.ML.GenAI.Core/Extension/ModuleExtension.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -197,6 +197,57 @@ public static Dictionary<string, string> InferDeviceMapForEachLayer(
 return deviceMap;
 }

 /// <summary>
 /// Infer the device map for each layer in the model.
 /// The device map is a dictionary where the key is the device id (e.g. "cuda:0") and the value is the memory size in bytes of the device.
 /// When inferring the device map, each layer in the model will be placed on the device in the order of the devices list.
 /// </summary>
 /// <param name="model"></param>
 /// <param name="numberOfLayerToBePlaced">a list of key-value pairs where the key is the device id (e.g. "cuda:0") and the value is the number of layers to be placed on the device.
 /// If you want to place all remaining layers on the device, set that value to -1.
 /// e.g. [{"cuda:0", 2}, {"cpu", -1}], the first 2 layers will be placed on "cuda:0" and the rest will be placed on "cpu".
 /// </param>
 /// <returns></returns>
 public static Dictionary<string, string> InferDeviceMapForEachLayer(
 this nn.Module model,
 IEnumerable<KeyValuePair<string, int>> numberOfLayerToBePlaced)
 {
 var layerSizeMap = model.GetSizeForEachDynamicLayerInBytes()
 .OrderByDescending(x => x.Value)
 .ToList();

 var deviceMap = new Dictionary<string, string>();
 foreach (var (device, count) in numberOfLayerToBePlaced)
 {
 if (count != -1)
 {
 var topK = layerSizeMap.Take(count).ToList();
 layerSizeMap = layerSizeMap.Skip(count).ToList();
 foreach (var (key, value) in topK)
 {
 deviceMap[key] = device;
 }
 }
 else
 {
 foreach (var (key, value) in layerSizeMap)
 {
 deviceMap[key] = device;
 }

 layerSizeMap.Clear();
 break;
 }
 }

 if (layerSizeMap.Count > 0)
 {
 throw new ArgumentException("The layer count is not enough to cover all layers, did you forget to set the last layer count to -1?");
 }

 return deviceMap;
 }

 internal static string Peek(this nn.Module model)
 {
 var sb = new StringBuilder();
Expand Down

11 changes: 4 additions & 7 deletions
src/Microsoft.ML.GenAI.Core/Microsoft.ML.GenAI.Core.csproj

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -8,23 +8,20 @@
 </PropertyGroup>

 <ItemGroup>
 <PackageReference Include="AutoGen.Core" Version="$(AutoGenVersion)" />
 <PackageReference Include="Microsoft.SemanticKernel.Abstractions" Version="$(SemanticKernelVersion)" />
 <PackageReference Include="System.Memory" Version="$(SystemMemoryVersion)" />
 <PackageReference Include="TorchSharp" Version="$(TorchSharpVersion)" />
 </ItemGroup>
<!-- 

 <ItemGroup Condition="'$(Configuration)' == 'Debug'">
 <PackageReference Include="libtorch-cpu-win-x64" Version="$(LibTorchVersion)" Condition="$([MSBuild]::IsOSPlatform('Windows'))" PrivateAssets="all" />
 <PackageReference Include="libtorch-cpu-linux-x64" Version="$(LibTorchVersion)" Condition="$([MSBuild]::IsOSPlatform('Linux'))" PrivateAssets="all" />
 <PackageReference Include="libtorch-cpu-osx-x64" Version="$(LibTorchVersion)" Condition="$([MSBuild]::IsOSPlatform('OSX'))" PrivateAssets="all" />
 </ItemGroup> -->

 <ItemGroup>
 <ProjectReference Include="..\Microsoft.ML.Tokenizers\Microsoft.ML.Tokenizers.csproj" />
 </ItemGroup>

 <ItemGroup>
 <InternalsVisibleTo Include="Microsoft.ML.GenAI.Phi" />
 <InternalsVisibleTo Include="Microsoft.ML.GenAI.LLaMA" />
 <InternalsVisibleTo Include="Microsoft.ML.GenAI.LLaMA.Tests" />
 <InternalsVisibleTo Include="Microsoft.ML.GenAI.Phi.Tests" />
 <InternalsVisibleTo Include="Microsoft.ML.GenAI.Core.Tests" />
 </ItemGroup>
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
[GenAI] Add LLaMA support#7220

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

LittleLittleCloud
merged 25 commits into

from

Aug 28, 2024
 Merged


[GenAI] Add LLaMA support
#7220
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


25 commits


Select commit
Hold shift + click to select a range
7b17731

add llama


LittleLittleCloud Aug 13, 2024
996548b

add test for tokenizer


LittleLittleCloud Aug 14, 2024
023c9f7

make llama 3.1 working


LittleLittleCloud Aug 14, 2024
63be655

update


LittleLittleCloud Aug 14, 2024
44b3302

add shape test for 70b and 405b


LittleLittleCloud Aug 15, 2024
3d4482d

clean up


LittleLittleCloud Aug 15, 2024
d3681dd

Merge branch 'u/xiaoyun/AddLLaMA' of https://github.com/LittleLittleC…


LittleLittleCloud Aug 16, 2024
c77b518

add tests


LittleLittleCloud Aug 16, 2024
6498e2e

update


LittleLittleCloud Aug 16, 2024
6557eac

fix error


LittleLittleCloud Aug 16, 2024
fd61cff

calculate rotary embedding in model layer


LittleLittleCloud Aug 16, 2024
a3f2e41

remove rotary_emb from attention


LittleLittleCloud Aug 16, 2024
9a03acc

update feed


LittleLittleCloud Aug 17, 2024
0542bf7

update .csproj


LittleLittleCloud Aug 17, 2024
485f9af

Update NuGet.config


LittleLittleCloud Aug 19, 2024
896a21b

fix test


LittleLittleCloud Aug 19, 2024
d3d5b61

pass device


LittleLittleCloud Aug 21, 2024
d161b49

fix test


LittleLittleCloud Aug 21, 2024
32b74e3

update constructor


LittleLittleCloud Aug 21, 2024
b2b155c

disable 405b test


LittleLittleCloud Aug 21, 2024
2b48080

update


LittleLittleCloud Aug 22, 2024
e0ba719

disable 70b test


LittleLittleCloud Aug 22, 2024
093a38e

use windows only fact


LittleLittleCloud Aug 26, 2024
51bdcc6

revert change


LittleLittleCloud Aug 28, 2024
19a2d7e

rename test to LLaMA3_1


LittleLittleCloud Aug 28, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


24 changes: 23 additions & 1 deletion
Microsoft.ML.sln

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -184,7 +184,11 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.Phi.Test
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.Samples", "docs\samples\Microsoft.ML.GenAI.Samples\Microsoft.ML.GenAI.Samples.csproj", "{1D4AD9A3-19AF-432B-889D-A63FE6D7BD47}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.ML.GenAI.Core.Tests", "test\Microsoft.ML.GenAI.Core.Tests\Microsoft.ML.GenAI.Core.Tests.csproj", "{14AB0804-D4CE-4634-B544-5A8587620783}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.Core.Tests", "test\Microsoft.ML.GenAI.Core.Tests\Microsoft.ML.GenAI.Core.Tests.csproj", "{14AB0804-D4CE-4634-B544-5A8587620783}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.LLaMA", "src\Microsoft.ML.GenAI.LLaMA\Microsoft.ML.GenAI.LLaMA.csproj", "{0AA6D5CB-195F-457A-8792-4221E76E6C44}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.ML.GenAI.LLaMA.Tests", "test\Microsoft.ML.GenAI.LLaMA.Tests\Microsoft.ML.GenAI.LLaMA.Tests.csproj", "{D202353D-6FAF-4263-9A01-BDCFBC92391F}"
EndProject
Global
	GlobalSection(SolutionConfigurationPlatforms) = preSolution
Expand DownExpand Up@@ -878,6 +882,22 @@ Global
 {14AB0804-D4CE-4634-B544-5A8587620783}.Release|Any CPU.Build.0 = Release|Any CPU
 {14AB0804-D4CE-4634-B544-5A8587620783}.Release|x64.ActiveCfg = Release|Any CPU
 {14AB0804-D4CE-4634-B544-5A8587620783}.Release|x64.Build.0 = Release|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Debug|Any CPU.Build.0 = Debug|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Debug|x64.ActiveCfg = Debug|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Debug|x64.Build.0 = Debug|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Release|Any CPU.ActiveCfg = Release|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Release|Any CPU.Build.0 = Release|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Release|x64.ActiveCfg = Release|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Release|x64.Build.0 = Release|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Debug|Any CPU.Build.0 = Debug|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Debug|x64.ActiveCfg = Debug|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Debug|x64.Build.0 = Debug|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Release|Any CPU.ActiveCfg = Release|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Release|Any CPU.Build.0 = Release|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Release|x64.ActiveCfg = Release|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Release|x64.Build.0 = Release|Any CPU
	EndGlobalSection
	GlobalSection(SolutionProperties) = preSolution
 HideSolutionNode = FALSE
Expand DownExpand Up@@ -969,6 +989,8 @@ Global
 {867FFC34-DFA7-400F-B9BB-85158326CE08} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
 {1D4AD9A3-19AF-432B-889D-A63FE6D7BD47} = {DA452A53-2E94-4433-B08C-041EDEC729E6}
 {14AB0804-D4CE-4634-B544-5A8587620783} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
 {0AA6D5CB-195F-457A-8792-4221E76E6C44} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
 {D202353D-6FAF-4263-9A01-BDCFBC92391F} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
	EndGlobalSection
	GlobalSection(ExtensibilityGlobals) = postSolution
 SolutionGuid = {41165AF1-35BB-4832-A189-73060F82B01D}
Expand Down

4 changes: 4 additions & 0 deletions
NuGet.config

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,7 @@
 <add key="dotnet5-roslyn" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json" />
 <add key="mlnet-daily" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/MachineLearning/nuget/v3/index.json" />
 <add key="mlnet-assets" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/machinelearning-assets/nuget/v3/index.json" />
 <add key="dotnet-libraries-transport" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-libraries-transport/nuget/v3/index.json" />
 <add key="dotnet8" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet8/nuget/v3/index.json" />
 </packageSources>
 <packageSourceMapping>
Expand DownExpand Up@@ -40,6 +41,9 @@
 <packageSource key="mlnet-assets">
 <package pattern="*" />
 </packageSource>
 <packageSource key="dotnet-libraries-transport">
 <package pattern="*" />
 </packageSource>
 <packageSource key="dotnet8">
 <package pattern="*" />
 </packageSource>
Expand Down

51 changes: 51 additions & 0 deletions
docs/samples/Microsoft.ML.GenAI.Samples/Llama/LLaMA3_1.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.Json;
using System.Threading.Tasks;
using AutoGen.Core;
using Microsoft.ML.GenAI.Core;
using Microsoft.ML.GenAI.Core.Extension;
using Microsoft.ML.GenAI.LLaMA;
using Microsoft.ML.Tokenizers;
using TorchSharp;
using static TorchSharp.torch;

namespace Microsoft.ML.GenAI.Samples.Llama;

internal class LlamaSample
{
 public static async void Run()
 {
 var device = "cuda";
 if (device == "cuda")
 {
 torch.InitializeDeviceType(DeviceType.CUDA);
 }

 var defaultType = ScalarType.Float16;
 torch.manual_seed(1);
 torch.set_default_dtype(defaultType);
 var weightFolder = @"C:\Users\xiaoyuz\source\repos\Meta-Llama-3.1-8B-Instruct";
 var configName = "config.json";
 var originalWeightFolder = Path.Combine(weightFolder, "original");

 Console.WriteLine("Loading Llama from huggingface model weight folder");
 var stopWatch = System.Diagnostics.Stopwatch.StartNew();
 stopWatch.Start();
 var tokenizer = LlamaTokenizerHelper.FromPretrained(originalWeightFolder);
 var model = LlamaForCausalLM.FromPretrained(weightFolder, configName, layersOnTargetDevice: -1);

 var pipeline = new CausalLMPipeline<TiktokenTokenizer, LlamaForCausalLM>(tokenizer, model, device);

 var agent = new LlamaCausalLMAgent(pipeline, "assistant")
 .RegisterPrintMessage();

 var task = """
 Write a C# program to print the sum of two numbers. Use top-level statement, put code between ```csharp and ```.
 """;

 await agent.SendAsync(task);
 }
}

1 change: 1 addition & 0 deletions
docs/samples/Microsoft.ML.GenAI.Samples/Microsoft.ML.GenAI.Samples.csproj

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -9,6 +9,7 @@

 <ItemGroup>
 <ProjectReference Include="..\..\..\src\Microsoft.ML.GenAI.Core\Microsoft.ML.GenAI.Core.csproj" />
 <ProjectReference Include="..\..\..\src\Microsoft.ML.GenAI.LLaMA\Microsoft.ML.GenAI.LLaMA.csproj" />
 <ProjectReference Include="..\..\..\src\Microsoft.ML.GenAI.Phi\Microsoft.ML.GenAI.Phi.csproj" />
 </ItemGroup>

Expand Down

2 changes: 1 addition & 1 deletion
docs/samples/Microsoft.ML.GenAI.Samples/Phi3Mini/AutoGenSample.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,7 +26,7 @@ public static async Task RunAsync()
 torch.manual_seed(1);
 torch.set_default_dtype(defaultType);
 var weightFolder = @"C:\Users\xiaoyuz\source\repos\Phi-3-mini-4k-instruct";
 var pipeline = Utils.LoadPhi3Mini4KFromFolder(weightFolder, device: device);
 var pipeline = Utils.LoadPhi3Mini4KFromFolder(weightFolder, device: device, quantizeToInt8: false);

 // agent
 var agent = new Phi3Agent(pipeline, "assistant")
Expand Down

2 changes: 1 addition & 1 deletion
docs/samples/Microsoft.ML.GenAI.Samples/Phi3Mini/Utils.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,7 +20,7 @@ public static ICausalLMPipeline<Tokenizer, Phi3ForCasualLM> LoadPhi3Mini4KFromFo
 string weightFolder,
 string configName = "config.json",
 string device = "cuda",
 int modelSizeOnCudaInGB = 16,
Comment thread
LittleLittleCloud marked this conversation as resolved.

 int modelSizeOnCudaInGB = 55,
 int modelSizeOnMemoryInGB = 64,
 int modelSizeOnDiskInGB = 200,
 bool quantizeToInt8 = false,
Expand Down

2 changes: 1 addition & 1 deletion
eng/Versions.props

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -96,7 +96,7 @@
 <MicrosoftMLTensorFlowTestModelsVersion>0.0.13-test</MicrosoftMLTensorFlowTestModelsVersion>
 <MicrosoftMLTestDatabasesVersion>0.0.6-test</MicrosoftMLTestDatabasesVersion>
 <MicrosoftMLTestModelsVersion>0.0.7-test</MicrosoftMLTestModelsVersion>
 <MicrosoftMLTestTokenizersVersion>2.0.0-beta.24219.1</MicrosoftMLTestTokenizersVersion>
 <MicrosoftMLTestTokenizersVersion>2.0.0-beta.24415.1</MicrosoftMLTestTokenizersVersion>
 <SystemDataSqlClientVersion>4.8.6</SystemDataSqlClientVersion>
 <SystemDataSQLiteCoreVersion>1.0.118</SystemDataSQLiteCoreVersion>
 <XunitCombinatorialVersion>1.6.24</XunitCombinatorialVersion>
Expand Down

51 changes: 51 additions & 0 deletions
src/Microsoft.ML.GenAI.Core/Extension/ModuleExtension.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -197,6 +197,57 @@ public static Dictionary<string, string> InferDeviceMapForEachLayer(
 return deviceMap;
 }

 /// <summary>
 /// Infer the device map for each layer in the model.
 /// The device map is a dictionary where the key is the device id (e.g. "cuda:0") and the value is the memory size in bytes of the device.
 /// When inferring the device map, each layer in the model will be placed on the device in the order of the devices list.
 /// </summary>
 /// <param name="model"></param>
 /// <param name="numberOfLayerToBePlaced">a list of key-value pairs where the key is the device id (e.g. "cuda:0") and the value is the number of layers to be placed on the device.
 /// If you want to place all remaining layers on the device, set that value to -1.
 /// e.g. [{"cuda:0", 2}, {"cpu", -1}], the first 2 layers will be placed on "cuda:0" and the rest will be placed on "cpu".
 /// </param>
 /// <returns></returns>
 public static Dictionary<string, string> InferDeviceMapForEachLayer(
 this nn.Module model,
 IEnumerable<KeyValuePair<string, int>> numberOfLayerToBePlaced)
 {
 var layerSizeMap = model.GetSizeForEachDynamicLayerInBytes()
 .OrderByDescending(x => x.Value)
 .ToList();

 var deviceMap = new Dictionary<string, string>();
 foreach (var (device, count) in numberOfLayerToBePlaced)
 {
 if (count != -1)
 {
 var topK = layerSizeMap.Take(count).ToList();
 layerSizeMap = layerSizeMap.Skip(count).ToList();
 foreach (var (key, value) in topK)
 {
 deviceMap[key] = device;
 }
 }
 else
 {
 foreach (var (key, value) in layerSizeMap)
 {
 deviceMap[key] = device;
 }

 layerSizeMap.Clear();
 break;
 }
 }

 if (layerSizeMap.Count > 0)
 {
 throw new ArgumentException("The layer count is not enough to cover all layers, did you forget to set the last layer count to -1?");
 }

 return deviceMap;
 }

 internal static string Peek(this nn.Module model)
 {
 var sb = new StringBuilder();
Expand Down

11 changes: 4 additions & 7 deletions
src/Microsoft.ML.GenAI.Core/Microsoft.ML.GenAI.Core.csproj

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -8,23 +8,20 @@
 </PropertyGroup>

 <ItemGroup>
 <PackageReference Include="AutoGen.Core" Version="$(AutoGenVersion)" />
 <PackageReference Include="Microsoft.SemanticKernel.Abstractions" Version="$(SemanticKernelVersion)" />
 <PackageReference Include="System.Memory" Version="$(SystemMemoryVersion)" />
 <PackageReference Include="TorchSharp" Version="$(TorchSharpVersion)" />
 </ItemGroup>
<!-- 

 <ItemGroup Condition="'$(Configuration)' == 'Debug'">
 <PackageReference Include="libtorch-cpu-win-x64" Version="$(LibTorchVersion)" Condition="$([MSBuild]::IsOSPlatform('Windows'))" PrivateAssets="all" />
 <PackageReference Include="libtorch-cpu-linux-x64" Version="$(LibTorchVersion)" Condition="$([MSBuild]::IsOSPlatform('Linux'))" PrivateAssets="all" />
 <PackageReference Include="libtorch-cpu-osx-x64" Version="$(LibTorchVersion)" Condition="$([MSBuild]::IsOSPlatform('OSX'))" PrivateAssets="all" />
 </ItemGroup> -->

 <ItemGroup>
 <ProjectReference Include="..\Microsoft.ML.Tokenizers\Microsoft.ML.Tokenizers.csproj" />
 </ItemGroup>

 <ItemGroup>
 <InternalsVisibleTo Include="Microsoft.ML.GenAI.Phi" />
 <InternalsVisibleTo Include="Microsoft.ML.GenAI.LLaMA" />
 <InternalsVisibleTo Include="Microsoft.ML.GenAI.LLaMA.Tests" />
 <InternalsVisibleTo Include="Microsoft.ML.GenAI.Phi.Tests" />
 <InternalsVisibleTo Include="Microsoft.ML.GenAI.Core.Tests" />
 </ItemGroup>
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
[GenAI] Add LLaMA support#7220

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

LittleLittleCloud
merged 25 commits into

from

Aug 28, 2024
 Merged


[GenAI] Add LLaMA support
#7220
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


25 commits


Select commit
Hold shift + click to select a range
7b17731

add llama


LittleLittleCloud Aug 13, 2024
996548b

add test for tokenizer


LittleLittleCloud Aug 14, 2024
023c9f7

make llama 3.1 working


LittleLittleCloud Aug 14, 2024
63be655

update


LittleLittleCloud Aug 14, 2024
44b3302

add shape test for 70b and 405b


LittleLittleCloud Aug 15, 2024
3d4482d

clean up


LittleLittleCloud Aug 15, 2024
d3681dd

Merge branch 'u/xiaoyun/AddLLaMA' of https://github.com/LittleLittleC…


LittleLittleCloud Aug 16, 2024
c77b518

add tests


LittleLittleCloud Aug 16, 2024
6498e2e

update


LittleLittleCloud Aug 16, 2024
6557eac

fix error


LittleLittleCloud Aug 16, 2024
fd61cff

calculate rotary embedding in model layer


LittleLittleCloud Aug 16, 2024
a3f2e41

remove rotary_emb from attention


LittleLittleCloud Aug 16, 2024
9a03acc

update feed


LittleLittleCloud Aug 17, 2024
0542bf7

update .csproj


LittleLittleCloud Aug 17, 2024
485f9af

Update NuGet.config


LittleLittleCloud Aug 19, 2024
896a21b

fix test


LittleLittleCloud Aug 19, 2024
d3d5b61

pass device


LittleLittleCloud Aug 21, 2024
d161b49

fix test


LittleLittleCloud Aug 21, 2024
32b74e3

update constructor


LittleLittleCloud Aug 21, 2024
b2b155c

disable 405b test


LittleLittleCloud Aug 21, 2024
2b48080

update


LittleLittleCloud Aug 22, 2024
e0ba719

disable 70b test


LittleLittleCloud Aug 22, 2024
093a38e

use windows only fact


LittleLittleCloud Aug 26, 2024
51bdcc6

revert change


LittleLittleCloud Aug 28, 2024
19a2d7e

rename test to LLaMA3_1


LittleLittleCloud Aug 28, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


24 changes: 23 additions & 1 deletion
Microsoft.ML.sln

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -184,7 +184,11 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.Phi.Test
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.Samples", "docs\samples\Microsoft.ML.GenAI.Samples\Microsoft.ML.GenAI.Samples.csproj", "{1D4AD9A3-19AF-432B-889D-A63FE6D7BD47}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.ML.GenAI.Core.Tests", "test\Microsoft.ML.GenAI.Core.Tests\Microsoft.ML.GenAI.Core.Tests.csproj", "{14AB0804-D4CE-4634-B544-5A8587620783}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.Core.Tests", "test\Microsoft.ML.GenAI.Core.Tests\Microsoft.ML.GenAI.Core.Tests.csproj", "{14AB0804-D4CE-4634-B544-5A8587620783}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.LLaMA", "src\Microsoft.ML.GenAI.LLaMA\Microsoft.ML.GenAI.LLaMA.csproj", "{0AA6D5CB-195F-457A-8792-4221E76E6C44}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.ML.GenAI.LLaMA.Tests", "test\Microsoft.ML.GenAI.LLaMA.Tests\Microsoft.ML.GenAI.LLaMA.Tests.csproj", "{D202353D-6FAF-4263-9A01-BDCFBC92391F}"
EndProject
Global
	GlobalSection(SolutionConfigurationPlatforms) = preSolution
Expand DownExpand Up@@ -878,6 +882,22 @@ Global
 {14AB0804-D4CE-4634-B544-5A8587620783}.Release|Any CPU.Build.0 = Release|Any CPU
 {14AB0804-D4CE-4634-B544-5A8587620783}.Release|x64.ActiveCfg = Release|Any CPU
 {14AB0804-D4CE-4634-B544-5A8587620783}.Release|x64.Build.0 = Release|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Debug|Any CPU.Build.0 = Debug|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Debug|x64.ActiveCfg = Debug|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Debug|x64.Build.0 = Debug|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Release|Any CPU.ActiveCfg = Release|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Release|Any CPU.Build.0 = Release|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Release|x64.ActiveCfg = Release|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Release|x64.Build.0 = Release|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Debug|Any CPU.Build.0 = Debug|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Debug|x64.ActiveCfg = Debug|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Debug|x64.Build.0 = Debug|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Release|Any CPU.ActiveCfg = Release|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Release|Any CPU.Build.0 = Release|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Release|x64.ActiveCfg = Release|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Release|x64.Build.0 = Release|Any CPU
	EndGlobalSection
	GlobalSection(SolutionProperties) = preSolution
 HideSolutionNode = FALSE
Expand DownExpand Up@@ -969,6 +989,8 @@ Global
 {867FFC34-DFA7-400F-B9BB-85158326CE08} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
 {1D4AD9A3-19AF-432B-889D-A63FE6D7BD47} = {DA452A53-2E94-4433-B08C-041EDEC729E6}
 {14AB0804-D4CE-4634-B544-5A8587620783} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
 {0AA6D5CB-195F-457A-8792-4221E76E6C44} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
 {D202353D-6FAF-4263-9A01-BDCFBC92391F} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
	EndGlobalSection
	GlobalSection(ExtensibilityGlobals) = postSolution
 SolutionGuid = {41165AF1-35BB-4832-A189-73060F82B01D}
Expand Down

4 changes: 4 additions & 0 deletions
NuGet.config

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,7 @@
 <add key="dotnet5-roslyn" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json" />
 <add key="mlnet-daily" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/MachineLearning/nuget/v3/index.json" />
 <add key="mlnet-assets" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/machinelearning-assets/nuget/v3/index.json" />
 <add key="dotnet-libraries-transport" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-libraries-transport/nuget/v3/index.json" />
 <add key="dotnet8" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet8/nuget/v3/index.json" />
 </packageSources>
 <packageSourceMapping>
Expand DownExpand Up@@ -40,6 +41,9 @@
 <packageSource key="mlnet-assets">
 <package pattern="*" />
 </packageSource>
 <packageSource key="dotnet-libraries-transport">
 <package pattern="*" />
 </packageSource>
 <packageSource key="dotnet8">
 <package pattern="*" />
 </packageSource>
Expand Down

51 changes: 51 additions & 0 deletions
docs/samples/Microsoft.ML.GenAI.Samples/Llama/LLaMA3_1.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.Json;
using System.Threading.Tasks;
using AutoGen.Core;
using Microsoft.ML.GenAI.Core;
using Microsoft.ML.GenAI.Core.Extension;
using Microsoft.ML.GenAI.LLaMA;
using Microsoft.ML.Tokenizers;
using TorchSharp;
using static TorchSharp.torch;

namespace Microsoft.ML.GenAI.Samples.Llama;

internal class LlamaSample
{
 public static async void Run()
 {
 var device = "cuda";
 if (device == "cuda")
 {
 torch.InitializeDeviceType(DeviceType.CUDA);
 }

 var defaultType = ScalarType.Float16;
 torch.manual_seed(1);
 torch.set_default_dtype(defaultType);
 var weightFolder = @"C:\Users\xiaoyuz\source\repos\Meta-Llama-3.1-8B-Instruct";
 var configName = "config.json";
 var originalWeightFolder = Path.Combine(weightFolder, "original");

 Console.WriteLine("Loading Llama from huggingface model weight folder");
 var stopWatch = System.Diagnostics.Stopwatch.StartNew();
 stopWatch.Start();
 var tokenizer = LlamaTokenizerHelper.FromPretrained(originalWeightFolder);
 var model = LlamaForCausalLM.FromPretrained(weightFolder, configName, layersOnTargetDevice: -1);

 var pipeline = new CausalLMPipeline<TiktokenTokenizer, LlamaForCausalLM>(tokenizer, model, device);

 var agent = new LlamaCausalLMAgent(pipeline, "assistant")
 .RegisterPrintMessage();

 var task = """
 Write a C# program to print the sum of two numbers. Use top-level statement, put code between ```csharp and ```.
 """;

 await agent.SendAsync(task);
 }
}

1 change: 1 addition & 0 deletions
docs/samples/Microsoft.ML.GenAI.Samples/Microsoft.ML.GenAI.Samples.csproj

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -9,6 +9,7 @@

 <ItemGroup>
 <ProjectReference Include="..\..\..\src\Microsoft.ML.GenAI.Core\Microsoft.ML.GenAI.Core.csproj" />
 <ProjectReference Include="..\..\..\src\Microsoft.ML.GenAI.LLaMA\Microsoft.ML.GenAI.LLaMA.csproj" />
 <ProjectReference Include="..\..\..\src\Microsoft.ML.GenAI.Phi\Microsoft.ML.GenAI.Phi.csproj" />
 </ItemGroup>

Expand Down

2 changes: 1 addition & 1 deletion
docs/samples/Microsoft.ML.GenAI.Samples/Phi3Mini/AutoGenSample.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,7 +26,7 @@ public static async Task RunAsync()
 torch.manual_seed(1);
 torch.set_default_dtype(defaultType);
 var weightFolder = @"C:\Users\xiaoyuz\source\repos\Phi-3-mini-4k-instruct";
 var pipeline = Utils.LoadPhi3Mini4KFromFolder(weightFolder, device: device);
 var pipeline = Utils.LoadPhi3Mini4KFromFolder(weightFolder, device: device, quantizeToInt8: false);

 // agent
 var agent = new Phi3Agent(pipeline, "assistant")
Expand Down

2 changes: 1 addition & 1 deletion
docs/samples/Microsoft.ML.GenAI.Samples/Phi3Mini/Utils.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,7 +20,7 @@ public static ICausalLMPipeline<Tokenizer, Phi3ForCasualLM> LoadPhi3Mini4KFromFo
 string weightFolder,
 string configName = "config.json",
 string device = "cuda",
 int modelSizeOnCudaInGB = 16,
Comment thread
LittleLittleCloud marked this conversation as resolved.

 int modelSizeOnCudaInGB = 55,
 int modelSizeOnMemoryInGB = 64,
 int modelSizeOnDiskInGB = 200,
 bool quantizeToInt8 = false,
Expand Down

2 changes: 1 addition & 1 deletion
eng/Versions.props

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -96,7 +96,7 @@
 <MicrosoftMLTensorFlowTestModelsVersion>0.0.13-test</MicrosoftMLTensorFlowTestModelsVersion>
 <MicrosoftMLTestDatabasesVersion>0.0.6-test</MicrosoftMLTestDatabasesVersion>
 <MicrosoftMLTestModelsVersion>0.0.7-test</MicrosoftMLTestModelsVersion>
 <MicrosoftMLTestTokenizersVersion>2.0.0-beta.24219.1</MicrosoftMLTestTokenizersVersion>
 <MicrosoftMLTestTokenizersVersion>2.0.0-beta.24415.1</MicrosoftMLTestTokenizersVersion>
 <SystemDataSqlClientVersion>4.8.6</SystemDataSqlClientVersion>
 <SystemDataSQLiteCoreVersion>1.0.118</SystemDataSQLiteCoreVersion>
 <XunitCombinatorialVersion>1.6.24</XunitCombinatorialVersion>
Expand Down

51 changes: 51 additions & 0 deletions
src/Microsoft.ML.GenAI.Core/Extension/ModuleExtension.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -197,6 +197,57 @@ public static Dictionary<string, string> InferDeviceMapForEachLayer(
 return deviceMap;
 }

 /// <summary>
 /// Infer the device map for each layer in the model.
 /// The device map is a dictionary where the key is the device id (e.g. "cuda:0") and the value is the memory size in bytes of the device.
 /// When inferring the device map, each layer in the model will be placed on the device in the order of the devices list.
 /// </summary>
 /// <param name="model"></param>
 /// <param name="numberOfLayerToBePlaced">a list of key-value pairs where the key is the device id (e.g. "cuda:0") and the value is the number of layers to be placed on the device.
 /// If you want to place all remaining layers on the device, set that value to -1.
 /// e.g. [{"cuda:0", 2}, {"cpu", -1}], the first 2 layers will be placed on "cuda:0" and the rest will be placed on "cpu".
 /// </param>
 /// <returns></returns>
 public static Dictionary<string, string> InferDeviceMapForEachLayer(
 this nn.Module model,
 IEnumerable<KeyValuePair<string, int>> numberOfLayerToBePlaced)
 {
 var layerSizeMap = model.GetSizeForEachDynamicLayerInBytes()
 .OrderByDescending(x => x.Value)
 .ToList();

 var deviceMap = new Dictionary<string, string>();
 foreach (var (device, count) in numberOfLayerToBePlaced)
 {
 if (count != -1)
 {
 var topK = layerSizeMap.Take(count).ToList();
 layerSizeMap = layerSizeMap.Skip(count).ToList();
 foreach (var (key, value) in topK)
 {
 deviceMap[key] = device;
 }
 }
 else
 {
 foreach (var (key, value) in layerSizeMap)
 {
 deviceMap[key] = device;
 }

 layerSizeMap.Clear();
 break;
 }
 }

 if (layerSizeMap.Count > 0)
 {
 throw new ArgumentException("The layer count is not enough to cover all layers, did you forget to set the last layer count to -1?");
 }

 return deviceMap;
 }

 internal static string Peek(this nn.Module model)
 {
 var sb = new StringBuilder();
Expand Down

11 changes: 4 additions & 7 deletions
src/Microsoft.ML.GenAI.Core/Microsoft.ML.GenAI.Core.csproj

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -8,23 +8,20 @@
 </PropertyGroup>

 <ItemGroup>
 <PackageReference Include="AutoGen.Core" Version="$(AutoGenVersion)" />
 <PackageReference Include="Microsoft.SemanticKernel.Abstractions" Version="$(SemanticKernelVersion)" />
 <PackageReference Include="System.Memory" Version="$(SystemMemoryVersion)" />
 <PackageReference Include="TorchSharp" Version="$(TorchSharpVersion)" />
 </ItemGroup>
<!-- 

 <ItemGroup Condition="'$(Configuration)' == 'Debug'">
 <PackageReference Include="libtorch-cpu-win-x64" Version="$(LibTorchVersion)" Condition="$([MSBuild]::IsOSPlatform('Windows'))" PrivateAssets="all" />
 <PackageReference Include="libtorch-cpu-linux-x64" Version="$(LibTorchVersion)" Condition="$([MSBuild]::IsOSPlatform('Linux'))" PrivateAssets="all" />
 <PackageReference Include="libtorch-cpu-osx-x64" Version="$(LibTorchVersion)" Condition="$([MSBuild]::IsOSPlatform('OSX'))" PrivateAssets="all" />
 </ItemGroup> -->

 <ItemGroup>
 <ProjectReference Include="..\Microsoft.ML.Tokenizers\Microsoft.ML.Tokenizers.csproj" />
 </ItemGroup>

 <ItemGroup>
 <InternalsVisibleTo Include="Microsoft.ML.GenAI.Phi" />
 <InternalsVisibleTo Include="Microsoft.ML.GenAI.LLaMA" />
 <InternalsVisibleTo Include="Microsoft.ML.GenAI.LLaMA.Tests" />
 <InternalsVisibleTo Include="Microsoft.ML.GenAI.Phi.Tests" />
 <InternalsVisibleTo Include="Microsoft.ML.GenAI.Core.Tests" />
 </ItemGroup>
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
[GenAI] Add LLaMA support#7220

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

LittleLittleCloud
merged 25 commits into

from

Aug 28, 2024
 Merged


[GenAI] Add LLaMA support
#7220
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


25 commits


Select commit
Hold shift + click to select a range
7b17731

add llama


LittleLittleCloud Aug 13, 2024
996548b

add test for tokenizer


LittleLittleCloud Aug 14, 2024
023c9f7

make llama 3.1 working


LittleLittleCloud Aug 14, 2024
63be655

update


LittleLittleCloud Aug 14, 2024
44b3302

add shape test for 70b and 405b


LittleLittleCloud Aug 15, 2024
3d4482d

clean up


LittleLittleCloud Aug 15, 2024
d3681dd

Merge branch 'u/xiaoyun/AddLLaMA' of https://github.com/LittleLittleC…


LittleLittleCloud Aug 16, 2024
c77b518

add tests


LittleLittleCloud Aug 16, 2024
6498e2e

update


LittleLittleCloud Aug 16, 2024
6557eac

fix error


LittleLittleCloud Aug 16, 2024
fd61cff

calculate rotary embedding in model layer


LittleLittleCloud Aug 16, 2024
a3f2e41

remove rotary_emb from attention


LittleLittleCloud Aug 16, 2024
9a03acc

update feed


LittleLittleCloud Aug 17, 2024
0542bf7

update .csproj


LittleLittleCloud Aug 17, 2024
485f9af

Update NuGet.config


LittleLittleCloud Aug 19, 2024
896a21b

fix test


LittleLittleCloud Aug 19, 2024
d3d5b61

pass device


LittleLittleCloud Aug 21, 2024
d161b49

fix test


LittleLittleCloud Aug 21, 2024
32b74e3

update constructor


LittleLittleCloud Aug 21, 2024
b2b155c

disable 405b test


LittleLittleCloud Aug 21, 2024
2b48080

update


LittleLittleCloud Aug 22, 2024
e0ba719

disable 70b test


LittleLittleCloud Aug 22, 2024
093a38e

use windows only fact


LittleLittleCloud Aug 26, 2024
51bdcc6

revert change


LittleLittleCloud Aug 28, 2024
19a2d7e

rename test to LLaMA3_1


LittleLittleCloud Aug 28, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


24 changes: 23 additions & 1 deletion
Microsoft.ML.sln

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -184,7 +184,11 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.Phi.Test
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.Samples", "docs\samples\Microsoft.ML.GenAI.Samples\Microsoft.ML.GenAI.Samples.csproj", "{1D4AD9A3-19AF-432B-889D-A63FE6D7BD47}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.ML.GenAI.Core.Tests", "test\Microsoft.ML.GenAI.Core.Tests\Microsoft.ML.GenAI.Core.Tests.csproj", "{14AB0804-D4CE-4634-B544-5A8587620783}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.Core.Tests", "test\Microsoft.ML.GenAI.Core.Tests\Microsoft.ML.GenAI.Core.Tests.csproj", "{14AB0804-D4CE-4634-B544-5A8587620783}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.LLaMA", "src\Microsoft.ML.GenAI.LLaMA\Microsoft.ML.GenAI.LLaMA.csproj", "{0AA6D5CB-195F-457A-8792-4221E76E6C44}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.ML.GenAI.LLaMA.Tests", "test\Microsoft.ML.GenAI.LLaMA.Tests\Microsoft.ML.GenAI.LLaMA.Tests.csproj", "{D202353D-6FAF-4263-9A01-BDCFBC92391F}"
EndProject
Global
	GlobalSection(SolutionConfigurationPlatforms) = preSolution
Expand DownExpand Up@@ -878,6 +882,22 @@ Global
 {14AB0804-D4CE-4634-B544-5A8587620783}.Release|Any CPU.Build.0 = Release|Any CPU
 {14AB0804-D4CE-4634-B544-5A8587620783}.Release|x64.ActiveCfg = Release|Any CPU
 {14AB0804-D4CE-4634-B544-5A8587620783}.Release|x64.Build.0 = Release|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Debug|Any CPU.Build.0 = Debug|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Debug|x64.ActiveCfg = Debug|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Debug|x64.Build.0 = Debug|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Release|Any CPU.ActiveCfg = Release|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Release|Any CPU.Build.0 = Release|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Release|x64.ActiveCfg = Release|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Release|x64.Build.0 = Release|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Debug|Any CPU.Build.0 = Debug|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Debug|x64.ActiveCfg = Debug|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Debug|x64.Build.0 = Debug|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Release|Any CPU.ActiveCfg = Release|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Release|Any CPU.Build.0 = Release|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Release|x64.ActiveCfg = Release|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Release|x64.Build.0 = Release|Any CPU
	EndGlobalSection
	GlobalSection(SolutionProperties) = preSolution
 HideSolutionNode = FALSE
Expand DownExpand Up@@ -969,6 +989,8 @@ Global
 {867FFC34-DFA7-400F-B9BB-85158326CE08} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
 {1D4AD9A3-19AF-432B-889D-A63FE6D7BD47} = {DA452A53-2E94-4433-B08C-041EDEC729E6}
 {14AB0804-D4CE-4634-B544-5A8587620783} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
 {0AA6D5CB-195F-457A-8792-4221E76E6C44} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
 {D202353D-6FAF-4263-9A01-BDCFBC92391F} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
	EndGlobalSection
	GlobalSection(ExtensibilityGlobals) = postSolution
 SolutionGuid = {41165AF1-35BB-4832-A189-73060F82B01D}
Expand Down

4 changes: 4 additions & 0 deletions
NuGet.config

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,7 @@
 <add key="dotnet5-roslyn" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json" />
 <add key="mlnet-daily" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/MachineLearning/nuget/v3/index.json" />
 <add key="mlnet-assets" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/machinelearning-assets/nuget/v3/index.json" />
 <add key="dotnet-libraries-transport" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-libraries-transport/nuget/v3/index.json" />
 <add key="dotnet8" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet8/nuget/v3/index.json" />
 </packageSources>
 <packageSourceMapping>
Expand DownExpand Up@@ -40,6 +41,9 @@
 <packageSource key="mlnet-assets">
 <package pattern="*" />
 </packageSource>
 <packageSource key="dotnet-libraries-transport">
 <package pattern="*" />
 </packageSource>
 <packageSource key="dotnet8">
 <package pattern="*" />
 </packageSource>
Expand Down

51 changes: 51 additions & 0 deletions
docs/samples/Microsoft.ML.GenAI.Samples/Llama/LLaMA3_1.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.Json;
using System.Threading.Tasks;
using AutoGen.Core;
using Microsoft.ML.GenAI.Core;
using Microsoft.ML.GenAI.Core.Extension;
using Microsoft.ML.GenAI.LLaMA;
using Microsoft.ML.Tokenizers;
using TorchSharp;
using static TorchSharp.torch;

namespace Microsoft.ML.GenAI.Samples.Llama;

internal class LlamaSample
{
 public static async void Run()
 {
 var device = "cuda";
 if (device == "cuda")
 {
 torch.InitializeDeviceType(DeviceType.CUDA);
 }

 var defaultType = ScalarType.Float16;
 torch.manual_seed(1);
 torch.set_default_dtype(defaultType);
 var weightFolder = @"C:\Users\xiaoyuz\source\repos\Meta-Llama-3.1-8B-Instruct";
 var configName = "config.json";
 var originalWeightFolder = Path.Combine(weightFolder, "original");

 Console.WriteLine("Loading Llama from huggingface model weight folder");
 var stopWatch = System.Diagnostics.Stopwatch.StartNew();
 stopWatch.Start();
 var tokenizer = LlamaTokenizerHelper.FromPretrained(originalWeightFolder);
 var model = LlamaForCausalLM.FromPretrained(weightFolder, configName, layersOnTargetDevice: -1);

 var pipeline = new CausalLMPipeline<TiktokenTokenizer, LlamaForCausalLM>(tokenizer, model, device);

 var agent = new LlamaCausalLMAgent(pipeline, "assistant")
 .RegisterPrintMessage();

 var task = """
 Write a C# program to print the sum of two numbers. Use top-level statement, put code between ```csharp and ```.
 """;

 await agent.SendAsync(task);
 }
}

1 change: 1 addition & 0 deletions
docs/samples/Microsoft.ML.GenAI.Samples/Microsoft.ML.GenAI.Samples.csproj

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -9,6 +9,7 @@

 <ItemGroup>
 <ProjectReference Include="..\..\..\src\Microsoft.ML.GenAI.Core\Microsoft.ML.GenAI.Core.csproj" />
 <ProjectReference Include="..\..\..\src\Microsoft.ML.GenAI.LLaMA\Microsoft.ML.GenAI.LLaMA.csproj" />
 <ProjectReference Include="..\..\..\src\Microsoft.ML.GenAI.Phi\Microsoft.ML.GenAI.Phi.csproj" />
 </ItemGroup>

Expand Down

2 changes: 1 addition & 1 deletion
docs/samples/Microsoft.ML.GenAI.Samples/Phi3Mini/AutoGenSample.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,7 +26,7 @@ public static async Task RunAsync()
 torch.manual_seed(1);
 torch.set_default_dtype(defaultType);
 var weightFolder = @"C:\Users\xiaoyuz\source\repos\Phi-3-mini-4k-instruct";
 var pipeline = Utils.LoadPhi3Mini4KFromFolder(weightFolder, device: device);
 var pipeline = Utils.LoadPhi3Mini4KFromFolder(weightFolder, device: device, quantizeToInt8: false);

 // agent
 var agent = new Phi3Agent(pipeline, "assistant")
Expand Down

2 changes: 1 addition & 1 deletion
docs/samples/Microsoft.ML.GenAI.Samples/Phi3Mini/Utils.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,7 +20,7 @@ public static ICausalLMPipeline<Tokenizer, Phi3ForCasualLM> LoadPhi3Mini4KFromFo
 string weightFolder,
 string configName = "config.json",
 string device = "cuda",
 int modelSizeOnCudaInGB = 16,
Comment thread
LittleLittleCloud marked this conversation as resolved.

 int modelSizeOnCudaInGB = 55,
 int modelSizeOnMemoryInGB = 64,
 int modelSizeOnDiskInGB = 200,
 bool quantizeToInt8 = false,
Expand Down

2 changes: 1 addition & 1 deletion
eng/Versions.props

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -96,7 +96,7 @@
 <MicrosoftMLTensorFlowTestModelsVersion>0.0.13-test</MicrosoftMLTensorFlowTestModelsVersion>
 <MicrosoftMLTestDatabasesVersion>0.0.6-test</MicrosoftMLTestDatabasesVersion>
 <MicrosoftMLTestModelsVersion>0.0.7-test</MicrosoftMLTestModelsVersion>
 <MicrosoftMLTestTokenizersVersion>2.0.0-beta.24219.1</MicrosoftMLTestTokenizersVersion>
 <MicrosoftMLTestTokenizersVersion>2.0.0-beta.24415.1</MicrosoftMLTestTokenizersVersion>
 <SystemDataSqlClientVersion>4.8.6</SystemDataSqlClientVersion>
 <SystemDataSQLiteCoreVersion>1.0.118</SystemDataSQLiteCoreVersion>
 <XunitCombinatorialVersion>1.6.24</XunitCombinatorialVersion>
Expand Down

51 changes: 51 additions & 0 deletions
src/Microsoft.ML.GenAI.Core/Extension/ModuleExtension.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -197,6 +197,57 @@ public static Dictionary<string, string> InferDeviceMapForEachLayer(
 return deviceMap;
 }

 /// <summary>
 /// Infer the device map for each layer in the model.
 /// The device map is a dictionary where the key is the device id (e.g. "cuda:0") and the value is the memory size in bytes of the device.
 /// When inferring the device map, each layer in the model will be placed on the device in the order of the devices list.
 /// </summary>
 /// <param name="model"></param>
 /// <param name="numberOfLayerToBePlaced">a list of key-value pairs where the key is the device id (e.g. "cuda:0") and the value is the number of layers to be placed on the device.
 /// If you want to place all remaining layers on the device, set that value to -1.
 /// e.g. [{"cuda:0", 2}, {"cpu", -1}], the first 2 layers will be placed on "cuda:0" and the rest will be placed on "cpu".
 /// </param>
 /// <returns></returns>
 public static Dictionary<string, string> InferDeviceMapForEachLayer(
 this nn.Module model,
 IEnumerable<KeyValuePair<string, int>> numberOfLayerToBePlaced)
 {
 var layerSizeMap = model.GetSizeForEachDynamicLayerInBytes()
 .OrderByDescending(x => x.Value)
 .ToList();

 var deviceMap = new Dictionary<string, string>();
 foreach (var (device, count) in numberOfLayerToBePlaced)
 {
 if (count != -1)
 {
 var topK = layerSizeMap.Take(count).ToList();
 layerSizeMap = layerSizeMap.Skip(count).ToList();
 foreach (var (key, value) in topK)
 {
 deviceMap[key] = device;
 }
 }
 else
 {
 foreach (var (key, value) in layerSizeMap)
 {
 deviceMap[key] = device;
 }

 layerSizeMap.Clear();
 break;
 }
 }

 if (layerSizeMap.Count > 0)
 {
 throw new ArgumentException("The layer count is not enough to cover all layers, did you forget to set the last layer count to -1?");
 }

 return deviceMap;
 }

 internal static string Peek(this nn.Module model)
 {
 var sb = new StringBuilder();
Expand Down

11 changes: 4 additions & 7 deletions
src/Microsoft.ML.GenAI.Core/Microsoft.ML.GenAI.Core.csproj

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -8,23 +8,20 @@
 </PropertyGroup>

 <ItemGroup>
 <PackageReference Include="AutoGen.Core" Version="$(AutoGenVersion)" />
 <PackageReference Include="Microsoft.SemanticKernel.Abstractions" Version="$(SemanticKernelVersion)" />
 <PackageReference Include="System.Memory" Version="$(SystemMemoryVersion)" />
 <PackageReference Include="TorchSharp" Version="$(TorchSharpVersion)" />
 </ItemGroup>
<!-- 

 <ItemGroup Condition="'$(Configuration)' == 'Debug'">
 <PackageReference Include="libtorch-cpu-win-x64" Version="$(LibTorchVersion)" Condition="$([MSBuild]::IsOSPlatform('Windows'))" PrivateAssets="all" />
 <PackageReference Include="libtorch-cpu-linux-x64" Version="$(LibTorchVersion)" Condition="$([MSBuild]::IsOSPlatform('Linux'))" PrivateAssets="all" />
 <PackageReference Include="libtorch-cpu-osx-x64" Version="$(LibTorchVersion)" Condition="$([MSBuild]::IsOSPlatform('OSX'))" PrivateAssets="all" />
 </ItemGroup> -->

 <ItemGroup>
 <ProjectReference Include="..\Microsoft.ML.Tokenizers\Microsoft.ML.Tokenizers.csproj" />
 </ItemGroup>

 <ItemGroup>
 <InternalsVisibleTo Include="Microsoft.ML.GenAI.Phi" />
 <InternalsVisibleTo Include="Microsoft.ML.GenAI.LLaMA" />
 <InternalsVisibleTo Include="Microsoft.ML.GenAI.LLaMA.Tests" />
 <InternalsVisibleTo Include="Microsoft.ML.GenAI.Phi.Tests" />
 <InternalsVisibleTo Include="Microsoft.ML.GenAI.Core.Tests" />
 </ItemGroup>
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
[GenAI] Add LLaMA support#7220

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

LittleLittleCloud
merged 25 commits into

from

Aug 28, 2024
 Merged


[GenAI] Add LLaMA support
#7220
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


25 commits


Select commit
Hold shift + click to select a range
7b17731

add llama


LittleLittleCloud Aug 13, 2024
996548b

add test for tokenizer


LittleLittleCloud Aug 14, 2024
023c9f7

make llama 3.1 working


LittleLittleCloud Aug 14, 2024
63be655

update


LittleLittleCloud Aug 14, 2024
44b3302

add shape test for 70b and 405b


LittleLittleCloud Aug 15, 2024
3d4482d

clean up


LittleLittleCloud Aug 15, 2024
d3681dd

Merge branch 'u/xiaoyun/AddLLaMA' of https://github.com/LittleLittleC…


LittleLittleCloud Aug 16, 2024
c77b518

add tests


LittleLittleCloud Aug 16, 2024
6498e2e

update


LittleLittleCloud Aug 16, 2024
6557eac

fix error


LittleLittleCloud Aug 16, 2024
fd61cff

calculate rotary embedding in model layer


LittleLittleCloud Aug 16, 2024
a3f2e41

remove rotary_emb from attention


LittleLittleCloud Aug 16, 2024
9a03acc

update feed


LittleLittleCloud Aug 17, 2024
0542bf7

update .csproj


LittleLittleCloud Aug 17, 2024
485f9af

Update NuGet.config


LittleLittleCloud Aug 19, 2024
896a21b

fix test


LittleLittleCloud Aug 19, 2024
d3d5b61

pass device


LittleLittleCloud Aug 21, 2024
d161b49

fix test


LittleLittleCloud Aug 21, 2024
32b74e3

update constructor


LittleLittleCloud Aug 21, 2024
b2b155c

disable 405b test


LittleLittleCloud Aug 21, 2024
2b48080

update


LittleLittleCloud Aug 22, 2024
e0ba719

disable 70b test


LittleLittleCloud Aug 22, 2024
093a38e

use windows only fact


LittleLittleCloud Aug 26, 2024
51bdcc6

revert change


LittleLittleCloud Aug 28, 2024
19a2d7e

rename test to LLaMA3_1


LittleLittleCloud Aug 28, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


24 changes: 23 additions & 1 deletion
Microsoft.ML.sln

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -184,7 +184,11 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.Phi.Test
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.Samples", "docs\samples\Microsoft.ML.GenAI.Samples\Microsoft.ML.GenAI.Samples.csproj", "{1D4AD9A3-19AF-432B-889D-A63FE6D7BD47}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.ML.GenAI.Core.Tests", "test\Microsoft.ML.GenAI.Core.Tests\Microsoft.ML.GenAI.Core.Tests.csproj", "{14AB0804-D4CE-4634-B544-5A8587620783}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.Core.Tests", "test\Microsoft.ML.GenAI.Core.Tests\Microsoft.ML.GenAI.Core.Tests.csproj", "{14AB0804-D4CE-4634-B544-5A8587620783}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.LLaMA", "src\Microsoft.ML.GenAI.LLaMA\Microsoft.ML.GenAI.LLaMA.csproj", "{0AA6D5CB-195F-457A-8792-4221E76E6C44}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.ML.GenAI.LLaMA.Tests", "test\Microsoft.ML.GenAI.LLaMA.Tests\Microsoft.ML.GenAI.LLaMA.Tests.csproj", "{D202353D-6FAF-4263-9A01-BDCFBC92391F}"
EndProject
Global
	GlobalSection(SolutionConfigurationPlatforms) = preSolution
Expand DownExpand Up@@ -878,6 +882,22 @@ Global
 {14AB0804-D4CE-4634-B544-5A8587620783}.Release|Any CPU.Build.0 = Release|Any CPU
 {14AB0804-D4CE-4634-B544-5A8587620783}.Release|x64.ActiveCfg = Release|Any CPU
 {14AB0804-D4CE-4634-B544-5A8587620783}.Release|x64.Build.0 = Release|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Debug|Any CPU.Build.0 = Debug|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Debug|x64.ActiveCfg = Debug|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Debug|x64.Build.0 = Debug|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Release|Any CPU.ActiveCfg = Release|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Release|Any CPU.Build.0 = Release|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Release|x64.ActiveCfg = Release|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Release|x64.Build.0 = Release|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Debug|Any CPU.Build.0 = Debug|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Debug|x64.ActiveCfg = Debug|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Debug|x64.Build.0 = Debug|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Release|Any CPU.ActiveCfg = Release|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Release|Any CPU.Build.0 = Release|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Release|x64.ActiveCfg = Release|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Release|x64.Build.0 = Release|Any CPU
	EndGlobalSection
	GlobalSection(SolutionProperties) = preSolution
 HideSolutionNode = FALSE
Expand DownExpand Up@@ -969,6 +989,8 @@ Global
 {867FFC34-DFA7-400F-B9BB-85158326CE08} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
 {1D4AD9A3-19AF-432B-889D-A63FE6D7BD47} = {DA452A53-2E94-4433-B08C-041EDEC729E6}
 {14AB0804-D4CE-4634-B544-5A8587620783} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
 {0AA6D5CB-195F-457A-8792-4221E76E6C44} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
 {D202353D-6FAF-4263-9A01-BDCFBC92391F} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
	EndGlobalSection
	GlobalSection(ExtensibilityGlobals) = postSolution
 SolutionGuid = {41165AF1-35BB-4832-A189-73060F82B01D}
Expand Down

4 changes: 4 additions & 0 deletions
NuGet.config

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,7 @@
 <add key="dotnet5-roslyn" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json" />
 <add key="mlnet-daily" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/MachineLearning/nuget/v3/index.json" />
 <add key="mlnet-assets" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/machinelearning-assets/nuget/v3/index.json" />
 <add key="dotnet-libraries-transport" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-libraries-transport/nuget/v3/index.json" />
 <add key="dotnet8" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet8/nuget/v3/index.json" />
 </packageSources>
 <packageSourceMapping>
Expand DownExpand Up@@ -40,6 +41,9 @@
 <packageSource key="mlnet-assets">
 <package pattern="*" />
 </packageSource>
 <packageSource key="dotnet-libraries-transport">
 <package pattern="*" />
 </packageSource>
 <packageSource key="dotnet8">
 <package pattern="*" />
 </packageSource>
Expand Down

51 changes: 51 additions & 0 deletions
docs/samples/Microsoft.ML.GenAI.Samples/Llama/LLaMA3_1.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.Json;
using System.Threading.Tasks;
using AutoGen.Core;
using Microsoft.ML.GenAI.Core;
using Microsoft.ML.GenAI.Core.Extension;
using Microsoft.ML.GenAI.LLaMA;
using Microsoft.ML.Tokenizers;
using TorchSharp;
using static TorchSharp.torch;

namespace Microsoft.ML.GenAI.Samples.Llama;

internal class LlamaSample
{
 public static async void Run()
 {
 var device = "cuda";
 if (device == "cuda")
 {
 torch.InitializeDeviceType(DeviceType.CUDA);
 }

 var defaultType = ScalarType.Float16;
 torch.manual_seed(1);
 torch.set_default_dtype(defaultType);
 var weightFolder = @"C:\Users\xiaoyuz\source\repos\Meta-Llama-3.1-8B-Instruct";
 var configName = "config.json";
 var originalWeightFolder = Path.Combine(weightFolder, "original");

 Console.WriteLine("Loading Llama from huggingface model weight folder");
 var stopWatch = System.Diagnostics.Stopwatch.StartNew();
 stopWatch.Start();
 var tokenizer = LlamaTokenizerHelper.FromPretrained(originalWeightFolder);
 var model = LlamaForCausalLM.FromPretrained(weightFolder, configName, layersOnTargetDevice: -1);

 var pipeline = new CausalLMPipeline<TiktokenTokenizer, LlamaForCausalLM>(tokenizer, model, device);

 var agent = new LlamaCausalLMAgent(pipeline, "assistant")
 .RegisterPrintMessage();

 var task = """
 Write a C# program to print the sum of two numbers. Use top-level statement, put code between ```csharp and ```.
 """;

 await agent.SendAsync(task);
 }
}

1 change: 1 addition & 0 deletions
docs/samples/Microsoft.ML.GenAI.Samples/Microsoft.ML.GenAI.Samples.csproj

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -9,6 +9,7 @@

 <ItemGroup>
 <ProjectReference Include="..\..\..\src\Microsoft.ML.GenAI.Core\Microsoft.ML.GenAI.Core.csproj" />
 <ProjectReference Include="..\..\..\src\Microsoft.ML.GenAI.LLaMA\Microsoft.ML.GenAI.LLaMA.csproj" />
 <ProjectReference Include="..\..\..\src\Microsoft.ML.GenAI.Phi\Microsoft.ML.GenAI.Phi.csproj" />
 </ItemGroup>

Expand Down

2 changes: 1 addition & 1 deletion
docs/samples/Microsoft.ML.GenAI.Samples/Phi3Mini/AutoGenSample.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,7 +26,7 @@ public static async Task RunAsync()
 torch.manual_seed(1);
 torch.set_default_dtype(defaultType);
 var weightFolder = @"C:\Users\xiaoyuz\source\repos\Phi-3-mini-4k-instruct";
 var pipeline = Utils.LoadPhi3Mini4KFromFolder(weightFolder, device: device);
 var pipeline = Utils.LoadPhi3Mini4KFromFolder(weightFolder, device: device, quantizeToInt8: false);

 // agent
 var agent = new Phi3Agent(pipeline, "assistant")
Expand Down

2 changes: 1 addition & 1 deletion
docs/samples/Microsoft.ML.GenAI.Samples/Phi3Mini/Utils.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,7 +20,7 @@ public static ICausalLMPipeline<Tokenizer, Phi3ForCasualLM> LoadPhi3Mini4KFromFo
 string weightFolder,
 string configName = "config.json",
 string device = "cuda",
 int modelSizeOnCudaInGB = 16,
Comment thread
LittleLittleCloud marked this conversation as resolved.

 int modelSizeOnCudaInGB = 55,
 int modelSizeOnMemoryInGB = 64,
 int modelSizeOnDiskInGB = 200,
 bool quantizeToInt8 = false,
Expand Down

2 changes: 1 addition & 1 deletion
eng/Versions.props

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -96,7 +96,7 @@
 <MicrosoftMLTensorFlowTestModelsVersion>0.0.13-test</MicrosoftMLTensorFlowTestModelsVersion>
 <MicrosoftMLTestDatabasesVersion>0.0.6-test</MicrosoftMLTestDatabasesVersion>
 <MicrosoftMLTestModelsVersion>0.0.7-test</MicrosoftMLTestModelsVersion>
 <MicrosoftMLTestTokenizersVersion>2.0.0-beta.24219.1</MicrosoftMLTestTokenizersVersion>
 <MicrosoftMLTestTokenizersVersion>2.0.0-beta.24415.1</MicrosoftMLTestTokenizersVersion>
 <SystemDataSqlClientVersion>4.8.6</SystemDataSqlClientVersion>
 <SystemDataSQLiteCoreVersion>1.0.118</SystemDataSQLiteCoreVersion>
 <XunitCombinatorialVersion>1.6.24</XunitCombinatorialVersion>
Expand Down

51 changes: 51 additions & 0 deletions
src/Microsoft.ML.GenAI.Core/Extension/ModuleExtension.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -197,6 +197,57 @@ public static Dictionary<string, string> InferDeviceMapForEachLayer(
 return deviceMap;
 }

 /// <summary>
 /// Infer the device map for each layer in the model.
 /// The device map is a dictionary where the key is the device id (e.g. "cuda:0") and the value is the memory size in bytes of the device.
 /// When inferring the device map, each layer in the model will be placed on the device in the order of the devices list.
 /// </summary>
 /// <param name="model"></param>
 /// <param name="numberOfLayerToBePlaced">a list of key-value pairs where the key is the device id (e.g. "cuda:0") and the value is the number of layers to be placed on the device.
 /// If you want to place all remaining layers on the device, set that value to -1.
 /// e.g. [{"cuda:0", 2}, {"cpu", -1}], the first 2 layers will be placed on "cuda:0" and the rest will be placed on "cpu".
 /// </param>
 /// <returns></returns>
 public static Dictionary<string, string> InferDeviceMapForEachLayer(
 this nn.Module model,
 IEnumerable<KeyValuePair<string, int>> numberOfLayerToBePlaced)
 {
 var layerSizeMap = model.GetSizeForEachDynamicLayerInBytes()
 .OrderByDescending(x => x.Value)
 .ToList();

 var deviceMap = new Dictionary<string, string>();
 foreach (var (device, count) in numberOfLayerToBePlaced)
 {
 if (count != -1)
 {
 var topK = layerSizeMap.Take(count).ToList();
 layerSizeMap = layerSizeMap.Skip(count).ToList();
 foreach (var (key, value) in topK)
 {
 deviceMap[key] = device;
 }
 }
 else
 {
 foreach (var (key, value) in layerSizeMap)
 {
 deviceMap[key] = device;
 }

 layerSizeMap.Clear();
 break;
 }
 }

 if (layerSizeMap.Count > 0)
 {
 throw new ArgumentException("The layer count is not enough to cover all layers, did you forget to set the last layer count to -1?");
 }

 return deviceMap;
 }

 internal static string Peek(this nn.Module model)
 {
 var sb = new StringBuilder();
Expand Down

11 changes: 4 additions & 7 deletions
src/Microsoft.ML.GenAI.Core/Microsoft.ML.GenAI.Core.csproj

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -8,23 +8,20 @@
 </PropertyGroup>

 <ItemGroup>
 <PackageReference Include="AutoGen.Core" Version="$(AutoGenVersion)" />
 <PackageReference Include="Microsoft.SemanticKernel.Abstractions" Version="$(SemanticKernelVersion)" />
 <PackageReference Include="System.Memory" Version="$(SystemMemoryVersion)" />
 <PackageReference Include="TorchSharp" Version="$(TorchSharpVersion)" />
 </ItemGroup>
<!-- 

 <ItemGroup Condition="'$(Configuration)' == 'Debug'">
 <PackageReference Include="libtorch-cpu-win-x64" Version="$(LibTorchVersion)" Condition="$([MSBuild]::IsOSPlatform('Windows'))" PrivateAssets="all" />
 <PackageReference Include="libtorch-cpu-linux-x64" Version="$(LibTorchVersion)" Condition="$([MSBuild]::IsOSPlatform('Linux'))" PrivateAssets="all" />
 <PackageReference Include="libtorch-cpu-osx-x64" Version="$(LibTorchVersion)" Condition="$([MSBuild]::IsOSPlatform('OSX'))" PrivateAssets="all" />
 </ItemGroup> -->

 <ItemGroup>
 <ProjectReference Include="..\Microsoft.ML.Tokenizers\Microsoft.ML.Tokenizers.csproj" />
 </ItemGroup>

 <ItemGroup>
 <InternalsVisibleTo Include="Microsoft.ML.GenAI.Phi" />
 <InternalsVisibleTo Include="Microsoft.ML.GenAI.LLaMA" />
 <InternalsVisibleTo Include="Microsoft.ML.GenAI.LLaMA.Tests" />
 <InternalsVisibleTo Include="Microsoft.ML.GenAI.Phi.Tests" />
 <InternalsVisibleTo Include="Microsoft.ML.GenAI.Core.Tests" />
 </ItemGroup>
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
[GenAI] Add LLaMA support#7220

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

LittleLittleCloud
merged 25 commits into

from

Aug 28, 2024
 Merged


[GenAI] Add LLaMA support
#7220
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


25 commits


Select commit
Hold shift + click to select a range
7b17731

add llama


LittleLittleCloud Aug 13, 2024
996548b

add test for tokenizer


LittleLittleCloud Aug 14, 2024
023c9f7

make llama 3.1 working


LittleLittleCloud Aug 14, 2024
63be655

update


LittleLittleCloud Aug 14, 2024
44b3302

add shape test for 70b and 405b


LittleLittleCloud Aug 15, 2024
3d4482d

clean up


LittleLittleCloud Aug 15, 2024
d3681dd

Merge branch 'u/xiaoyun/AddLLaMA' of https://github.com/LittleLittleC…


LittleLittleCloud Aug 16, 2024
c77b518

add tests


LittleLittleCloud Aug 16, 2024
6498e2e

update


LittleLittleCloud Aug 16, 2024
6557eac

fix error


LittleLittleCloud Aug 16, 2024
fd61cff

calculate rotary embedding in model layer


LittleLittleCloud Aug 16, 2024
a3f2e41

remove rotary_emb from attention


LittleLittleCloud Aug 16, 2024
9a03acc

update feed


LittleLittleCloud Aug 17, 2024
0542bf7

update .csproj


LittleLittleCloud Aug 17, 2024
485f9af

Update NuGet.config


LittleLittleCloud Aug 19, 2024
896a21b

fix test


LittleLittleCloud Aug 19, 2024
d3d5b61

pass device


LittleLittleCloud Aug 21, 2024
d161b49

fix test


LittleLittleCloud Aug 21, 2024
32b74e3

update constructor


LittleLittleCloud Aug 21, 2024
b2b155c

disable 405b test


LittleLittleCloud Aug 21, 2024
2b48080

update


LittleLittleCloud Aug 22, 2024
e0ba719

disable 70b test


LittleLittleCloud Aug 22, 2024
093a38e

use windows only fact


LittleLittleCloud Aug 26, 2024
51bdcc6

revert change


LittleLittleCloud Aug 28, 2024
19a2d7e

rename test to LLaMA3_1


LittleLittleCloud Aug 28, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


24 changes: 23 additions & 1 deletion
Microsoft.ML.sln

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -184,7 +184,11 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.Phi.Test
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.Samples", "docs\samples\Microsoft.ML.GenAI.Samples\Microsoft.ML.GenAI.Samples.csproj", "{1D4AD9A3-19AF-432B-889D-A63FE6D7BD47}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.ML.GenAI.Core.Tests", "test\Microsoft.ML.GenAI.Core.Tests\Microsoft.ML.GenAI.Core.Tests.csproj", "{14AB0804-D4CE-4634-B544-5A8587620783}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.Core.Tests", "test\Microsoft.ML.GenAI.Core.Tests\Microsoft.ML.GenAI.Core.Tests.csproj", "{14AB0804-D4CE-4634-B544-5A8587620783}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.LLaMA", "src\Microsoft.ML.GenAI.LLaMA\Microsoft.ML.GenAI.LLaMA.csproj", "{0AA6D5CB-195F-457A-8792-4221E76E6C44}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.ML.GenAI.LLaMA.Tests", "test\Microsoft.ML.GenAI.LLaMA.Tests\Microsoft.ML.GenAI.LLaMA.Tests.csproj", "{D202353D-6FAF-4263-9A01-BDCFBC92391F}"
EndProject
Global
	GlobalSection(SolutionConfigurationPlatforms) = preSolution
Expand DownExpand Up@@ -878,6 +882,22 @@ Global
 {14AB0804-D4CE-4634-B544-5A8587620783}.Release|Any CPU.Build.0 = Release|Any CPU
 {14AB0804-D4CE-4634-B544-5A8587620783}.Release|x64.ActiveCfg = Release|Any CPU
 {14AB0804-D4CE-4634-B544-5A8587620783}.Release|x64.Build.0 = Release|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Debug|Any CPU.Build.0 = Debug|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Debug|x64.ActiveCfg = Debug|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Debug|x64.Build.0 = Debug|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Release|Any CPU.ActiveCfg = Release|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Release|Any CPU.Build.0 = Release|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Release|x64.ActiveCfg = Release|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Release|x64.Build.0 = Release|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Debug|Any CPU.Build.0 = Debug|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Debug|x64.ActiveCfg = Debug|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Debug|x64.Build.0 = Debug|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Release|Any CPU.ActiveCfg = Release|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Release|Any CPU.Build.0 = Release|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Release|x64.ActiveCfg = Release|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Release|x64.Build.0 = Release|Any CPU
	EndGlobalSection
	GlobalSection(SolutionProperties) = preSolution
 HideSolutionNode = FALSE
Expand DownExpand Up@@ -969,6 +989,8 @@ Global
 {867FFC34-DFA7-400F-B9BB-85158326CE08} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
 {1D4AD9A3-19AF-432B-889D-A63FE6D7BD47} = {DA452A53-2E94-4433-B08C-041EDEC729E6}
 {14AB0804-D4CE-4634-B544-5A8587620783} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
 {0AA6D5CB-195F-457A-8792-4221E76E6C44} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
 {D202353D-6FAF-4263-9A01-BDCFBC92391F} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
	EndGlobalSection
	GlobalSection(ExtensibilityGlobals) = postSolution
 SolutionGuid = {41165AF1-35BB-4832-A189-73060F82B01D}
Expand Down

4 changes: 4 additions & 0 deletions
NuGet.config

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,7 @@
 <add key="dotnet5-roslyn" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json" />
 <add key="mlnet-daily" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/MachineLearning/nuget/v3/index.json" />
 <add key="mlnet-assets" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/machinelearning-assets/nuget/v3/index.json" />
 <add key="dotnet-libraries-transport" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-libraries-transport/nuget/v3/index.json" />
 <add key="dotnet8" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet8/nuget/v3/index.json" />
 </packageSources>
 <packageSourceMapping>
Expand DownExpand Up@@ -40,6 +41,9 @@
 <packageSource key="mlnet-assets">
 <package pattern="*" />
 </packageSource>
 <packageSource key="dotnet-libraries-transport">
 <package pattern="*" />
 </packageSource>
 <packageSource key="dotnet8">
 <package pattern="*" />
 </packageSource>
Expand Down

51 changes: 51 additions & 0 deletions
docs/samples/Microsoft.ML.GenAI.Samples/Llama/LLaMA3_1.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.Json;
using System.Threading.Tasks;
using AutoGen.Core;
using Microsoft.ML.GenAI.Core;
using Microsoft.ML.GenAI.Core.Extension;
using Microsoft.ML.GenAI.LLaMA;
using Microsoft.ML.Tokenizers;
using TorchSharp;
using static TorchSharp.torch;

namespace Microsoft.ML.GenAI.Samples.Llama;

internal class LlamaSample
{
 public static async void Run()
 {
 var device = "cuda";
 if (device == "cuda")
 {
 torch.InitializeDeviceType(DeviceType.CUDA);
 }

 var defaultType = ScalarType.Float16;
 torch.manual_seed(1);
 torch.set_default_dtype(defaultType);
 var weightFolder = @"C:\Users\xiaoyuz\source\repos\Meta-Llama-3.1-8B-Instruct";
 var configName = "config.json";
 var originalWeightFolder = Path.Combine(weightFolder, "original");

 Console.WriteLine("Loading Llama from huggingface model weight folder");
 var stopWatch = System.Diagnostics.Stopwatch.StartNew();
 stopWatch.Start();
 var tokenizer = LlamaTokenizerHelper.FromPretrained(originalWeightFolder);
 var model = LlamaForCausalLM.FromPretrained(weightFolder, configName, layersOnTargetDevice: -1);

 var pipeline = new CausalLMPipeline<TiktokenTokenizer, LlamaForCausalLM>(tokenizer, model, device);

 var agent = new LlamaCausalLMAgent(pipeline, "assistant")
 .RegisterPrintMessage();

 var task = """
 Write a C# program to print the sum of two numbers. Use top-level statement, put code between ```csharp and ```.
 """;

 await agent.SendAsync(task);
 }
}

1 change: 1 addition & 0 deletions
docs/samples/Microsoft.ML.GenAI.Samples/Microsoft.ML.GenAI.Samples.csproj

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -9,6 +9,7 @@

 <ItemGroup>
 <ProjectReference Include="..\..\..\src\Microsoft.ML.GenAI.Core\Microsoft.ML.GenAI.Core.csproj" />
 <ProjectReference Include="..\..\..\src\Microsoft.ML.GenAI.LLaMA\Microsoft.ML.GenAI.LLaMA.csproj" />
 <ProjectReference Include="..\..\..\src\Microsoft.ML.GenAI.Phi\Microsoft.ML.GenAI.Phi.csproj" />
 </ItemGroup>

Expand Down

2 changes: 1 addition & 1 deletion
docs/samples/Microsoft.ML.GenAI.Samples/Phi3Mini/AutoGenSample.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,7 +26,7 @@ public static async Task RunAsync()
 torch.manual_seed(1);
 torch.set_default_dtype(defaultType);
 var weightFolder = @"C:\Users\xiaoyuz\source\repos\Phi-3-mini-4k-instruct";
 var pipeline = Utils.LoadPhi3Mini4KFromFolder(weightFolder, device: device);
 var pipeline = Utils.LoadPhi3Mini4KFromFolder(weightFolder, device: device, quantizeToInt8: false);

 // agent
 var agent = new Phi3Agent(pipeline, "assistant")
Expand Down

2 changes: 1 addition & 1 deletion
docs/samples/Microsoft.ML.GenAI.Samples/Phi3Mini/Utils.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,7 +20,7 @@ public static ICausalLMPipeline<Tokenizer, Phi3ForCasualLM> LoadPhi3Mini4KFromFo
 string weightFolder,
 string configName = "config.json",
 string device = "cuda",
 int modelSizeOnCudaInGB = 16,
Comment thread
LittleLittleCloud marked this conversation as resolved.

 int modelSizeOnCudaInGB = 55,
 int modelSizeOnMemoryInGB = 64,
 int modelSizeOnDiskInGB = 200,
 bool quantizeToInt8 = false,
Expand Down

2 changes: 1 addition & 1 deletion
eng/Versions.props

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -96,7 +96,7 @@
 <MicrosoftMLTensorFlowTestModelsVersion>0.0.13-test</MicrosoftMLTensorFlowTestModelsVersion>
 <MicrosoftMLTestDatabasesVersion>0.0.6-test</MicrosoftMLTestDatabasesVersion>
 <MicrosoftMLTestModelsVersion>0.0.7-test</MicrosoftMLTestModelsVersion>
 <MicrosoftMLTestTokenizersVersion>2.0.0-beta.24219.1</MicrosoftMLTestTokenizersVersion>
 <MicrosoftMLTestTokenizersVersion>2.0.0-beta.24415.1</MicrosoftMLTestTokenizersVersion>
 <SystemDataSqlClientVersion>4.8.6</SystemDataSqlClientVersion>
 <SystemDataSQLiteCoreVersion>1.0.118</SystemDataSQLiteCoreVersion>
 <XunitCombinatorialVersion>1.6.24</XunitCombinatorialVersion>
Expand Down

51 changes: 51 additions & 0 deletions
src/Microsoft.ML.GenAI.Core/Extension/ModuleExtension.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -197,6 +197,57 @@ public static Dictionary<string, string> InferDeviceMapForEachLayer(
 return deviceMap;
 }

 /// <summary>
 /// Infer the device map for each layer in the model.
 /// The device map is a dictionary where the key is the device id (e.g. "cuda:0") and the value is the memory size in bytes of the device.
 /// When inferring the device map, each layer in the model will be placed on the device in the order of the devices list.
 /// </summary>
 /// <param name="model"></param>
 /// <param name="numberOfLayerToBePlaced">a list of key-value pairs where the key is the device id (e.g. "cuda:0") and the value is the number of layers to be placed on the device.
 /// If you want to place all remaining layers on the device, set that value to -1.
 /// e.g. [{"cuda:0", 2}, {"cpu", -1}], the first 2 layers will be placed on "cuda:0" and the rest will be placed on "cpu".
 /// </param>
 /// <returns></returns>
 public static Dictionary<string, string> InferDeviceMapForEachLayer(
 this nn.Module model,
 IEnumerable<KeyValuePair<string, int>> numberOfLayerToBePlaced)
 {
 var layerSizeMap = model.GetSizeForEachDynamicLayerInBytes()
 .OrderByDescending(x => x.Value)
 .ToList();

 var deviceMap = new Dictionary<string, string>();
 foreach (var (device, count) in numberOfLayerToBePlaced)
 {
 if (count != -1)
 {
 var topK = layerSizeMap.Take(count).ToList();
 layerSizeMap = layerSizeMap.Skip(count).ToList();
 foreach (var (key, value) in topK)
 {
 deviceMap[key] = device;
 }
 }
 else
 {
 foreach (var (key, value) in layerSizeMap)
 {
 deviceMap[key] = device;
 }

 layerSizeMap.Clear();
 break;
 }
 }

 if (layerSizeMap.Count > 0)
 {
 throw new ArgumentException("The layer count is not enough to cover all layers, did you forget to set the last layer count to -1?");
 }

 return deviceMap;
 }

 internal static string Peek(this nn.Module model)
 {
 var sb = new StringBuilder();
Expand Down

11 changes: 4 additions & 7 deletions
src/Microsoft.ML.GenAI.Core/Microsoft.ML.GenAI.Core.csproj

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -8,23 +8,20 @@
 </PropertyGroup>

 <ItemGroup>
 <PackageReference Include="AutoGen.Core" Version="$(AutoGenVersion)" />
 <PackageReference Include="Microsoft.SemanticKernel.Abstractions" Version="$(SemanticKernelVersion)" />
 <PackageReference Include="System.Memory" Version="$(SystemMemoryVersion)" />
 <PackageReference Include="TorchSharp" Version="$(TorchSharpVersion)" />
 </ItemGroup>
<!-- 

 <ItemGroup Condition="'$(Configuration)' == 'Debug'">
 <PackageReference Include="libtorch-cpu-win-x64" Version="$(LibTorchVersion)" Condition="$([MSBuild]::IsOSPlatform('Windows'))" PrivateAssets="all" />
 <PackageReference Include="libtorch-cpu-linux-x64" Version="$(LibTorchVersion)" Condition="$([MSBuild]::IsOSPlatform('Linux'))" PrivateAssets="all" />
 <PackageReference Include="libtorch-cpu-osx-x64" Version="$(LibTorchVersion)" Condition="$([MSBuild]::IsOSPlatform('OSX'))" PrivateAssets="all" />
 </ItemGroup> -->

 <ItemGroup>
 <ProjectReference Include="..\Microsoft.ML.Tokenizers\Microsoft.ML.Tokenizers.csproj" />
 </ItemGroup>

 <ItemGroup>
 <InternalsVisibleTo Include="Microsoft.ML.GenAI.Phi" />
 <InternalsVisibleTo Include="Microsoft.ML.GenAI.LLaMA" />
 <InternalsVisibleTo Include="Microsoft.ML.GenAI.LLaMA.Tests" />
 <InternalsVisibleTo Include="Microsoft.ML.GenAI.Phi.Tests" />
 <InternalsVisibleTo Include="Microsoft.ML.GenAI.Core.Tests" />
 </ItemGroup>
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
[GenAI] Add LLaMA support#7220

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

LittleLittleCloud
merged 25 commits into

from

Aug 28, 2024
 Merged


[GenAI] Add LLaMA support
#7220
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


25 commits


Select commit
Hold shift + click to select a range
7b17731

add llama


LittleLittleCloud Aug 13, 2024
996548b

add test for tokenizer


LittleLittleCloud Aug 14, 2024
023c9f7

make llama 3.1 working


LittleLittleCloud Aug 14, 2024
63be655

update


LittleLittleCloud Aug 14, 2024
44b3302

add shape test for 70b and 405b


LittleLittleCloud Aug 15, 2024
3d4482d

clean up


LittleLittleCloud Aug 15, 2024
d3681dd

Merge branch 'u/xiaoyun/AddLLaMA' of https://github.com/LittleLittleC…


LittleLittleCloud Aug 16, 2024
c77b518

add tests


LittleLittleCloud Aug 16, 2024
6498e2e

update


LittleLittleCloud Aug 16, 2024
6557eac

fix error


LittleLittleCloud Aug 16, 2024
fd61cff

calculate rotary embedding in model layer


LittleLittleCloud Aug 16, 2024
a3f2e41

remove rotary_emb from attention


LittleLittleCloud Aug 16, 2024
9a03acc

update feed


LittleLittleCloud Aug 17, 2024
0542bf7

update .csproj


LittleLittleCloud Aug 17, 2024
485f9af

Update NuGet.config


LittleLittleCloud Aug 19, 2024
896a21b

fix test


LittleLittleCloud Aug 19, 2024
d3d5b61

pass device


LittleLittleCloud Aug 21, 2024
d161b49

fix test


LittleLittleCloud Aug 21, 2024
32b74e3

update constructor


LittleLittleCloud Aug 21, 2024
b2b155c

disable 405b test


LittleLittleCloud Aug 21, 2024
2b48080

update


LittleLittleCloud Aug 22, 2024
e0ba719

disable 70b test


LittleLittleCloud Aug 22, 2024
093a38e

use windows only fact


LittleLittleCloud Aug 26, 2024
51bdcc6

revert change


LittleLittleCloud Aug 28, 2024
19a2d7e

rename test to LLaMA3_1


LittleLittleCloud Aug 28, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


24 changes: 23 additions & 1 deletion
Microsoft.ML.sln

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -184,7 +184,11 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.Phi.Test
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.Samples", "docs\samples\Microsoft.ML.GenAI.Samples\Microsoft.ML.GenAI.Samples.csproj", "{1D4AD9A3-19AF-432B-889D-A63FE6D7BD47}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.ML.GenAI.Core.Tests", "test\Microsoft.ML.GenAI.Core.Tests\Microsoft.ML.GenAI.Core.Tests.csproj", "{14AB0804-D4CE-4634-B544-5A8587620783}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.Core.Tests", "test\Microsoft.ML.GenAI.Core.Tests\Microsoft.ML.GenAI.Core.Tests.csproj", "{14AB0804-D4CE-4634-B544-5A8587620783}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.LLaMA", "src\Microsoft.ML.GenAI.LLaMA\Microsoft.ML.GenAI.LLaMA.csproj", "{0AA6D5CB-195F-457A-8792-4221E76E6C44}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.ML.GenAI.LLaMA.Tests", "test\Microsoft.ML.GenAI.LLaMA.Tests\Microsoft.ML.GenAI.LLaMA.Tests.csproj", "{D202353D-6FAF-4263-9A01-BDCFBC92391F}"
EndProject
Global
	GlobalSection(SolutionConfigurationPlatforms) = preSolution
Expand DownExpand Up@@ -878,6 +882,22 @@ Global
 {14AB0804-D4CE-4634-B544-5A8587620783}.Release|Any CPU.Build.0 = Release|Any CPU
 {14AB0804-D4CE-4634-B544-5A8587620783}.Release|x64.ActiveCfg = Release|Any CPU
 {14AB0804-D4CE-4634-B544-5A8587620783}.Release|x64.Build.0 = Release|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Debug|Any CPU.Build.0 = Debug|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Debug|x64.ActiveCfg = Debug|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Debug|x64.Build.0 = Debug|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Release|Any CPU.ActiveCfg = Release|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Release|Any CPU.Build.0 = Release|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Release|x64.ActiveCfg = Release|Any CPU
 {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Release|x64.Build.0 = Release|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Debug|Any CPU.Build.0 = Debug|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Debug|x64.ActiveCfg = Debug|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Debug|x64.Build.0 = Debug|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Release|Any CPU.ActiveCfg = Release|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Release|Any CPU.Build.0 = Release|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Release|x64.ActiveCfg = Release|Any CPU
 {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Release|x64.Build.0 = Release|Any CPU
	EndGlobalSection
	GlobalSection(SolutionProperties) = preSolution
 HideSolutionNode = FALSE
Expand DownExpand Up@@ -969,6 +989,8 @@ Global
 {867FFC34-DFA7-400F-B9BB-85158326CE08} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
 {1D4AD9A3-19AF-432B-889D-A63FE6D7BD47} = {DA452A53-2E94-4433-B08C-041EDEC729E6}
 {14AB0804-D4CE-4634-B544-5A8587620783} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
 {0AA6D5CB-195F-457A-8792-4221E76E6C44} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
 {D202353D-6FAF-4263-9A01-BDCFBC92391F} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
	EndGlobalSection
	GlobalSection(ExtensibilityGlobals) = postSolution
 SolutionGuid = {41165AF1-35BB-4832-A189-73060F82B01D}
Expand Down

4 changes: 4 additions & 0 deletions
NuGet.config

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,7 @@
 <add key="dotnet5-roslyn" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json" />
 <add key="mlnet-daily" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/MachineLearning/nuget/v3/index.json" />
 <add key="mlnet-assets" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/machinelearning-assets/nuget/v3/index.json" />
 <add key="dotnet-libraries-transport" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-libraries-transport/nuget/v3/index.json" />
 <add key="dotnet8" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet8/nuget/v3/index.json" />
 </packageSources>
 <packageSourceMapping>
Expand DownExpand Up@@ -40,6 +41,9 @@
 <packageSource key="mlnet-assets">
 <package pattern="*" />
 </packageSource>
 <packageSource key="dotnet-libraries-transport">
 <package pattern="*" />
 </packageSource>
 <packageSource key="dotnet8">
 <package pattern="*" />
 </packageSource>
Expand Down

51 changes: 51 additions & 0 deletions
docs/samples/Microsoft.ML.GenAI.Samples/Llama/LLaMA3_1.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.Json;
using System.Threading.Tasks;
using AutoGen.Core;
using Microsoft.ML.GenAI.Core;
using Microsoft.ML.GenAI.Core.Extension;
using Microsoft.ML.GenAI.LLaMA;
using Microsoft.ML.Tokenizers;
using TorchSharp;
using static TorchSharp.torch;

namespace Microsoft.ML.GenAI.Samples.Llama;

internal class LlamaSample
{
 public static async void Run()
 {
 var device = "cuda";
 if (device == "cuda")
 {
 torch.InitializeDeviceType(DeviceType.CUDA);
 }

 var defaultType = ScalarType.Float16;
 torch.manual_seed(1);
 torch.set_default_dtype(defaultType);
 var weightFolder = @"C:\Users\xiaoyuz\source\repos\Meta-Llama-3.1-8B-Instruct";
 var configName = "config.json";
 var originalWeightFolder = Path.Combine(weightFolder, "original");

 Console.WriteLine("Loading Llama from huggingface model weight folder");
 var stopWatch = System.Diagnostics.Stopwatch.StartNew();
 stopWatch.Start();
 var tokenizer = LlamaTokenizerHelper.FromPretrained(originalWeightFolder);
 var model = LlamaForCausalLM.FromPretrained(weightFolder, configName, layersOnTargetDevice: -1);

 var pipeline = new CausalLMPipeline<TiktokenTokenizer, LlamaForCausalLM>(tokenizer, model, device);

 var agent = new LlamaCausalLMAgent(pipeline, "assistant")
 .RegisterPrintMessage();

 var task = """
 Write a C# program to print the sum of two numbers. Use top-level statement, put code between ```csharp and ```.
 """;

 await agent.SendAsync(task);
 }
}

1 change: 1 addition & 0 deletions
docs/samples/Microsoft.ML.GenAI.Samples/Microsoft.ML.GenAI.Samples.csproj

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -9,6 +9,7 @@

 <ItemGroup>
 <ProjectReference Include="..\..\..\src\Microsoft.ML.GenAI.Core\Microsoft.ML.GenAI.Core.csproj" />
 <ProjectReference Include="..\..\..\src\Microsoft.ML.GenAI.LLaMA\Microsoft.ML.GenAI.LLaMA.csproj" />
 <ProjectReference Include="..\..\..\src\Microsoft.ML.GenAI.Phi\Microsoft.ML.GenAI.Phi.csproj" />
 </ItemGroup>

Expand Down

2 changes: 1 addition & 1 deletion
docs/samples/Microsoft.ML.GenAI.Samples/Phi3Mini/AutoGenSample.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,7 +26,7 @@ public static async Task RunAsync()
 torch.manual_seed(1);
 torch.set_default_dtype(defaultType);
 var weightFolder = @"C:\Users\xiaoyuz\source\repos\Phi-3-mini-4k-instruct";
 var pipeline = Utils.LoadPhi3Mini4KFromFolder(weightFolder, device: device);
 var pipeline = Utils.LoadPhi3Mini4KFromFolder(weightFolder, device: device, quantizeToInt8: false);

 // agent
 var agent = new Phi3Agent(pipeline, "assistant")
Expand Down

2 changes: 1 addition & 1 deletion
docs/samples/Microsoft.ML.GenAI.Samples/Phi3Mini/Utils.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,7 +20,7 @@ public static ICausalLMPipeline<Tokenizer, Phi3ForCasualLM> LoadPhi3Mini4KFromFo
 string weightFolder,
 string configName = "config.json",
 string device = "cuda",
 int modelSizeOnCudaInGB = 16,
Comment thread
LittleLittleCloud marked this conversation as resolved.

 int modelSizeOnCudaInGB = 55,
 int modelSizeOnMemoryInGB = 64,
 int modelSizeOnDiskInGB = 200,
 bool quantizeToInt8 = false,
Expand Down

2 changes: 1 addition & 1 deletion
eng/Versions.props

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -96,7 +96,7 @@
 <MicrosoftMLTensorFlowTestModelsVersion>0.0.13-test</MicrosoftMLTensorFlowTestModelsVersion>
 <MicrosoftMLTestDatabasesVersion>0.0.6-test</MicrosoftMLTestDatabasesVersion>
 <MicrosoftMLTestModelsVersion>0.0.7-test</MicrosoftMLTestModelsVersion>
 <MicrosoftMLTestTokenizersVersion>2.0.0-beta.24219.1</MicrosoftMLTestTokenizersVersion>
 <MicrosoftMLTestTokenizersVersion>2.0.0-beta.24415.1</MicrosoftMLTestTokenizersVersion>
 <SystemDataSqlClientVersion>4.8.6</SystemDataSqlClientVersion>
 <SystemDataSQLiteCoreVersion>1.0.118</SystemDataSQLiteCoreVersion>
 <XunitCombinatorialVersion>1.6.24</XunitCombinatorialVersion>
Expand Down

51 changes: 51 additions & 0 deletions
src/Microsoft.ML.GenAI.Core/Extension/ModuleExtension.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -197,6 +197,57 @@ public static Dictionary<string, string> InferDeviceMapForEachLayer(
 return deviceMap;
 }

 /// <summary>
 /// Infer the device map for each layer in the model.
 /// The device map is a dictionary where the key is the device id (e.g. "cuda:0") and the value is the memory size in bytes of the device.
 /// When inferring the device map, each layer in the model will be placed on the device in the order of the devices list.
 /// </summary>
 /// <param name="model"></param>
 /// <param name="numberOfLayerToBePlaced">a list of key-value pairs where the key is the device id (e.g. "cuda:0") and the value is the number of layers to be placed on the device.
 /// If you want to place all remaining layers on the device, set that value to -1.
 /// e.g. [{"cuda:0", 2}, {"cpu", -1}], the first 2 layers will be placed on "cuda:0" and the rest will be placed on "cpu".
 /// </param>
 /// <returns></returns>
 public static Dictionary<string, string> InferDeviceMapForEachLayer(
 this nn.Module model,
 IEnumerable<KeyValuePair<string, int>> numberOfLayerToBePlaced)
 {
 var layerSizeMap = model.GetSizeForEachDynamicLayerInBytes()
 .OrderByDescending(x => x.Value)
 .ToList();

 var deviceMap = new Dictionary<string, string>();
 foreach (var (device, count) in numberOfLayerToBePlaced)
 {
 if (count != -1)
 {
 var topK = layerSizeMap.Take(count).ToList();
 layerSizeMap = layerSizeMap.Skip(count).ToList();
 foreach (var (key, value) in topK)
 {
 deviceMap[key] = device;
 }
 }
 else
 {
 foreach (var (key, value) in layerSizeMap)
 {
 deviceMap[key] = device;
 }

 layerSizeMap.Clear();
 break;
 }
 }

 if (layerSizeMap.Count > 0)
 {
 throw new ArgumentException("The layer count is not enough to cover all layers, did you forget to set the last layer count to -1?");
 }

 return deviceMap;
 }

 internal static string Peek(this nn.Module model)
 {
 var sb = new StringBuilder();
Expand Down

11 changes: 4 additions & 7 deletions
src/Microsoft.ML.GenAI.Core/Microsoft.ML.GenAI.Core.csproj

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -8,23 +8,20 @@
 </PropertyGroup>

 <ItemGroup>
 <PackageReference Include="AutoGen.Core" Version="$(AutoGenVersion)" />
 <PackageReference Include="Microsoft.SemanticKernel.Abstractions" Version="$(SemanticKernelVersion)" />
 <PackageReference Include="System.Memory" Version="$(SystemMemoryVersion)" />
 <PackageReference Include="TorchSharp" Version="$(TorchSharpVersion)" />
 </ItemGroup>
<!-- 

 <ItemGroup Condition="'$(Configuration)' == 'Debug'">
 <PackageReference Include="libtorch-cpu-win-x64" Version="$(LibTorchVersion)" Condition="$([MSBuild]::IsOSPlatform('Windows'))" PrivateAssets="all" />
 <PackageReference Include="libtorch-cpu-linux-x64" Version="$(LibTorchVersion)" Condition="$([MSBuild]::IsOSPlatform('Linux'))" PrivateAssets="all" />
 <PackageReference Include="libtorch-cpu-osx-x64" Version="$(LibTorchVersion)" Condition="$([MSBuild]::IsOSPlatform('OSX'))" PrivateAssets="all" />
 </ItemGroup> -->

 <ItemGroup>
 <ProjectReference Include="..\Microsoft.ML.Tokenizers\Microsoft.ML.Tokenizers.csproj" />
 </ItemGroup>

 <ItemGroup>
 <InternalsVisibleTo Include="Microsoft.ML.GenAI.Phi" />
 <InternalsVisibleTo Include="Microsoft.ML.GenAI.LLaMA" />
 <InternalsVisibleTo Include="Microsoft.ML.GenAI.LLaMA.Tests" />
 <InternalsVisibleTo Include="Microsoft.ML.GenAI.Phi.Tests" />
 <InternalsVisibleTo Include="Microsoft.ML.GenAI.Core.Tests" />
 </ItemGroup>
Expand Down
Loading