如何通过azure hdinsight上的c#运行Yarn作业?

nwlqm0z1  于 2021-06-04  发布在  Hadoop
关注(0)|答案(0)|浏览(283)

因此,目前,我一直在使用以下代码将mapreduce作业提交到hdinsight集群。我想将在yarn上运行的作业提交到同一集群。我应该如何修改此代码以执行相同的操作?你能推荐一些指导我的教程吗?谢谢!

namespace Project2
{

class Program
{
    static void Main(string[] args)
    {
        // Azure subscription and HDInsight cluster 
        string subscriptionID = "something"; 
        string clusterName = "hdinsightsample1";
        string certfriendlyname = "something"; 
        // HDInsight blob storage account
        string storageAccountName = "bangsql";
        string storageAccountKey = "something==";
        string containerName = "hdinsightsample1";
        // HDInsight job output
        string downloadFile = "example/data/WordCountOutput/part-r-00000";
        // Get the Azure certificate
        Console.WriteLine(DateTime.Now.ToLongTimeString() + "\tGetting the Windows Azure certificate from the Certificate store");
        X509Certificate2 azureCert = GetAzureCertificate(certfriendlyname);
        // Submit the HDInsight job and wait for the job completion
        Console.WriteLine(DateTime.Now.ToLongTimeString() + "\tSubmitting the HDInsight job and start execution");
        Task t = SubmitHadoopJob(subscriptionID, clusterName, azureCert);
        try
        {
            t.Wait();
        }
        catch (Exception e)
        {
            Console.WriteLine(e.Message + " " + e.InnerException);
        }
        // Print the MapReduce job output
        Console.WriteLine(DateTime.Now.ToLongTimeString() + "\tGetting the output of the HDInsight job");

        GetHDInsightJobResult(storageAccountName, storageAccountKey, containerName, downloadFile);
        Console.WriteLine("Job completed");
        Console.ReadKey();
    }*/

    private static X509Certificate2 GetAzureCertificate(string certfriendlyname)
    {
        // Get certificate object from certificate store using the friendly name to 
        // identify it
        X509Certificate2 azureCert = null;
        X509Store store = new X509Store();
        store.Open(OpenFlags.ReadOnly);
        X509Certificate2Collection certCollection = store.Certificates.Find(X509FindType.FindByIssuerName, "Windows Azure Tools", false);
        // Find the certificate required
        foreach (X509Certificate2 cert in certCollection)
        {
            if (cert.FriendlyName.Equals(certfriendlyname))
            {
                azureCert = cert;
                break;
            }
        }
        return azureCert;
    }

    private static async System.Threading.Tasks.Task SubmitHadoopJob(string subscriptionID, string clusterName, X509Certificate2 azureCert)
    {
    JobSubmissionCertificateCredential creds = new JobSubmissionCertificateCredential(new Guid(subscriptionID), azureCert, clusterName);
    // Define the MapReduce job
    MapReduceJobCreateParameters mrJobDefinition = new MapReduceJobCreateParameters()
    {
    JarFile = "wasb:///example/jars/hadoop-mapreduce-examples.jar",
    ClassName = "wordcount"
    };
    mrJobDefinition.Arguments.Add("wasb:///example/data/harrypotter.txt");
    mrJobDefinition.Arguments.Add("wasb:///example/data/WordCountOutput");
    // Create a Hadoop client to connect to HDInsight
    var jobClient = JobSubmissionClientFactory.Connect(creds);
    // Run the MapReduce job 
    JobCreationResults mrJobResults = await jobClient.CreateMapReduceJobAsync(mrJobDefinition);
    }

    private static void GetHDInsightJobResult(string storageAccountName, string storageAccountKey, string containerName, string downloadFile)
    {
        Stream stream = new MemoryStream();
        CloudStorageAccount storageAccount = CloudStorageAccount.Parse("DefaultEndpointsProtocol=https;AccountName=" + storageAccountName + ";AccountKey=" + storageAccountKey);
        CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
        CloudBlobContainer blobContainer = blobClient.GetContainerReference(containerName);
        CloudBlockBlob blockBlob = blobContainer.GetBlockBlobReference(downloadFile);
        try
        {
            blockBlob.DownloadToStream(stream);
        }
        catch (Exception e)
        {
            Console.WriteLine(e.InnerException);
        }
            stream.Position = 0;
        StreamReader reader = new StreamReader(stream);
        Console.WriteLine(reader.ReadToEnd());
    }

}

}

暂无答案!

目前还没有任何答案,快来回答吧!

相关问题