configured json log collection

Houssem Dellai · Houssem Dellai · commit 96b61461ea79 · 2025-10-12T09:20:40.000+02:00
diff --git a/.infracost/pricing.gob b/.infracost/pricing.gob
diff --git a/.infracost/terraform_modules/manifest-a04f9d1714a0a471f764336a6a9e8ed9.json b/.infracost/terraform_modules/manifest-a04f9d1714a0a471f764336a6a9e8ed9.json
@@ -0,0 +1 @@
+{"Path":"d:\\projects\\azure-monitoring-course\\110_vm_monitoring_ama_vnet_flow_logs","Version":"2.0","Modules":[]}
diff --git a/.infracost/terraform_modules/manifest-bd696e7cc82c1c0a104d9092961beae4.json b/.infracost/terraform_modules/manifest-bd696e7cc82c1c0a104d9092961beae4.json
@@ -0,0 +1 @@
+{"Path":"d:\\projects\\azure-monitoring-course\\100_vm_monitoring_ama","Version":"2.0","Modules":[]}
diff --git a/100_vm_monitoring_ama/.terraform.lock.hcl b/100_vm_monitoring_ama/.terraform.lock.hcl
diff --git a/100_vm_monitoring_ama/dcr-linux.tf b/100_vm_monitoring_ama/dcr-linux.tf
@@ -4,6 +4,10 @@ resource "azurerm_monitor_data_collection_rule" "dcr_linux" {
   location            = azurerm_resource_group.rg.location
   kind                = "Linux"
 
+  identity {
+    type = "SystemAssigned"
+  }
+
   destinations {
     log_analytics {
       workspace_resource_id = azurerm_log_analytics_workspace.law.id
@@ -26,6 +30,29 @@ resource "azurerm_monitor_data_collection_rule" "dcr_linux" {
     # }
   }
 
+  data_flow {
+    streams       = ["Custom-Json-MyApplication_CL"]
+    destinations  = ["destination-log"]
+    output_stream = "Custom-MyApplication_CL"
+    transform_kql = "source"
+    #     transform_kql = <<EOT
+    #     source
+    #     | project TimeGenerated = now(),
+    #               Computer = "",
+    #               FilePath = "",
+    #               Message = "",
+    #               Level = "",
+    #               SourceLine = "",
+    #               ThreadId = 0,
+    #               RawData = "",
+    #               FixedValue = ""
+    #   EOT
+    # transform_kql = "source | project TimeGenerated = Timestamp, ThreadId, SourceLine, Level, Message, FixedValue"
+    # transform_kql = "source | project TimeGenerated = now()" # "source | project TimeGenerated = Time, Computer, Message = AdditionalContext"
+    # transform_kql = "source | project TimeGenerated = now() | project LogMessage = 'RawData'" # "source | project TimeGenerated = Time, Computer, Message = AdditionalContext"
+    # transform_kql = "source | project d = split(RawData,",") | project TimeGenerated=todatetime(d[0]), Code=toint(d[1]), Severity=tostring(d[2]), Module=tostring(d[3]), Message=tostring(d[4])"
+  }
+
   data_flow {
     streams      = ["Microsoft-Syslog"]
     destinations = ["destination-log"]
@@ -45,20 +72,16 @@ resource "azurerm_monitor_data_collection_rule" "dcr_linux" {
     }
 
     log_file {
-      name          = "datasource-logfile"
-      format        = "text"
-      streams       = ["Custom-MyTableRawData"]
-      file_patterns = ["C:\\JavaLogs\\*.log"]
-      settings {
-        text {
-          record_start_timestamp_format = "ISO 8601"
-        }
-      }
+      name          = "Custom-Json-MyApplication_CL"
+      format        = "json" # "text"
+      streams       = ["Custom-Json-MyApplication_CL"]
+      file_patterns = ["/var/log/myapplication.log"]
     }
 
     performance_counter {
       name    = "CustomPerfCounters"
       streams = ["Microsoft-Perf"]
+      sampling_frequency_in_seconds = 30
       counter_specifiers = [
         "Processor(*)\\% Processor Time",
         "Processor(*)\\% Idle Time",
@@ -74,7 +97,42 @@ resource "azurerm_monitor_data_collection_rule" "dcr_linux" {
         "System(*)\\Unique Users",
         "System(*)\\CPUs"
       ]
-      sampling_frequency_in_seconds = 30
+    }
+  }
+
+  stream_declaration {
+    stream_name = "Custom-Json-MyApplication_CL"
+    column {
+      name = "TimeGenerated"
+      type = "datetime"
+    }
+    column {
+      name = "Computer"
+      type = "string"
+    }
+    column {
+      name = "FilePath"
+      type = "string"
+    }
+    column {
+      name = "Level"
+      type = "string"
+    }
+    column {
+      name = "LogMessage"
+      type = "string"
+    }
+    column {
+      name = "MachineName"
+      type = "string"
+    }
+    column {
+      name = "MachineIP"
+      type = "string"
+    }
+    column {
+      name = "FixedValue"
+      type = "string"
     }
   }
 }
diff --git a/100_vm_monitoring_ama/dcr-vm-insights.tf b/100_vm_monitoring_ama/dcr-vm-insights.tf
@@ -4,6 +4,10 @@ resource "azurerm_monitor_data_collection_rule" "dcr_vm_insights" {
   location            = azurerm_resource_group.rg.location
   kind                = "Linux"
 
+  identity {
+    type = "SystemAssigned"
+  }
+
   destinations {
     log_analytics {
       workspace_resource_id = azurerm_log_analytics_workspace.law.id
diff --git a/100_vm_monitoring_ama/dcr-windows.tf b/100_vm_monitoring_ama/dcr-windows.tf
@@ -4,6 +4,10 @@ resource "azurerm_monitor_data_collection_rule" "dcr_windows" {
   location            = azurerm_resource_group.rg.location
   kind                = "Windows"
 
+  identity {
+    type = "SystemAssigned"
+  }
+
   destinations {
     log_analytics {
       workspace_resource_id = azurerm_log_analytics_workspace.law.id
diff --git a/100_vm_monitoring_ama/generate-logs.sh b/100_vm_monitoring_ama/generate-logs.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+sudo bash -c '
+i=0
+while true; do
+    echo "{
+        \"TimeGenerated\":\"$(date)\",
+        \"Level\":\"Info\",
+        \"LogMessage\":\"This is demo log $i\",
+        \"MachineName\":\"$(hostname)\",
+        \"MachineIP\":\"$(hostname -i)\"
+    }" >> /var/log/myapplication.log
+    i=$((i + 1))
+    sleep 1
+done
+'
diff --git a/100_vm_monitoring_ama/log_analytics-custom-table.tf b/100_vm_monitoring_ama/log_analytics-custom-table.tf
@@ -0,0 +1,50 @@
+
+resource "azapi_resource" "table" {
+  type      = "Microsoft.OperationalInsights/workspaces/tables@2025-02-01"
+  parent_id = azurerm_log_analytics_workspace.law.id
+  name      = "MyApplication_CL"
+  body = {
+    properties = {
+      plan            = "Analytics"
+      retentionInDays = 30
+      schema = {
+        name = "MyApplication_CL"
+        columns = [
+          {
+            name = "TimeGenerated"
+            type = "datetime"
+          },
+          {
+            name = "Computer"
+            type = "string"
+          },
+          {
+            name = "FilePath"
+            type = "string"
+          },
+          {
+            name = "Level"
+            type = "string"
+          },
+          {
+            name = "LogMessage"
+            type = "string"
+          },
+          {
+            name = "MachineName"
+            type = "string"
+          },
+          {
+            name = "MachineIP"
+            type = "string"
+          },
+          {
+            name = "FixedValue"
+            type = "string"
+          }
+        ]
+      }
+      totalRetentionInDays = 30
+    }
+  }
+}
diff --git a/100_vm_monitoring_ama/log_analytics.tf b/100_vm_monitoring_ama/log_analytics.tf
@@ -1,5 +1,5 @@
 resource "azurerm_log_analytics_workspace" "law" {
-  name                = "log-analytics"
+  name                = "log-analytics-${var.prefix}"
   resource_group_name = azurerm_resource_group.rg.name
   location            = azurerm_resource_group.rg.location
   sku                 = "PerGB2018" # PerGB2018, Free, PerNode, Premium, Standard, Standalone, Unlimited, CapacityReservation
diff --git a/100_vm_monitoring_ama/providers.tf b/100_vm_monitoring_ama/providers.tf
@@ -7,6 +7,10 @@ terraform {
       source  = "hashicorp/azurerm"
       version = ">= 4.44.0"
     }
+    azapi = {
+      source  = "Azure/azapi"
+      version = ">= 2.7.0"
+    }
   }
 }
 
@@ -18,4 +22,6 @@ provider "azurerm" {
       prevent_deletion_if_contains_resources = false
     }
   }
-}
+}
+
+provider "azapi" {}
diff --git a/100_vm_monitoring_ama/variables.tf b/100_vm_monitoring_ama/variables.tf
@@ -1,4 +1,4 @@
 variable "prefix" {
   type        = string
-  default     = "060"
+  default     = "100"
 }
diff --git a/100_vm_monitoring_ama/vm-linux.tf b/100_vm_monitoring_ama/vm-linux.tf
@@ -23,7 +23,7 @@ resource "azurerm_linux_virtual_machine" "vm_linux" {
   name                            = "vm-linux"
   resource_group_name             = azurerm_resource_group.rg.name
   location                        = azurerm_resource_group.rg.location
-  size                            = "Standard_D4ads_v6"
+  size                            = "Standard_D2ads_v6"
   disable_password_authentication = false
   admin_username                  = "azureuser"
   admin_password                  = "@Aa123456789"
@@ -32,7 +32,7 @@ resource "azurerm_linux_virtual_machine" "vm_linux" {
   eviction_policy                 = "Delete"
   disk_controller_type            = "NVMe" # "SCSI" # "IDE" # "SCSI" is the default value. "NVMe" is only supported for Ephemeral OS Disk.
 
-  # custom_data = filebase64("./install-mitmproxy-with-wireguard.sh")
+  custom_data = filebase64("./generate-logs.sh")
 
   identity {
     type = "SystemAssigned"
@@ -42,7 +42,7 @@ resource "azurerm_linux_virtual_machine" "vm_linux" {
     name                 = "os-disk-linux"
     caching              = "ReadOnly"        # "ReadWrite" # None, ReadOnly and ReadWrite.
     storage_account_type = "StandardSSD_LRS" # "Standard_LRS"
-    disk_size_gb         = 128
+    disk_size_gb         = 64
 
     diff_disk_settings {
       option    = "Local"    # Specifies the Ephemeral Disk Settings for the OS Disk. At this time the only possible value is Local.

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+{"Path":"d:\\projects\\azure-monitoring-course\\110_vm_monitoring_ama_vnet_flow_logs","Version":"2.0","Modules":[]}`