Building Autonomous AI Agents with CrewAI and Next.js

Introduction

As artificial intelligence shifts from standalone prompt engineering to autonomous, multi-agent orchestrations, developers are increasingly looking for ways to integrate autonomous AI workflows directly into modern web applications.

In this article, we explore how to build and integrate a multi-agent system powered by CrewAI and Python into a Next.js 16 (App Router) platform, enabling real-time background task execution, Server-Sent Event (SSE) streaming, and automated content publishing.

---

Architecture Overview

Combining a Python AI agent execution layer with a Next.js frontend delivers the best of both worlds:

1. Python & CrewAI: Leverages specialized AI agents with distinct roles, goals, and backstories to execute sequential research and content drafting tasks.
2. Next.js Server-Sent Events (SSE): Streams agent stdout execution logs and status progress bars to the web browser in real time without blocking the UI thread.
3. Automated Publishing Pipeline: Pushes finalized markdown content with structured YAML frontmatter directly into the repository filesystem, instantly making newly generated articles live on the public site.

┌────────────────────────┐      POST /api/agent       ┌────────────────────────┐
│  Next.js CMS Web UI    │ ─────────────────────────> │ Next.js API Route      │
│  (React 19 / Client)   │ <───────────────────────── │ (Server-Sent Events)   │
└────────────────────────┘     Real-Time Stream       └───────────┬────────────┘
                                                                  │ spawn process
                                                                  ▼
┌────────────────────────┐    Creates .md Article     ┌────────────────────────┐
│ Public Portfolio Blog  │ <───────────────────────── │ CrewAI Python System   │
│ (content/blog/*.md)    │                            │ (Researcher & Writer)  │
└────────────────────────┘                            └────────────────────────┘

---

1. Defining Multi-Agent Roles in CrewAI

CrewAI enables defining teams of AI agents that collaborate sequentially or hierarchically. Here is how we configure a Senior Content Researcher and a Professional Tech Blogger:

from crewai import Agent, Crew, Process, Task, LLM

# Initialize Local or Remote LLM
llm = LLM(
    model="ollama/llama3",
    base_url="http://localhost:11434",
    api_key="NA"
)

# 1. Senior Content Researcher Agent
researcher = Agent(
    role="Senior Content Researcher",
    goal="Find key angles, technical insights, and outline guidelines based on topic.",
    backstory="An expert web researcher who understands audience engagement and modern SEO strategies.",
    verbose=False,
    llm=llm,
)

# 2. Professional Tech Blogger Agent
writer = Agent(
    role="Professional Tech Blogger",
    goal="Write engaging, SEO-optimized technical blog posts formatted in clean Markdown.",
    backstory="You write clear, compelling, and well-structured articles avoiding fluff and delivering high reader value.",
    verbose=False,
    llm=llm,
)

---

2. Orchestrating Sequential Execution Tasks

Next, we tie the agents together into structured tasks. The output of the research task feeds directly into the writing task:

def run_blog_agent(topic, use_case, category="Technical"):
    output_filepath = f"content/blog/{slugify(topic)}.md"

    research_task = Task(
        description=f"Topic: {topic}\nGoal: {use_case}\nOutline target audience hooks and H2/H3 subheadings.",
        expected_output="A comprehensive outline with technical points.",
        agent=researcher,
    )

    writing_task = Task(
        description=f"Using the outline, draft a complete blog post formatted in Markdown satisfying use case: '{use_case}'.",
        expected_output="A fully written blog post in Markdown format.",
        agent=writer,
        output_file=output_filepath,
    )

    crew = Crew(
        agents=[researcher, writer],
        tasks=[research_task, writing_task],
        process=Process.sequential,
        verbose=True,
    )

    crew.kickoff()

---

3. Real-Time SSE Streaming in Next.js

To provide a sleek user experience, the Next.js API route spawns the Python process and streams terminal outputs to the browser using ReadableStream:

import { NextRequest } from "next/server";
import { spawn } from "child_process";
import path from "path";

export async function POST(req: NextRequest) {
  const { topic, use_case, category } = await req.json();
  const encoder = new TextEncoder();

  const stream = new ReadableStream({
    start(controller) {
      const sendEvent = (data: Record<string, any>) => {
        controller.enqueue(encoder.encode(`data: ${JSON.stringify(data)}\n\n`));
      };

      const child = spawn("python", ["agent.py", "--topic", topic, "--use_case", use_case, "--category", category]);

      child.stdout.on("data", (chunk: Buffer) => {
        const text = chunk.toString("utf-8");
        sendEvent({ type: "log", text });
      });

      child.on("close", (code) => {
        if (code === 0) {
          sendEvent({ type: "complete", topic, text: "Generation finished!" });
        }
        controller.close();
      });
    },
  });

  return new Response(stream, {
    headers: {
      "Content-Type": "text/event-stream",
      "Cache-Control": "no-cache",
    },
  });
}

---

4. Automatic Publishing & Markdown Rendering

Once the Python agent completes writing the file, the Next.js static site generator or dynamic file reader loads the newly created Markdown file without requiring manual server restarts or database migrations:

// lib/posts.ts
import fs from "fs";
import matter from "gray-matter";

export function getPostBySlug(slug: string) {
  const filePath = path.join(process.cwd(), "content/blog", `${slug}.md`);
  const fileContent = fs.readFileSync(filePath, "utf-8");
  const { data, content } = matter(fileContent);

  return {
    title: data.title,
    date: data.date,
    category: data.category,
    summary: data.summary,
    content,
  };
}

---

Conclusion

Combining CrewAI multi-agent systems with Next.js empowers engineers to automate research, generation, and publishing workflows directly inside modern web applications. By streaming agent progress in real time via SSE and persisting Markdown files directly to disk, you achieve an efficient, autonomous, and self-publishing content pipeline.